From 36036fec8b84302c084f000fdf3f83f904e4df09 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 27 Nov 2020 10:32:20 +0100 Subject: [PATCH 01/25] Add constexpr functions for chunk arithmetic Fixes #137. --- include/global/chunk_calc_util.hpp | 202 ++++++++++++++++++++--------- 1 file changed, 139 insertions(+), 63 deletions(-) diff --git a/include/global/chunk_calc_util.hpp b/include/global/chunk_calc_util.hpp index bbe14622e..770ee6adb 100644 --- a/include/global/chunk_calc_util.hpp +++ b/include/global/chunk_calc_util.hpp @@ -14,112 +14,188 @@ #ifndef GEKKOFS_CHNK_CALC_UTIL_HPP #define GEKKOFS_CHNK_CALC_UTIL_HPP +#include +#include #include namespace gkfs::util { /** - * Compute the base2 logarithm for 64 bit integers + * Check whether integer `n` is a power of 2. + * + * @param [in] n the number to check. + * @returns `true` if `n` is a power of 2; `false` otherwise. */ -inline int -log2(uint64_t n) { +constexpr bool +is_power_of_2(uint64_t n) { + return n && (!(n & (n - 1u))); +} - /* see - * http://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers - */ - static const int table[64] = { - 0, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, 33, 42, 3, 61, - 51, 37, 40, 49, 18, 28, 20, 55, 30, 34, 11, 43, 14, 22, 4, 62, - 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, 56, - 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5, 63}; - - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n |= n >> 32; - - return table[(n * 0x03f6eaf2cd271461) >> 58]; +/** + * Compute the base2 logarithm for 64 bit integers. + * + * @param [in] n the number from which to compute the log2. + * @returns the base 2 logarithm of `n`. + */ +constexpr std::size_t +log2(uint64_t n) { + return 8u * sizeof(uint64_t) - __builtin_clzll(n) - 1; } +/** + * Check whether @n is divisible by @block_size. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] n the number to check. + * @param [in] block_size + * @returns true if @n is divisible by @block_size; false otherwise. + */ +constexpr bool +is_divisible(const uint64_t n, const size_t block_size) { + using gkfs::util::log2; + assert(is_power_of_2(block_size)); + return !(n & ((1u << log2(block_size)) - 1)); +} /** - * Align an @offset to the closest left side chunk boundary + * Given a file @offset and a @block_size, align the @offset to its + * closest left-side block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset to align. + * @param [in] block_size the block size used to compute boundaries. + * @returns an offset aligned to the left-side block boundary. */ -inline off64_t -chnk_lalign(const off64_t offset, const size_t chnk_size) { - return offset & ~(chnk_size - 1); +constexpr uint64_t +chnk_lalign(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return static_cast(offset) & ~(block_size - 1u); } /** - * Align an @offset to the closest right side chunk boundary + * Given a file @offset and a @block_size, align the @offset to its + * closest right-side block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset to align. + * @param [in] block_size the block size used to compute boundaries. + * @returns an offset aligned to the right-side block boundary. */ -inline off64_t -chnk_ralign(const off64_t offset, const size_t chnk_size) { - return chnk_lalign(offset + chnk_size, chnk_size); +constexpr uint64_t +chnk_ralign(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return chnk_lalign(offset, block_size) + block_size; } /** - * Return the padding (bytes) that separates the @offset from the closest - * left side chunk boundary + * Return the overrun bytes that separate @offset from the closest left side + * block boundary. * - * If @offset is a boundary the resulting padding will be 0 + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset for which the overrun distance should be + * computed. + * @param [in] block_size the block size used to compute boundaries. + * @returns the distance in bytes between the left-side boundary of @offset */ -inline size_t -chnk_lpad(const off64_t offset, const size_t chnk_size) { - return offset % chnk_size; +constexpr size_t +chnk_lpad(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return static_cast(offset) & (block_size - 1u); } /** - * Return the padding (bytes) that separates the @offset from the closest - * right side chunk boundary + * Return the underrun bytes that separate @offset from the closest right side + * block boundary. + * + * @note This function assumes that block_size is a power of 2. * - * If @offset is a boundary the resulting padding will be 0 + * @param [in] offset the offset for which the overrun distance should be + * computed. + * @param [in] block_size the block size used to compute boundaries. + * @returns the distance in bytes between the right-side boundary of @offset */ -inline size_t -chnk_rpad(const off64_t offset, const size_t chnk_size) { - return (-offset) % chnk_size; +constexpr size_t +chnk_rpad(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return chnk_ralign(offset, block_size) - offset; } /** - * Given an @offset calculates the chunk number to which the @offset belongs + * Given an @offset and a @block_size, compute the block index to which @offset + * belongs. * - * chunk_id(8,4) = 2; - * chunk_id(7,4) = 1; - * chunk_id(2,4) = 0; - * chunk_id(0,4) = 0; + * @note Block indexes are (conceptually) computed by dividing @offset + * by @block_size, with index 0 referring to block [0, block_size - 1], + * index 1 to block [block_size, 2 * block_size - 1], and so on up to + * a maximum index FILE_LENGTH / block_size. + * + * @note This function assumes that @block_size is a power of 2. + * + * @param [in] offset the offset for which the block index should be computed. + * @param [in] block_size the block_size that should be used to compute the + * index. + * @returns the index of the block containing @offset. */ -inline uint64_t -chnk_id_for_offset(const off64_t offset, const size_t chnk_size) { - /* - * This does not work for offsets that use the 64th bit, i.e., - * 9223372036854775808. 9223372036854775808 - 1 uses 63 bits and still - * works. `offset / chnk_size` works with the 64th bit. With this number we - * can address more than 19,300,000 exabytes of data though. Hi future me? - */ - return static_cast(chnk_lalign(offset, chnk_size) >> - log2(chnk_size)); +constexpr uint64_t +chnk_id_for_offset(const uint64_t offset, const size_t block_size) { + + using gkfs::util::log2; + + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return static_cast(chnk_lalign(offset, block_size) >> + log2(block_size)); } /** - * Return the number of chunks involved in an operation that operates - * from @offset for a certain amount of bytes (@count). + * Compute the number of blocks involved in an operation affecting the + * regions from [@offset, to @offset + @count). + * + * @note This function assumes that @block_size is a power of 2. + * @note This function assumes that @offset + @count does not + * overflow. + * + * @param [in] offset the operation's initial offset. + * @param [in] count the number of bytes affected by the operation. + * @param [in] chnk_size the block size that should be used to compute the + * number of blocks. + * @returns the number of blocks affected by the operation. */ -inline uint64_t -chnk_count_for_offset(const off64_t offset, const size_t count, +constexpr std::size_t +chnk_count_for_offset(const uint64_t offset, const size_t count, const size_t chnk_size) { - off64_t chnk_start = chnk_lalign(offset, chnk_size); - off64_t chnk_end = chnk_lalign(offset + count - 1, chnk_size); + using gkfs::util::log2; + + // These checks are automatically removed in release builds + assert(is_power_of_2(chnk_size)); + +#if defined(__GNUC__) && !defined(__clang__) + assert(!__builtin_add_overflow_p(offset, count, static_cast(0))); +#else + assert(offset + count > offset); +#endif + + const uint64_t chnk_start = chnk_lalign(offset, chnk_size); + const uint64_t chnk_end = chnk_lalign(offset + count, chnk_size); + const size_t mask = -!!count; // this is either 0 or ~0 - return static_cast((chnk_end >> log2(chnk_size)) - - (chnk_start >> log2(chnk_size)) + 1); + return (((chnk_end >> log2(chnk_size)) - (chnk_start >> log2(chnk_size)) + + !is_divisible(offset + count, chnk_size))) & + mask; } } // namespace gkfs::util -- GitLab From 2e752ad394ea0efec75deaced9c8c03fed84c64a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 27 Nov 2020 13:10:11 +0100 Subject: [PATCH 02/25] Add tests for chnk_lalign() --- tests/unit/CMakeLists.txt | 1 + tests/unit/test_util_numeric.cpp | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 tests/unit/test_util_numeric.cpp diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 08a6f139e..315c61418 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -34,6 +34,7 @@ target_link_libraries(catch2_main add_executable(tests test_example_00.cpp test_example_01.cpp + test_util_numeric.cpp ) target_link_libraries(tests diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp new file mode 100644 index 000000000..6ea1f0477 --- /dev/null +++ b/tests/unit/test_util_numeric.cpp @@ -0,0 +1,72 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +#include +#include +#include + +constexpr auto test_repetitions = 250u; + +SCENARIO(" offsets can be left-aligned to block size boundaries ", + "[utils][numeric][chnk_lalign]") { + + using namespace gkfs::util; + + GIVEN(" a block size ") { + + const std::size_t block_size = + GENERATE(filter([](uint64_t bs) { return is_power_of_2(bs); }, + range(0, 100000))); + + WHEN(" offset is 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is 0 ") { + const uint64_t aligned_offset = chnk_lalign(offset, block_size); + REQUIRE(aligned_offset == 0); + } + } + + WHEN(" offset is smaller than block size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is 0 ") { + const uint64_t aligned_offset = chnk_lalign(offset, block_size); + REQUIRE(aligned_offset == 0); + } + } + + WHEN(" offset is larger than block size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is the left boundary of the " + "containing block ") { + const uint64_t aligned_offset = chnk_lalign(offset, block_size); + const uint64_t exp_offset = + static_cast(offset / block_size) * block_size; + REQUIRE(aligned_offset == exp_offset); + } + } + } +} -- GitLab From 842dd32f74a3d06465970c044637d4d20c98fb0a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 27 Nov 2020 13:10:33 +0100 Subject: [PATCH 03/25] Add tests for chnk_ralign() --- tests/unit/test_util_numeric.cpp | 56 ++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 6ea1f0477..837653112 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -70,3 +70,59 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", } } } + +SCENARIO(" offsets can be right-aligned to block size boundaries ", + "[utils][numeric][chnk_ralign]") { + + using namespace gkfs::util; + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is block_size ") { + const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t expected_offset = block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + + WHEN(" offset is smaller than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is 0 ") { + const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t expected_offset = block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is the right boundary of the " + "containing block ") { + const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t expected_offset = + static_cast(offset / block_size + 1) * + block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + } +} -- GitLab From bfa149b023a0b81c41660ecdb123d6e31b696a16 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 27 Nov 2020 18:31:06 +0100 Subject: [PATCH 04/25] Add tests for is_power_of_2() --- tests/unit/test_util_numeric.cpp | 83 ++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 837653112..cd83e5410 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -15,13 +15,90 @@ #include #include +using namespace gkfs::util; constexpr auto test_repetitions = 250u; +namespace { + +/** + * Check if @n is a power of two by (rather inefficiently) + * performing successive divisions by 2 in an attempt to reach 1. + * + * @param n the number to check + * @returns true if @n is a power of 2, false otherwise + */ +bool +check_power_of_2(uint64_t n) { + if(n == 0) { + return false; + } + + while(n != 1) { + if(n % 2 != 0) { + return false; + } + n /= 2; + } + + return true; +} + +} // namespace + + +SCENARIO(" powers of 2 can be correctly detected ", + "[utils][numeric][is_power_of_2]") { + + GIVEN(" a positive number ") { + + WHEN(" n is 0 ") { + + const uint64_t n = 0; + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == false); + } + } + + WHEN(" n is 1 ") { + + const uint64_t n = 1; + + THEN(" is_power_of_2(n) returns true ") { + REQUIRE(is_power_of_2(n) == true); + } + } + + WHEN(" n is neither 0 nor 1 ") { + + AND_WHEN(" n is a power of 2 ") { + + const std::size_t n = GENERATE( + filter([](uint64_t m) { return check_power_of_2(m); }, + range(0, 10000))); + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == true); + } + } + + AND_WHEN(" n is not a power of 2 ") { + + const std::size_t n = GENERATE( + filter([](uint64_t m) { return !check_power_of_2(m); }, + range(0, 10000))); + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == false); + } + } + } + } +} + SCENARIO(" offsets can be left-aligned to block size boundaries ", "[utils][numeric][chnk_lalign]") { - using namespace gkfs::util; - GIVEN(" a block size ") { const std::size_t block_size = @@ -74,8 +151,6 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", SCENARIO(" offsets can be right-aligned to block size boundaries ", "[utils][numeric][chnk_ralign]") { - using namespace gkfs::util; - GIVEN(" a block size ") { const std::size_t block_size = GENERATE(filter( -- GitLab From 97880220c617218af305018b980dd59a6d3f640f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 28 Nov 2020 12:37:10 +0100 Subject: [PATCH 05/25] Add tests for chnk_lpad() --- tests/unit/test_util_numeric.cpp | 83 ++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index cd83e5410..0f37b5843 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -201,3 +201,86 @@ SCENARIO(" offsets can be right-aligned to block size boundaries ", } } } + +SCENARIO(" overrun distance can be computed correctly ", + "[utils][numeric][chnk_lpad]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals 0 ") { + const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t expected_overrun = 0; + REQUIRE(overrun == expected_overrun); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals offset ") { + const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t expected_overrun = offset; + REQUIRE(overrun == expected_overrun); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals block_size - 1 ") { + const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t expected_overrun = block_size - 1; + REQUIRE(overrun == expected_overrun); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals 0 ") { + const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t expected_overrun = 0; + REQUIRE(overrun == expected_overrun); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals the difference between " + "offset and its closest block's left boundary ") { + const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t expected_overrun = + offset - + static_cast(offset / block_size) * block_size; + REQUIRE(overrun == expected_overrun); + } + } + } +} -- GitLab From 00e96a89904a4e44d2c7f56f30e72cf3e7ef2f1f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sun, 29 Nov 2020 16:52:07 +0100 Subject: [PATCH 06/25] Add tests for chnk_rpad() --- tests/unit/test_util_numeric.cpp | 84 ++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 0f37b5843..542dcdb34 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -284,3 +284,87 @@ SCENARIO(" overrun distance can be computed correctly ", } } } + +SCENARIO(" underrun distance can be computed correctly ", + "[utils][numeric][chnk_rpad]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size ") { + const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t expected_underrun = block_size; + REQUIRE(underrun == expected_underrun); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals offset ") { + const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t expected_underrun = block_size - offset; + REQUIRE(underrun == expected_underrun); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size - 1 ") { + const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t expected_underrun = block_size - offset; + REQUIRE(underrun == expected_underrun); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size ") { + const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t expected_underrun = block_size; + REQUIRE(underrun == expected_underrun); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals the difference between " + "offset and its closest block's right boundary ") { + const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t expected_underrun = + static_cast(offset / block_size + 1) * + block_size - + offset; + REQUIRE(underrun == expected_underrun); + } + } + } +} -- GitLab From 7d9f477cd3d80892d4029b7a97fb42cda05a727d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 30 Nov 2020 13:23:07 +0100 Subject: [PATCH 07/25] Add tests for chnk_id_for_offset() --- tests/unit/test_util_numeric.cpp | 120 +++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 542dcdb34..b8026bccc 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -368,3 +368,123 @@ SCENARIO(" underrun distance can be computed correctly ", } } } + +SCENARIO(" chunk IDs can be computed correctly ", + "[utils][numeric][chnk_id_for_offset]") { + + GIVEN(" an offset and a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 1 ") { + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = 1; + REQUIRE(id == expected_id); + } + } + + WHEN(" offset is larger than block_size ") { + + AND_WHEN(" block_size < offset < 2^63 - 1 ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, + random(block_size, + std::numeric_limits::max() / 2 - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + + // The following test specifically exercises issue #137 + AND_WHEN(" offset == 2^63 ") { + + const uint64_t offset = + std::numeric_limits::max() / 2 + 1; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + + // The following test specifically exercises issue #137 + AND_WHEN(" offset == 2^64 - 1") { + + const uint64_t offset = std::numeric_limits::max(); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + } + } +} -- GitLab From dfa36a4669771bcbca52bd48629368e43f034ef9 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 2 Dec 2020 15:51:12 +0100 Subject: [PATCH 08/25] Add tests for is_divisible() --- tests/unit/test_util_numeric.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index b8026bccc..17d6f3aa3 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -96,6 +96,24 @@ SCENARIO(" powers of 2 can be correctly detected ", } } +SCENARIO(" divisibility by powers of 2 can be correctly detected ", + "[utils][numeric][is_divisible]") { + + GIVEN(" a number and a block_size ") { + + const uint64_t n = GENERATE(range(0, 1000), range(20000, 23000), + std::numeric_limits::max()); + const std::size_t block_size = + GENERATE(filter([](uint64_t bs) { return is_power_of_2(bs); }, + range(0, 10000))); + + CAPTURE(n, block_size); + + bool expected = n % block_size == 0; + REQUIRE(is_divisible(n, block_size) == expected); + } +} + SCENARIO(" offsets can be left-aligned to block size boundaries ", "[utils][numeric][chnk_lalign]") { -- GitLab From f7e6a3d817e6dbc6881bd52e348f594c326d2ecb Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 2 Dec 2020 15:57:14 +0100 Subject: [PATCH 09/25] Add tests for chnk_count_for_offset() --- tests/unit/test_util_numeric.cpp | 456 +++++++++++++++++++++++++++++++ 1 file changed, 456 insertions(+) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 17d6f3aa3..e1223fc9e 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -506,3 +506,459 @@ SCENARIO(" chunk IDs can be computed correctly ", } } } + +SCENARIO(" the number of chunks involved in an operation can be computed " + "correctly ", + "[utils][numeric][chnk_count_for_offset]") { + + GIVEN(" an offset, an operation size, and a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset < block_size ") { + + AND_WHEN(" offset == 0 ") { + + const uint64_t offset = 0; + + AND_WHEN(" offset + size == 0 ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 0 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" 0 < offset + size < block_size ") { + + const size_t size = GENERATE_COPY(take( + test_reps, random(std::size_t{1}, block_size))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" 0 < offset + size < block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, block_size - offset))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count equals 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = block_size - offset; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == block_size - 1 ") { + + const size_t offset = block_size - 1; + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 0 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 1; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + } + + WHEN(" offset == block_size ") { + + const uint64_t offset = block_size; + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE(range(0u, test_reps)); + const size_t size = m * block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == M ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY(take( + test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + WHEN(" offset > block_size ") { + + AND_WHEN(" block_size < offset < 2^63 - 1 ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, + random(block_size, + std::numeric_limits::max() / 2 - 1))); + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE_COPY(range(2u, test_reps)); + const size_t size = m * block_size - (offset % block_size); + + CAPTURE(offset, size, block_size, m); + + THEN(" the computed block count == M ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - + offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == 2^63 ") { + + const uint64_t offset = + std::numeric_limits::max() / 2 + 1; + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE(range(0u, test_reps)); + const size_t size = m * block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == M ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - + offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == 2^64 - 1 ") { + + const uint64_t offset = std::numeric_limits::max(); + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + chnk_count_for_offset(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 ") { + // TODO: here we should check that we actually hit an + // assert(), but Catch2 does not have facilities to + // support this yet + } + } + } + } +} -- GitLab From 8bbeb81ce0de229c85c63731013816ef515f50a5 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 2 Dec 2020 15:57:31 +0100 Subject: [PATCH 10/25] Reduce test repetitions --- tests/unit/test_util_numeric.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index e1223fc9e..250c3bc53 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -16,7 +16,7 @@ #include using namespace gkfs::util; -constexpr auto test_repetitions = 250u; +constexpr auto test_reps = 200u; namespace { -- GitLab From 1e226b24d9273a8ef1e64c237a35dcc7a09a58d2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 3 Dec 2020 17:46:01 +0100 Subject: [PATCH 11/25] Rename: gkfs::util -> gkfs::utils --- include/client/logging.hpp | 2 +- include/client/make_array.hpp | 4 ++-- include/client/preload_util.hpp | 4 ++-- include/client/syscalls/args.hpp | 16 ++++++------- include/daemon/util.hpp | 4 ++-- include/global/chunk_calc_util.hpp | 10 ++++---- src/client/gkfs_functions.cpp | 28 +++++++++++----------- src/client/hooks.cpp | 2 +- src/client/logging.cpp | 2 +- src/client/open_file_map.cpp | 16 ++++++------- src/client/preload.cpp | 8 +++---- src/client/preload_util.cpp | 4 ++-- src/client/rpc/forward_data.cpp | 24 +++++++++---------- src/daemon/daemon.cpp | 4 ++-- src/daemon/handler/srv_data.cpp | 4 ++-- src/daemon/ops/data.cpp | 4 ++-- src/daemon/util.cpp | 4 ++-- tests/integration/harness/gkfs.io/main.cpp | 2 +- tests/unit/test_util_numeric.cpp | 2 +- 19 files changed, 72 insertions(+), 72 deletions(-) diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 2225f08d1..db769d648 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -106,7 +106,7 @@ static const auto constexpr most = log_level::print_most; static const auto constexpr all = log_level::print_all; static const auto constexpr help = log_level::print_help; -static const auto constexpr level_names = util::make_array( +static const auto constexpr level_names = utils::make_array( "syscall", "syscall", // sycall_entry uses the same name as syscall "info", "critical", "error", "warning", "hermes", "mercury", "debug"); diff --git a/include/client/make_array.hpp b/include/client/make_array.hpp index 2d1618c60..ffd5ef3b3 100644 --- a/include/client/make_array.hpp +++ b/include/client/make_array.hpp @@ -16,7 +16,7 @@ #include -namespace gkfs::util { +namespace gkfs::utils { template constexpr auto @@ -28,6 +28,6 @@ make_array(T&&... values) -> std::array< sizeof...(T)>{std::forward(values)...}; } -} // namespace gkfs::util +} // namespace gkfs::utils #endif // LIBGKFS_UTILS_MAKE_ARRAY_HPP diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index f42fcd979..c8dc9fcfd 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -48,7 +48,7 @@ class async_engine; extern std::unique_ptr ld_network_service; // function definitions -namespace gkfs::util { +namespace gkfs::utils { template constexpr typename std::underlying_type::type to_underlying(E e) { @@ -74,6 +74,6 @@ read_hosts_file(); void connect_to_hosts(const std::vector>& hosts); -} // namespace gkfs::util +} // namespace gkfs::utils #endif // GEKKOFS_PRELOAD_UTIL_HPP diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index e2829626c..3df34ea73 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -291,7 +291,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for lseek() whence arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SEEK_SET), FLAG_ENTRY(SEEK_CUR), FLAG_ENTRY(SEEK_END) @@ -314,7 +314,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for mmap() prot arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(PROT_NONE), FLAG_ENTRY(PROT_READ), FLAG_ENTRY(PROT_WRITE), @@ -339,7 +339,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for mmap() flags arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(MAP_SHARED), FLAG_ENTRY(MAP_PRIVATE), #ifdef MAP_SHARED_VALIDATE @@ -379,7 +379,7 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for clone() flags arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(CLONE_VM), FLAG_ENTRY(CLONE_FS), FLAG_ENTRY(CLONE_FILES), @@ -431,7 +431,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for signum args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SIGHUP), FLAG_ENTRY(SIGINT), FLAG_ENTRY(SIGQUIT), @@ -485,7 +485,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for sigproc how args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SIG_BLOCK), FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); @@ -572,13 +572,13 @@ format_open_flags_to(FmtBuffer& buffer, /* Names for O_ACCMODE args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(O_RDONLY), FLAG_ENTRY(O_WRONLY), FLAG_ENTRY(O_RDWR)); const auto extra_flag_names = - util::make_array( + utils::make_array( #ifdef O_EXEC FLAG_ENTRY(O_EXEC), #endif diff --git a/include/daemon/util.hpp b/include/daemon/util.hpp index c8845847a..6e2c7cea8 100644 --- a/include/daemon/util.hpp +++ b/include/daemon/util.hpp @@ -15,13 +15,13 @@ #define GEKKOFS_DAEMON_UTIL_HPP namespace gkfs { -namespace util { +namespace utils { void populate_hosts_file(); void destroy_hosts_file(); -} // namespace util +} // namespace utils } // namespace gkfs #endif // GEKKOFS_DAEMON_UTIL_HPP diff --git a/include/global/chunk_calc_util.hpp b/include/global/chunk_calc_util.hpp index 770ee6adb..753eae15b 100644 --- a/include/global/chunk_calc_util.hpp +++ b/include/global/chunk_calc_util.hpp @@ -18,7 +18,7 @@ #include #include -namespace gkfs::util { +namespace gkfs::utils { /** * Check whether integer `n` is a power of 2. @@ -53,7 +53,7 @@ log2(uint64_t n) { */ constexpr bool is_divisible(const uint64_t n, const size_t block_size) { - using gkfs::util::log2; + using gkfs::utils::log2; assert(is_power_of_2(block_size)); return !(n & ((1u << log2(block_size)) - 1)); } @@ -151,7 +151,7 @@ chnk_rpad(const uint64_t offset, const size_t block_size) { constexpr uint64_t chnk_id_for_offset(const uint64_t offset, const size_t block_size) { - using gkfs::util::log2; + using gkfs::utils::log2; // This check is automatically removed in release builds assert(is_power_of_2(block_size)); @@ -178,7 +178,7 @@ constexpr std::size_t chnk_count_for_offset(const uint64_t offset, const size_t count, const size_t chnk_size) { - using gkfs::util::log2; + using gkfs::utils::log2; // These checks are automatically removed in release builds assert(is_power_of_2(chnk_size)); @@ -198,6 +198,6 @@ chnk_count_for_offset(const uint64_t offset, const size_t count, mask; } -} // namespace gkfs::util +} // namespace gkfs::utils #endif diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 4ffb1c90a..1884817e3 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -84,7 +84,7 @@ int check_parent_dir(const std::string& path) { #if CREATE_CHECK_PARENTS auto p_comp = gkfs::path::dirname(path); - auto md = gkfs::util::get_metadata(p_comp); + auto md = gkfs::utils::get_metadata(p_comp); if(!md) { if(errno == ENOENT) { LOG(DEBUG, "Parent component does not exist: '{}'", p_comp); @@ -130,7 +130,7 @@ gkfs_open(const std::string& path, mode_t mode, int flags) { } bool exists = true; - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { if(errno == ENOENT) { exists = false; @@ -253,7 +253,7 @@ gkfs_create(const std::string& path, mode_t mode) { */ int gkfs_remove(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { return -1; } @@ -276,7 +276,7 @@ gkfs_remove(const std::string& path) { */ int gkfs_access(const std::string& path, const int mask, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { errno = ENOENT; return -1; @@ -294,11 +294,11 @@ gkfs_access(const std::string& path, const int mask, bool follow_links) { */ int gkfs_stat(const string& path, struct stat* buf, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { return -1; } - gkfs::util::metadata_to_stat(path, *md, *buf); + gkfs::utils::metadata_to_stat(path, *md, *buf); return 0; } @@ -318,14 +318,14 @@ gkfs_stat(const string& path, struct stat* buf, bool follow_links) { int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, struct statx* buf, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { return -1; } struct stat tmp {}; - gkfs::util::metadata_to_stat(path, *md, tmp); + gkfs::utils::metadata_to_stat(path, *md, tmp); buf->stx_mask = 0; buf->stx_blksize = tmp.st_blksize; @@ -545,7 +545,7 @@ gkfs_truncate(const std::string& path, off_t length) { return -1; } - auto md = gkfs::util::get_metadata(path, true); + auto md = gkfs::utils::get_metadata(path, true); if(!md) { return -1; } @@ -863,7 +863,7 @@ gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { int gkfs_opendir(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { return -1; } @@ -891,7 +891,7 @@ gkfs_opendir(const std::string& path) { */ int gkfs_rmdir(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { LOG(DEBUG, "Path '{}' does not exist: ", path); errno = ENOENT; @@ -1088,7 +1088,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { * Here if the target is a directory we raise a NOTSUP error. * So that application know we don't support link to directory. */ - auto target_md = gkfs::util::get_metadata(target_path, false); + auto target_md = gkfs::utils::get_metadata(target_path, false); if(target_md != nullptr) { auto trg_mode = target_md->mode(); if(!(S_ISREG(trg_mode) || S_ISLNK(trg_mode))) { @@ -1103,7 +1103,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { return -1; } - auto link_md = gkfs::util::get_metadata(path, false); + auto link_md = gkfs::utils::get_metadata(path, false); if(link_md != nullptr) { LOG(DEBUG, "Link exists: '{}'", path); errno = EEXIST; @@ -1130,7 +1130,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { */ int gkfs_readlink(const std::string& path, char* buf, int bufsize) { - auto md = gkfs::util::get_metadata(path, false); + auto md = gkfs::utils::get_metadata(path, false); if(md == nullptr) { LOG(DEBUG, "Named link doesn't exist"); return -1; diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 0dec98e74..59d98d79b 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -614,7 +614,7 @@ hook_chdir(const char* path) { bool internal = CTX->relativize_path(path, rel_path); if(internal) { // path falls in our namespace - auto md = gkfs::util::get_metadata(rel_path); + auto md = gkfs::utils::get_metadata(rel_path); if(md == nullptr) { LOG(ERROR, "{}() path does not exists", __func__); return -ENOENT; diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 613f4e18c..1f21f290f 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -40,7 +40,7 @@ struct opt_info { #define STR_AND_LEN(strbuf) strbuf, sizeof(strbuf) - 1 -static const auto constexpr debug_opts = util::make_array( +static const auto constexpr debug_opts = utils::make_array( opt_info{STR_AND_LEN("none"), {"don't print any messages"}, log::none}, diff --git a/src/client/open_file_map.cpp b/src/client/open_file_map.cpp index f88b3eccb..6ecec24dc 100644 --- a/src/client/open_file_map.cpp +++ b/src/client/open_file_map.cpp @@ -31,17 +31,17 @@ OpenFile::OpenFile(const string& path, const int flags, FileType type) : type_(type), path_(path) { // set flags to OpenFile if(flags & O_CREAT) - flags_[gkfs::util::to_underlying(OpenFile_flags::creat)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::creat)] = true; if(flags & O_APPEND) - flags_[gkfs::util::to_underlying(OpenFile_flags::append)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::append)] = true; if(flags & O_TRUNC) - flags_[gkfs::util::to_underlying(OpenFile_flags::trunc)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::trunc)] = true; if(flags & O_RDONLY) - flags_[gkfs::util::to_underlying(OpenFile_flags::rdonly)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::rdonly)] = true; if(flags & O_WRONLY) - flags_[gkfs::util::to_underlying(OpenFile_flags::wronly)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::wronly)] = true; if(flags & O_RDWR) - flags_[gkfs::util::to_underlying(OpenFile_flags::rdwr)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::rdwr)] = true; pos_ = 0; // If O_APPEND flag is used, it will be used before each write. } @@ -73,13 +73,13 @@ OpenFile::pos(unsigned long pos) { bool OpenFile::get_flag(OpenFile_flags flag) { lock_guard lock(pos_mutex_); - return flags_[gkfs::util::to_underlying(flag)]; + return flags_[gkfs::utils::to_underlying(flag)]; } void OpenFile::set_flag(OpenFile_flags flag, bool value) { lock_guard lock(flag_mutex_); - flags_[gkfs::util::to_underlying(flag)] = value; + flags_[gkfs::utils::to_underlying(flag)] = value; } FileType diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 812c54c56..8792ae10b 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -97,7 +97,7 @@ init_ld_environment_() { vector> hosts{}; try { LOG(INFO, "Loading peer addresses..."); - hosts = gkfs::util::read_hosts_file(); + hosts = gkfs::utils::read_hosts_file(); } catch(const std::exception& e) { exit_error_msg(EXIT_FAILURE, "Failed to load hosts addresses: "s + e.what()); @@ -111,7 +111,7 @@ init_ld_environment_() { } try { - gkfs::util::connect_to_hosts(hosts); + gkfs::utils::connect_to_hosts(hosts); } catch(const std::exception& e) { exit_error_msg(EXIT_FAILURE, "Failed to connect to hosts: "s + e.what()); @@ -120,7 +120,7 @@ init_ld_environment_() { /* Setup distributor */ #ifdef GKFS_ENABLE_FORWARDING try { - gkfs::util::load_forwarding_map(); + gkfs::utils::load_forwarding_map(); LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); } catch(std::exception& e) { @@ -160,7 +160,7 @@ forwarding_mapper(void* p) { while(forwarding_running) { try { - gkfs::util::load_forwarding_map(); + gkfs::utils::load_forwarding_map(); if(previous != CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index c88a10c95..d4cc99488 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -163,7 +163,7 @@ load_hostfile(const std::string& path) { } // namespace -namespace gkfs::util { +namespace gkfs::utils { /** * Retrieve metadata from daemon @@ -398,4 +398,4 @@ connect_to_hosts(const vector>& hosts) { CTX->hosts(addrs); } -} // namespace gkfs::util \ No newline at end of file +} // namespace gkfs::utils \ No newline at end of file diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 2961673ea..1fcb25d52 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -55,9 +55,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, off64_t offset = append_flag ? in_offset : (updated_metadentry_size - write_size); - auto chnk_start = gkfs::util::chnk_id_for_offset( + auto chnk_start = gkfs::utils::chnk_id_for_offset( offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::util::chnk_id_for_offset( + auto chnk_end = gkfs::utils::chnk_id_for_offset( (offset + write_size) - 1, gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so @@ -126,12 +126,12 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { total_chunk_size -= - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize); + gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= gkfs::util::chnk_rpad( + total_chunk_size -= gkfs::utils::chnk_rpad( offset + write_size, gkfs::config::rpc::chunksize); } @@ -145,7 +145,7 @@ forward_write(const string& path, const void* buf, const bool append_flag, path, // first offset in targets is the chunk with // a potential offset - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize), + gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), @@ -230,9 +230,9 @@ forward_read(const string& path, void* buf, const off64_t offset, // Calculate chunkid boundaries and numbers so that daemons know in which // interval to look for chunks - auto chnk_start = gkfs::util::chnk_id_for_offset( + auto chnk_start = gkfs::utils::chnk_id_for_offset( offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::util::chnk_id_for_offset( + auto chnk_end = gkfs::utils::chnk_id_for_offset( (offset + read_size - 1), gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so @@ -301,12 +301,12 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { total_chunk_size -= - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize); + gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= gkfs::util::chnk_rpad( + total_chunk_size -= gkfs::utils::chnk_rpad( offset + read_size, gkfs::config::rpc::chunksize); } @@ -320,7 +320,7 @@ forward_read(const string& path, void* buf, const off64_t offset, path, // first offset in targets is the chunk with // a potential offset - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize), + gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), @@ -404,9 +404,9 @@ forward_truncate(const std::string& path, size_t current_size, // Find out which data servers need to delete data chunks in order to // contact only them - const unsigned int chunk_start = gkfs::util::chnk_id_for_offset( + const unsigned int chunk_start = gkfs::utils::chnk_id_for_offset( new_size, gkfs::config::rpc::chunksize); - const unsigned int chunk_end = gkfs::util::chnk_id_for_offset( + const unsigned int chunk_end = gkfs::utils::chnk_id_for_offset( current_size - new_size - 1, gkfs::config::rpc::chunksize); std::unordered_set hosts; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index b8d120131..f8b7f9928 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -258,7 +258,7 @@ init_environment() { } // setup hostfile to let clients know that a daemon is running on this host if(!GKFS_DATA->hosts_file().empty()) { - gkfs::util::populate_hosts_file(); + gkfs::utils::populate_hosts_file(); } GKFS_DATA->spdlogger()->info("Startup successful. Daemon is ready."); } @@ -301,7 +301,7 @@ destroy_enviroment() { if(!GKFS_DATA->hosts_file().empty()) { GKFS_DATA->spdlogger()->debug("{}() Removing hosts file", __func__); try { - gkfs::util::destroy_hosts_file(); + gkfs::utils::destroy_hosts_file(); } catch(const fs::filesystem_error& e) { GKFS_DATA->spdlogger()->debug("{}() hosts file not found", __func__); diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index a66d1748e..01a1a8bed 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -460,7 +460,7 @@ rpc_srv_read(hg_handle_t handle) { origin_offsets[chnk_id_curr] = 0; bulk_buf_ptrs[chnk_id_curr] = chnk_ptr; chnk_sizes[chnk_id_curr] = offset_transfer_size; - // util variables + // utils variables chnk_ptr += offset_transfer_size; chnk_size_left_host -= offset_transfer_size; } else { @@ -481,7 +481,7 @@ rpc_srv_read(hg_handle_t handle) { transfer_size = chnk_size_left_host; bulk_buf_ptrs[chnk_id_curr] = chnk_ptr; chnk_sizes[chnk_id_curr] = transfer_size; - // util variables + // utils variables chnk_ptr += transfer_size; chnk_size_left_host -= transfer_size; } diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 741bd7d09..3be55fef0 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -49,11 +49,11 @@ ChunkTruncateOperation::truncate_abt(void* _arg) { int err_response = 0; try { // get chunk from where to cut off - auto chunk_id_start = gkfs::util::chnk_id_for_offset( + auto chunk_id_start = gkfs::utils::chnk_id_for_offset( size, gkfs::config::rpc::chunksize); // do not last delete chunk if it is in the middle of a chunk auto left_pad = - gkfs::util::chnk_lpad(size, gkfs::config::rpc::chunksize); + gkfs::utils::chnk_lpad(size, gkfs::config::rpc::chunksize); if(left_pad != 0) { GKFS_DATA->storage()->truncate_chunk_file(path, chunk_id_start, left_pad); diff --git a/src/daemon/util.cpp b/src/daemon/util.cpp index 4e6fede35..6a8cc0d34 100644 --- a/src/daemon/util.cpp +++ b/src/daemon/util.cpp @@ -20,7 +20,7 @@ using namespace std; -namespace gkfs::util { +namespace gkfs::utils { void populate_hosts_file() { @@ -48,4 +48,4 @@ destroy_hosts_file() { std::remove(GKFS_DATA->hosts_file().c_str()); } -} // namespace gkfs::util +} // namespace gkfs::utils diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index b04eb2a86..b82233d3c 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -41,7 +41,7 @@ init_commands(CLI::App& app) { write_validate_init(app); write_random_init(app); truncate_init(app); - // util + // utils file_compare_init(app); } diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 250c3bc53..601a73c06 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -15,7 +15,7 @@ #include #include -using namespace gkfs::util; +using namespace gkfs::utils; constexpr auto test_reps = 200u; namespace { -- GitLab From 845b17f1a6675cf28a84e6e1b22206a003c832a0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 18:15:54 +0100 Subject: [PATCH 12/25] Move pow2 arithmetic functions to own namespace --- include/global/chunk_calc_util.hpp | 10 +++--- src/client/rpc/forward_data.cpp | 51 ++++++++++++++++-------------- src/daemon/ops/data.cpp | 11 ++++--- tests/unit/test_util_numeric.cpp | 2 +- 4 files changed, 41 insertions(+), 33 deletions(-) diff --git a/include/global/chunk_calc_util.hpp b/include/global/chunk_calc_util.hpp index 753eae15b..5d5314a9c 100644 --- a/include/global/chunk_calc_util.hpp +++ b/include/global/chunk_calc_util.hpp @@ -18,7 +18,7 @@ #include #include -namespace gkfs::utils { +namespace gkfs::utils::arithmetic { /** * Check whether integer `n` is a power of 2. @@ -53,7 +53,7 @@ log2(uint64_t n) { */ constexpr bool is_divisible(const uint64_t n, const size_t block_size) { - using gkfs::utils::log2; + using gkfs::utils::arithmetic::log2; assert(is_power_of_2(block_size)); return !(n & ((1u << log2(block_size)) - 1)); } @@ -151,7 +151,7 @@ chnk_rpad(const uint64_t offset, const size_t block_size) { constexpr uint64_t chnk_id_for_offset(const uint64_t offset, const size_t block_size) { - using gkfs::utils::log2; + using gkfs::utils::arithmetic::log2; // This check is automatically removed in release builds assert(is_power_of_2(block_size)); @@ -178,7 +178,7 @@ constexpr std::size_t chnk_count_for_offset(const uint64_t offset, const size_t count, const size_t chnk_size) { - using gkfs::utils::log2; + using gkfs::utils::arithmetic::log2; // These checks are automatically removed in release builds assert(is_power_of_2(chnk_size)); @@ -198,6 +198,6 @@ chnk_count_for_offset(const uint64_t offset, const size_t count, mask; } -} // namespace gkfs::utils +} // namespace gkfs::utils::arithmetic #endif diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 1fcb25d52..94620de25 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -48,6 +48,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, const off64_t in_offset, const size_t write_size, const int64_t updated_metadentry_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(write_size > 0); // Calculate chunkid boundaries and numbers so that daemons know in @@ -55,10 +58,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, off64_t offset = append_flag ? in_offset : (updated_metadentry_size - write_size); - auto chnk_start = gkfs::utils::chnk_id_for_offset( - offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::utils::chnk_id_for_offset( - (offset + write_size) - 1, gkfs::config::rpc::chunksize); + auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize); + auto chnk_end = chnk_id_for_offset((offset + write_size) - 1, + gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -125,14 +127,13 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { - total_chunk_size -= - gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize); + total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= gkfs::utils::chnk_rpad( - offset + write_size, gkfs::config::rpc::chunksize); + total_chunk_size -= chnk_rpad(offset + write_size, + gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); @@ -145,8 +146,8 @@ forward_write(const string& path, const void* buf, const bool append_flag, path, // first offset in targets is the chunk with // a potential offset - gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize), - target, CTX->hosts().size(), + chnk_lpad(offset, gkfs::config::rpc::chunksize), target, + CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), // chunk start id of this write @@ -228,12 +229,14 @@ pair forward_read(const string& path, void* buf, const off64_t offset, const size_t read_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + // Calculate chunkid boundaries and numbers so that daemons know in which // interval to look for chunks - auto chnk_start = gkfs::utils::chnk_id_for_offset( - offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::utils::chnk_id_for_offset( - (offset + read_size - 1), gkfs::config::rpc::chunksize); + auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize); + auto chnk_end = chnk_id_for_offset((offset + read_size - 1), + gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -300,14 +303,13 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { - total_chunk_size -= - gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize); + total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= gkfs::utils::chnk_rpad( - offset + read_size, gkfs::config::rpc::chunksize); + total_chunk_size -= + chnk_rpad(offset + read_size, gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); @@ -320,8 +322,8 @@ forward_read(const string& path, void* buf, const off64_t offset, path, // first offset in targets is the chunk with // a potential offset - gkfs::utils::chnk_lpad(offset, gkfs::config::rpc::chunksize), - target, CTX->hosts().size(), + chnk_lpad(offset, gkfs::config::rpc::chunksize), target, + CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), // chunk start id of this write @@ -400,13 +402,16 @@ int forward_truncate(const std::string& path, size_t current_size, size_t new_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(current_size > new_size); // Find out which data servers need to delete data chunks in order to // contact only them - const unsigned int chunk_start = gkfs::utils::chnk_id_for_offset( - new_size, gkfs::config::rpc::chunksize); - const unsigned int chunk_end = gkfs::utils::chnk_id_for_offset( + const unsigned int chunk_start = + chnk_id_for_offset(new_size, gkfs::config::rpc::chunksize); + const unsigned int chunk_end = chnk_id_for_offset( current_size - new_size - 1, gkfs::config::rpc::chunksize); std::unordered_set hosts; diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 3be55fef0..060421456 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -41,6 +41,10 @@ namespace data { */ void ChunkTruncateOperation::truncate_abt(void* _arg) { + + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(_arg); // Unpack args auto* arg = static_cast(_arg); @@ -49,11 +53,10 @@ ChunkTruncateOperation::truncate_abt(void* _arg) { int err_response = 0; try { // get chunk from where to cut off - auto chunk_id_start = gkfs::utils::chnk_id_for_offset( - size, gkfs::config::rpc::chunksize); + auto chunk_id_start = + chnk_id_for_offset(size, gkfs::config::rpc::chunksize); // do not last delete chunk if it is in the middle of a chunk - auto left_pad = - gkfs::utils::chnk_lpad(size, gkfs::config::rpc::chunksize); + auto left_pad = chnk_lpad(size, gkfs::config::rpc::chunksize); if(left_pad != 0) { GKFS_DATA->storage()->truncate_chunk_file(path, chunk_id_start, left_pad); diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index 601a73c06..bca37b20a 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -15,7 +15,7 @@ #include #include -using namespace gkfs::utils; +using namespace gkfs::utils::arithmetic; constexpr auto test_reps = 200u; namespace { -- GitLab From 23e26ad07b533df236160bd8a01ebc9d4e8b2c6d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 20:19:06 +0100 Subject: [PATCH 13/25] Add convenience library for optimized arithmetics --- .../{ => arithmetic}/chunk_calc_util.hpp | 0 src/client/CMakeLists.txt | 7 +++--- src/client/rpc/forward_data.cpp | 2 +- src/daemon/handler/srv_data.cpp | 2 +- src/daemon/ops/data.cpp | 2 +- src/global/CMakeLists.txt | 2 ++ src/global/arithmetic/CMakeLists.txt | 23 +++++++++++++++++++ tests/unit/test_util_numeric.cpp | 2 +- 8 files changed, 33 insertions(+), 7 deletions(-) rename include/global/{ => arithmetic}/chunk_calc_util.hpp (100%) create mode 100644 src/global/arithmetic/CMakeLists.txt diff --git a/include/global/chunk_calc_util.hpp b/include/global/arithmetic/chunk_calc_util.hpp similarity index 100% rename from include/global/chunk_calc_util.hpp rename to include/global/arithmetic/chunk_calc_util.hpp diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index e6488538e..3cf754729 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -42,7 +42,6 @@ set(PRELOAD_HEADERS ../../include/client/syscalls/syscall.hpp ../../include/client/syscalls/detail/syscall_info.h ../../include/global/cmake_configure.hpp - ../../include/global/chunk_calc_util.hpp ../../include/global/global_defs.hpp ../../include/global/path_util.hpp ../../include/global/rpc/rpc_types.hpp @@ -70,7 +69,9 @@ set(PRELOAD_INCLUDE_DIRS add_library(gkfs_intercept SHARED ${PRELOAD_SRC} ${PRELOAD_HEADERS}) -target_link_libraries(gkfs_intercept ${PRELOAD_LINK_LIBRARIES}) +target_link_libraries(gkfs_intercept + arithmetic + ${PRELOAD_LINK_LIBRARIES}) target_include_directories(gkfs_intercept PRIVATE ${PRELOAD_INCLUDE_DIRS}) @@ -115,4 +116,4 @@ if (GKFS_ENABLE_FORWARDING) ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -endif () \ No newline at end of file +endif () diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 94620de25..5637c6e67 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 01a1a8bed..8446f2388 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #ifdef GKFS_ENABLE_AGIOS #include diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 060421456..5e5a4bdc6 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include extern "C" { diff --git a/src/global/CMakeLists.txt b/src/global/CMakeLists.txt index 61455f69a..fb611cf9a 100644 --- a/src/global/CMakeLists.txt +++ b/src/global/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(arithmetic) + add_library(distributor STATIC) set_property(TARGET distributor PROPERTY POSITION_INDEPENDENT_CODE ON) target_sources(distributor diff --git a/src/global/arithmetic/CMakeLists.txt b/src/global/arithmetic/CMakeLists.txt new file mode 100644 index 000000000..186a0c9c1 --- /dev/null +++ b/src/global/arithmetic/CMakeLists.txt @@ -0,0 +1,23 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany +# +# This software was partially supported by the +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). +# +# This software was partially supported by the +# ADA-FS project under the SPPEXA project funded by the DFG. +# +# SPDX-License-Identifier: MIT +################################################################################ + +add_library(arithmetic INTERFACE) +target_sources(arithmetic + INTERFACE + ${INCLUDE_DIR}/global/arithmetic/chunk_calc_util.hpp + ) + +target_include_directories(arithmetic + INTERFACE + ${INCLUDE_DIR}/global/arithmetic/ + ) \ No newline at end of file diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_util_numeric.cpp index bca37b20a..0d7d970f1 100644 --- a/tests/unit/test_util_numeric.cpp +++ b/tests/unit/test_util_numeric.cpp @@ -12,7 +12,7 @@ */ #include -#include +#include #include using namespace gkfs::utils::arithmetic; -- GitLab From 36581c93d4158d9160ba25ad7a601aeb13880064 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 20:33:17 +0100 Subject: [PATCH 14/25] Rename: test_util_numeric.cpp -> test_utils_arithmetic.cpp --- tests/unit/CMakeLists.txt | 3 ++- .../unit/{test_util_numeric.cpp => test_utils_arithmetic.cpp} | 0 2 files changed, 2 insertions(+), 1 deletion(-) rename tests/unit/{test_util_numeric.cpp => test_utils_arithmetic.cpp} (100%) diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 315c61418..3109f9b47 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -34,12 +34,13 @@ target_link_libraries(catch2_main add_executable(tests test_example_00.cpp test_example_01.cpp - test_util_numeric.cpp + test_utils_arithmetic.cpp ) target_link_libraries(tests catch2_main fmt::fmt + arithmetic ) # Catch2's contrib folder includes some helper functions diff --git a/tests/unit/test_util_numeric.cpp b/tests/unit/test_utils_arithmetic.cpp similarity index 100% rename from tests/unit/test_util_numeric.cpp rename to tests/unit/test_utils_arithmetic.cpp -- GitLab From 30acbbca3853a2f48efbd79092701dc628a24ee2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 20:48:55 +0100 Subject: [PATCH 15/25] Rename: chunk_calc_util.hpp -> arithmetic.hpp --- .../arithmetic/{chunk_calc_util.hpp => arithmetic.hpp} | 6 +++--- src/client/rpc/forward_data.cpp | 2 +- src/daemon/handler/srv_data.cpp | 2 +- src/daemon/ops/data.cpp | 2 +- src/global/arithmetic/CMakeLists.txt | 4 ++-- tests/unit/test_utils_arithmetic.cpp | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) rename include/global/arithmetic/{chunk_calc_util.hpp => arithmetic.hpp} (98%) diff --git a/include/global/arithmetic/chunk_calc_util.hpp b/include/global/arithmetic/arithmetic.hpp similarity index 98% rename from include/global/arithmetic/chunk_calc_util.hpp rename to include/global/arithmetic/arithmetic.hpp index 5d5314a9c..be603d67e 100644 --- a/include/global/arithmetic/chunk_calc_util.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -11,8 +11,8 @@ SPDX-License-Identifier: MIT */ -#ifndef GEKKOFS_CHNK_CALC_UTIL_HPP -#define GEKKOFS_CHNK_CALC_UTIL_HPP +#ifndef GKFS_GLOBAL_ARITHMETIC_HPP +#define GKFS_GLOBAL_ARITHMETIC_HPP #include #include @@ -200,4 +200,4 @@ chnk_count_for_offset(const uint64_t offset, const size_t count, } // namespace gkfs::utils::arithmetic -#endif +#endif // GKFS_GLOBAL_ARITHMETIC_HPP \ No newline at end of file diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 5637c6e67..05964abcc 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index 8446f2388..027955c25 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #ifdef GKFS_ENABLE_AGIOS #include diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 5e5a4bdc6..7eed27f93 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include extern "C" { diff --git a/src/global/arithmetic/CMakeLists.txt b/src/global/arithmetic/CMakeLists.txt index 186a0c9c1..16c35079c 100644 --- a/src/global/arithmetic/CMakeLists.txt +++ b/src/global/arithmetic/CMakeLists.txt @@ -14,10 +14,10 @@ add_library(arithmetic INTERFACE) target_sources(arithmetic INTERFACE - ${INCLUDE_DIR}/global/arithmetic/chunk_calc_util.hpp + ${INCLUDE_DIR}/global/arithmetic/arithmetic.hpp ) target_include_directories(arithmetic INTERFACE ${INCLUDE_DIR}/global/arithmetic/ - ) \ No newline at end of file + ) diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index 0d7d970f1..cc21f763b 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -12,8 +12,8 @@ */ #include -#include #include +#include using namespace gkfs::utils::arithmetic; constexpr auto test_reps = 200u; -- GitLab From 6ed0c096152248009930a12c60e0d4a9c731093f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 20:54:38 +0100 Subject: [PATCH 16/25] Rename: chnk_lalign() -> align_left() --- include/global/arithmetic/arithmetic.hpp | 10 +++++----- tests/unit/test_utils_arithmetic.cpp | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index be603d67e..45758e017 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -69,7 +69,7 @@ is_divisible(const uint64_t n, const size_t block_size) { * @returns an offset aligned to the left-side block boundary. */ constexpr uint64_t -chnk_lalign(const uint64_t offset, const size_t block_size) { +align_left(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); return static_cast(offset) & ~(block_size - 1u); @@ -90,7 +90,7 @@ constexpr uint64_t chnk_ralign(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return chnk_lalign(offset, block_size) + block_size; + return align_left(offset, block_size) + block_size; } @@ -155,7 +155,7 @@ chnk_id_for_offset(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return static_cast(chnk_lalign(offset, block_size) >> + return static_cast(align_left(offset, block_size) >> log2(block_size)); } @@ -189,8 +189,8 @@ chnk_count_for_offset(const uint64_t offset, const size_t count, assert(offset + count > offset); #endif - const uint64_t chnk_start = chnk_lalign(offset, chnk_size); - const uint64_t chnk_end = chnk_lalign(offset + count, chnk_size); + const uint64_t chnk_start = align_left(offset, chnk_size); + const uint64_t chnk_end = align_left(offset + count, chnk_size); const size_t mask = -!!count; // this is either 0 or ~0 return (((chnk_end >> log2(chnk_size)) - (chnk_start >> log2(chnk_size)) + diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index cc21f763b..589008b79 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -115,7 +115,7 @@ SCENARIO(" divisibility by powers of 2 can be correctly detected ", } SCENARIO(" offsets can be left-aligned to block size boundaries ", - "[utils][numeric][chnk_lalign]") { + "[utils][numeric][align_left]") { GIVEN(" a block size ") { @@ -130,7 +130,7 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", CAPTURE(offset, block_size); THEN(" the left-aligned offset is 0 ") { - const uint64_t aligned_offset = chnk_lalign(offset, block_size); + const uint64_t aligned_offset = align_left(offset, block_size); REQUIRE(aligned_offset == 0); } } @@ -143,7 +143,7 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", CAPTURE(offset, block_size); THEN(" the left-aligned offset is 0 ") { - const uint64_t aligned_offset = chnk_lalign(offset, block_size); + const uint64_t aligned_offset = align_left(offset, block_size); REQUIRE(aligned_offset == 0); } } @@ -157,7 +157,7 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", THEN(" the left-aligned offset is the left boundary of the " "containing block ") { - const uint64_t aligned_offset = chnk_lalign(offset, block_size); + const uint64_t aligned_offset = align_left(offset, block_size); const uint64_t exp_offset = static_cast(offset / block_size) * block_size; REQUIRE(aligned_offset == exp_offset); -- GitLab From 6971fcf780eb7d77d7d577dfe76d8aad6d987fb4 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 4 Dec 2020 20:59:32 +0100 Subject: [PATCH 17/25] Rename: chnk_ralign() -> align_right() --- include/global/arithmetic/arithmetic.hpp | 4 ++-- tests/unit/test_utils_arithmetic.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index 45758e017..ec6218b6a 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -87,7 +87,7 @@ align_left(const uint64_t offset, const size_t block_size) { * @returns an offset aligned to the right-side block boundary. */ constexpr uint64_t -chnk_ralign(const uint64_t offset, const size_t block_size) { +align_right(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); return align_left(offset, block_size) + block_size; @@ -128,7 +128,7 @@ constexpr size_t chnk_rpad(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return chnk_ralign(offset, block_size) - offset; + return align_right(offset, block_size) - offset; } diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index 589008b79..6f6cb372c 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -167,7 +167,7 @@ SCENARIO(" offsets can be left-aligned to block size boundaries ", } SCENARIO(" offsets can be right-aligned to block size boundaries ", - "[utils][numeric][chnk_ralign]") { + "[utils][numeric][align_right]") { GIVEN(" a block size ") { @@ -181,7 +181,7 @@ SCENARIO(" offsets can be right-aligned to block size boundaries ", CAPTURE(offset, block_size); THEN(" the right-aligned offset is block_size ") { - const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t aligned_offset = align_right(offset, block_size); const uint64_t expected_offset = block_size; REQUIRE(aligned_offset == expected_offset); } @@ -195,7 +195,7 @@ SCENARIO(" offsets can be right-aligned to block size boundaries ", CAPTURE(offset, block_size); THEN(" the right-aligned offset is 0 ") { - const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t aligned_offset = align_right(offset, block_size); const uint64_t expected_offset = block_size; REQUIRE(aligned_offset == expected_offset); } @@ -210,7 +210,7 @@ SCENARIO(" offsets can be right-aligned to block size boundaries ", THEN(" the right-aligned offset is the right boundary of the " "containing block ") { - const uint64_t aligned_offset = chnk_ralign(offset, block_size); + const uint64_t aligned_offset = align_right(offset, block_size); const uint64_t expected_offset = static_cast(offset / block_size + 1) * block_size; -- GitLab From aec6cf9579f643c1b01d59a53cf38af9f80bfc9d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 5 Dec 2020 09:40:16 +0100 Subject: [PATCH 18/25] Rename: chnk_lpad() -> block_overrun() --- include/global/arithmetic/arithmetic.hpp | 2 +- src/client/rpc/forward_data.cpp | 10 ++++++---- src/daemon/ops/data.cpp | 2 +- tests/unit/test_utils_arithmetic.cpp | 12 ++++++------ 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index ec6218b6a..db66c8ed1 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -106,7 +106,7 @@ align_right(const uint64_t offset, const size_t block_size) { * @returns the distance in bytes between the left-side boundary of @offset */ constexpr size_t -chnk_lpad(const uint64_t offset, const size_t block_size) { +block_overrun(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); return static_cast(offset) & (block_size - 1u); diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 05964abcc..042ecee1d 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -127,7 +127,8 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { - total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize); + total_chunk_size -= + block_overrun(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract @@ -146,7 +147,7 @@ forward_write(const string& path, const void* buf, const bool append_flag, path, // first offset in targets is the chunk with // a potential offset - chnk_lpad(offset, gkfs::config::rpc::chunksize), target, + block_overrun(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), @@ -303,7 +304,8 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { - total_chunk_size -= chnk_lpad(offset, gkfs::config::rpc::chunksize); + total_chunk_size -= + block_overrun(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract @@ -322,7 +324,7 @@ forward_read(const string& path, void* buf, const off64_t offset, path, // first offset in targets is the chunk with // a potential offset - chnk_lpad(offset, gkfs::config::rpc::chunksize), target, + block_overrun(offset, gkfs::config::rpc::chunksize), target, CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 7eed27f93..b3bff78c9 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -56,7 +56,7 @@ ChunkTruncateOperation::truncate_abt(void* _arg) { auto chunk_id_start = chnk_id_for_offset(size, gkfs::config::rpc::chunksize); // do not last delete chunk if it is in the middle of a chunk - auto left_pad = chnk_lpad(size, gkfs::config::rpc::chunksize); + auto left_pad = block_overrun(size, gkfs::config::rpc::chunksize); if(left_pad != 0) { GKFS_DATA->storage()->truncate_chunk_file(path, chunk_id_start, left_pad); diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index 6f6cb372c..fd851cd7a 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -221,7 +221,7 @@ SCENARIO(" offsets can be right-aligned to block size boundaries ", } SCENARIO(" overrun distance can be computed correctly ", - "[utils][numeric][chnk_lpad]") { + "[utils][numeric][block_overrun]") { GIVEN(" a block size ") { @@ -237,7 +237,7 @@ SCENARIO(" overrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed overrun distance equals 0 ") { - const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t overrun = block_overrun(offset, block_size); const uint64_t expected_overrun = 0; REQUIRE(overrun == expected_overrun); } @@ -251,7 +251,7 @@ SCENARIO(" overrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed overrun distance equals offset ") { - const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t overrun = block_overrun(offset, block_size); const uint64_t expected_overrun = offset; REQUIRE(overrun == expected_overrun); } @@ -264,7 +264,7 @@ SCENARIO(" overrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed overrun distance equals block_size - 1 ") { - const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t overrun = block_overrun(offset, block_size); const uint64_t expected_overrun = block_size - 1; REQUIRE(overrun == expected_overrun); } @@ -278,7 +278,7 @@ SCENARIO(" overrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed overrun distance equals 0 ") { - const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t overrun = block_overrun(offset, block_size); const uint64_t expected_overrun = 0; REQUIRE(overrun == expected_overrun); } @@ -293,7 +293,7 @@ SCENARIO(" overrun distance can be computed correctly ", THEN(" the computed overrun distance equals the difference between " "offset and its closest block's left boundary ") { - const uint64_t overrun = chnk_lpad(offset, block_size); + const uint64_t overrun = block_overrun(offset, block_size); const uint64_t expected_overrun = offset - static_cast(offset / block_size) * block_size; -- GitLab From f00309d9db33ec047051de3bbae5ba388e6432a0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 5 Dec 2020 09:41:26 +0100 Subject: [PATCH 19/25] Rename: chnk_rpad() -> block_underrun() --- include/global/arithmetic/arithmetic.hpp | 2 +- src/client/rpc/forward_data.cpp | 8 ++++---- tests/unit/test_utils_arithmetic.cpp | 15 +++++++++------ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index db66c8ed1..d73b713da 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -125,7 +125,7 @@ block_overrun(const uint64_t offset, const size_t block_size) { * @returns the distance in bytes between the right-side boundary of @offset */ constexpr size_t -chnk_rpad(const uint64_t offset, const size_t block_size) { +block_underrun(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); return align_right(offset, block_size) - offset; diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 042ecee1d..52fe7fbc8 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -133,8 +133,8 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= chnk_rpad(offset + write_size, - gkfs::config::rpc::chunksize); + total_chunk_size -= block_underrun(offset + write_size, + gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); @@ -310,8 +310,8 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of last chunk must subtract if(target == chnk_end_target) { - total_chunk_size -= - chnk_rpad(offset + read_size, gkfs::config::rpc::chunksize); + total_chunk_size -= block_underrun(offset + read_size, + gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index fd851cd7a..d5d3ce952 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -304,7 +304,7 @@ SCENARIO(" overrun distance can be computed correctly ", } SCENARIO(" underrun distance can be computed correctly ", - "[utils][numeric][chnk_rpad]") { + "[utils][numeric][block_underrun]") { GIVEN(" a block size ") { @@ -320,7 +320,8 @@ SCENARIO(" underrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed underrun distance equals block_size ") { - const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t underrun = + block_underrun(offset, block_size); const uint64_t expected_underrun = block_size; REQUIRE(underrun == expected_underrun); } @@ -334,7 +335,8 @@ SCENARIO(" underrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed underrun distance equals offset ") { - const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t underrun = + block_underrun(offset, block_size); const uint64_t expected_underrun = block_size - offset; REQUIRE(underrun == expected_underrun); } @@ -347,7 +349,8 @@ SCENARIO(" underrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed underrun distance equals block_size - 1 ") { - const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t underrun = + block_underrun(offset, block_size); const uint64_t expected_underrun = block_size - offset; REQUIRE(underrun == expected_underrun); } @@ -361,7 +364,7 @@ SCENARIO(" underrun distance can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed underrun distance equals block_size ") { - const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t underrun = block_underrun(offset, block_size); const uint64_t expected_underrun = block_size; REQUIRE(underrun == expected_underrun); } @@ -376,7 +379,7 @@ SCENARIO(" underrun distance can be computed correctly ", THEN(" the computed underrun distance equals the difference between " "offset and its closest block's right boundary ") { - const uint64_t underrun = chnk_rpad(offset, block_size); + const uint64_t underrun = block_underrun(offset, block_size); const uint64_t expected_underrun = static_cast(offset / block_size + 1) * block_size - -- GitLab From 67a0ed2f98b8d4a332608c9f00f1d570cb06ee3a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 5 Dec 2020 10:59:16 +0100 Subject: [PATCH 20/25] Rename: chnk_id_for_offset() -> block_index() --- include/global/arithmetic/arithmetic.hpp | 2 +- src/client/rpc/forward_data.cpp | 18 +++++++++--------- src/daemon/ops/data.cpp | 3 +-- tests/unit/test_utils_arithmetic.cpp | 16 ++++++++-------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index d73b713da..22ff7c5de 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -149,7 +149,7 @@ block_underrun(const uint64_t offset, const size_t block_size) { * @returns the index of the block containing @offset. */ constexpr uint64_t -chnk_id_for_offset(const uint64_t offset, const size_t block_size) { +block_index(const uint64_t offset, const size_t block_size) { using gkfs::utils::arithmetic::log2; diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 52fe7fbc8..7a01074b4 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -58,9 +58,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, off64_t offset = append_flag ? in_offset : (updated_metadentry_size - write_size); - auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize); - auto chnk_end = chnk_id_for_offset((offset + write_size) - 1, - gkfs::config::rpc::chunksize); + auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); + auto chnk_end = block_index((offset + write_size) - 1, + gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -235,9 +235,9 @@ forward_read(const string& path, void* buf, const off64_t offset, // Calculate chunkid boundaries and numbers so that daemons know in which // interval to look for chunks - auto chnk_start = chnk_id_for_offset(offset, gkfs::config::rpc::chunksize); - auto chnk_end = chnk_id_for_offset((offset + read_size - 1), - gkfs::config::rpc::chunksize); + auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); + auto chnk_end = + block_index((offset + read_size - 1), gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -412,9 +412,9 @@ forward_truncate(const std::string& path, size_t current_size, // Find out which data servers need to delete data chunks in order to // contact only them const unsigned int chunk_start = - chnk_id_for_offset(new_size, gkfs::config::rpc::chunksize); - const unsigned int chunk_end = chnk_id_for_offset( - current_size - new_size - 1, gkfs::config::rpc::chunksize); + block_index(new_size, gkfs::config::rpc::chunksize); + const unsigned int chunk_end = block_index(current_size - new_size - 1, + gkfs::config::rpc::chunksize); std::unordered_set hosts; for(unsigned int chunk_id = chunk_start; chunk_id <= chunk_end; diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index b3bff78c9..b2fd78ae1 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -53,8 +53,7 @@ ChunkTruncateOperation::truncate_abt(void* _arg) { int err_response = 0; try { // get chunk from where to cut off - auto chunk_id_start = - chnk_id_for_offset(size, gkfs::config::rpc::chunksize); + auto chunk_id_start = block_index(size, gkfs::config::rpc::chunksize); // do not last delete chunk if it is in the middle of a chunk auto left_pad = block_overrun(size, gkfs::config::rpc::chunksize); if(left_pad != 0) { diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index d5d3ce952..58f95d766 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -391,7 +391,7 @@ SCENARIO(" underrun distance can be computed correctly ", } SCENARIO(" chunk IDs can be computed correctly ", - "[utils][numeric][chnk_id_for_offset]") { + "[utils][numeric][block_index]") { GIVEN(" an offset and a block size ") { @@ -407,7 +407,7 @@ SCENARIO(" chunk IDs can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed chunk ID equals 0 ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = 0; REQUIRE(id == expected_id); } @@ -421,7 +421,7 @@ SCENARIO(" chunk IDs can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed chunk ID equals 0 ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = 0; REQUIRE(id == expected_id); } @@ -434,7 +434,7 @@ SCENARIO(" chunk IDs can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed chunk ID equals 0 ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = 0; REQUIRE(id == expected_id); } @@ -448,7 +448,7 @@ SCENARIO(" chunk IDs can be computed correctly ", CAPTURE(offset, block_size); THEN(" the computed chunk ID equals 1 ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = 1; REQUIRE(id == expected_id); } @@ -468,7 +468,7 @@ SCENARIO(" chunk IDs can be computed correctly ", THEN(" the computed chunk ID is equal to dividing the offset by " "the block_size ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = offset / block_size; REQUIRE(id == expected_id); } @@ -485,7 +485,7 @@ SCENARIO(" chunk IDs can be computed correctly ", THEN(" the computed chunk ID is equal to dividing the offset by " "the block_size ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = offset / block_size; REQUIRE(id == expected_id); } @@ -501,7 +501,7 @@ SCENARIO(" chunk IDs can be computed correctly ", THEN(" the computed chunk ID is equal to dividing the offset by " "the block_size ") { - const uint64_t id = chnk_id_for_offset(offset, block_size); + const uint64_t id = block_index(offset, block_size); const uint64_t expected_id = offset / block_size; REQUIRE(id == expected_id); } -- GitLab From f0503d7be679aa9c7ce787c8939b6e0e805ee1ba Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 5 Dec 2020 11:00:12 +0100 Subject: [PATCH 21/25] Rename: chnk_count_for_offset() -> block_count() --- include/global/arithmetic/arithmetic.hpp | 24 +++++------ tests/unit/test_utils_arithmetic.cpp | 54 +++++++++++------------- 2 files changed, 37 insertions(+), 41 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index 22ff7c5de..c41dc7602 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -169,32 +169,32 @@ block_index(const uint64_t offset, const size_t block_size) { * overflow. * * @param [in] offset the operation's initial offset. - * @param [in] count the number of bytes affected by the operation. - * @param [in] chnk_size the block size that should be used to compute the + * @param [in] size the number of bytes affected by the operation. + * @param [in] block_size the block size that should be used to compute the * number of blocks. * @returns the number of blocks affected by the operation. */ constexpr std::size_t -chnk_count_for_offset(const uint64_t offset, const size_t count, - const size_t chnk_size) { +block_count(const uint64_t offset, const size_t size, const size_t block_size) { using gkfs::utils::arithmetic::log2; // These checks are automatically removed in release builds - assert(is_power_of_2(chnk_size)); + assert(is_power_of_2(block_size)); #if defined(__GNUC__) && !defined(__clang__) - assert(!__builtin_add_overflow_p(offset, count, static_cast(0))); + assert(!__builtin_add_overflow_p(offset, size, static_cast(0))); #else - assert(offset + count > offset); + assert(offset + size > offset); #endif - const uint64_t chnk_start = align_left(offset, chnk_size); - const uint64_t chnk_end = align_left(offset + count, chnk_size); - const size_t mask = -!!count; // this is either 0 or ~0 + const uint64_t first_block = align_left(offset, block_size); + const uint64_t final_block = align_left(offset + size, block_size); + const size_t mask = -!!size; // this is either 0 or ~0 - return (((chnk_end >> log2(chnk_size)) - (chnk_start >> log2(chnk_size)) + - !is_divisible(offset + count, chnk_size))) & + return (((final_block >> log2(block_size)) - + (first_block >> log2(block_size)) + + !is_divisible(offset + size, block_size))) & mask; } diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index 58f95d766..bd94cc45c 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -512,7 +512,7 @@ SCENARIO(" chunk IDs can be computed correctly ", SCENARIO(" the number of chunks involved in an operation can be computed " "correctly ", - "[utils][numeric][chnk_count_for_offset]") { + "[utils][numeric][block_count]") { GIVEN(" an offset, an operation size, and a block size ") { @@ -533,7 +533,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 0 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -548,7 +548,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 1; REQUIRE(n == expected_n); } @@ -562,7 +562,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 1; REQUIRE(n == expected_n); } @@ -580,7 +580,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -604,7 +604,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -620,7 +620,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count equals 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 1; REQUIRE(n == expected_n); } @@ -634,7 +634,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 1; REQUIRE(n == expected_n); } @@ -652,7 +652,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -675,7 +675,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 0 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -689,7 +689,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 1; REQUIRE(n == expected_n); } @@ -707,7 +707,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -730,8 +730,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " CAPTURE(offset, size, block_size); THEN(" the computed block count == 1 ") { - const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + const std::size_t n = block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -744,8 +743,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " CAPTURE(offset, size, block_size); THEN(" the computed block count == M ") { - const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + const std::size_t n = block_count(offset, size, block_size); const std::size_t expected_n = m; REQUIRE(n == expected_n); } @@ -760,8 +758,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { - const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + const std::size_t n = block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + ((offset + size) % block_size ? 1u : 0); @@ -777,8 +774,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { - const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + const std::size_t n = block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + ((offset + size) % block_size ? 1u : 0); @@ -805,7 +801,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -819,7 +815,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == M ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = m; REQUIRE(n == expected_n); } @@ -836,7 +832,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -854,7 +850,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -878,7 +874,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } @@ -892,7 +888,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == M ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = m; REQUIRE(n == expected_n); } @@ -909,7 +905,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -927,7 +923,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count corresponds to the number " "of blocks involved in the operation ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = (offset + size) / block_size - offset / block_size + @@ -950,7 +946,7 @@ SCENARIO(" the number of chunks involved in an operation can be computed " THEN(" the computed block count == 1 ") { const std::size_t n = - chnk_count_for_offset(offset, size, block_size); + block_count(offset, size, block_size); const std::size_t expected_n = 0; REQUIRE(n == expected_n); } -- GitLab From 1219e0ba5bedc4ef563661a28b13b44d9165a20a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 5 Dec 2020 13:34:26 +0100 Subject: [PATCH 22/25] Enable pow2-arithmetic tests in CI --- .gitlab-ci.yml | 9 +++++++++ tests/unit/CMakeLists.txt | 4 +++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ed082493..295c783b3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,15 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" +unit: + stage: test + script: + - ctest -j $(nproc) -L unit::all + artifacts: + when: on_failure + paths: + - Testing + test wr: stage: test script: diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 3109f9b47..a5568fb52 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -47,7 +47,9 @@ target_link_libraries(tests # to auto-discover Catch tests and register them in CTest set(CMAKE_MODULE_PATH "${catch2_SOURCE_DIR}/contrib" ${CMAKE_MODULE_PATH}) include(Catch) -catch_discover_tests(tests) +catch_discover_tests(tests + PROPERTIES LABELS "unit::all" + ) if(GKFS_INSTALL_TESTS) install(TARGETS tests -- GitLab From 0113f84d086f6866e746bad3b1b88c96153a783e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 8 Dec 2020 12:31:37 +0100 Subject: [PATCH 23/25] Fix transfer size if offset + size ends in block boundary --- src/client/rpc/forward_data.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 7a01074b4..e0885ca01 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -132,7 +132,8 @@ forward_write(const string& path, const void* buf, const bool append_flag, } // receiver of last chunk must subtract - if(target == chnk_end_target) { + if(target == chnk_end_target && + !is_divisible(offset + write_size, gkfs::config::rpc::chunksize)) { total_chunk_size -= block_underrun(offset + write_size, gkfs::config::rpc::chunksize); } @@ -309,7 +310,8 @@ forward_read(const string& path, void* buf, const off64_t offset, } // receiver of last chunk must subtract - if(target == chnk_end_target) { + if(target == chnk_end_target && + !is_divisible(offset + read_size, gkfs::config::rpc::chunksize)) { total_chunk_size -= block_underrun(offset + read_size, gkfs::config::rpc::chunksize); } -- GitLab From b8807932383a53a5b2968e9ce2e272e702e5edb8 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 8 Dec 2020 12:34:46 +0100 Subject: [PATCH 24/25] Rename: is_divisible() -> is_aligned() --- include/global/arithmetic/arithmetic.hpp | 7 ++++--- src/client/rpc/forward_data.cpp | 4 ++-- tests/unit/test_utils_arithmetic.cpp | 4 ++-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index c41dc7602..587af0ea6 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -43,7 +43,8 @@ log2(uint64_t n) { } /** - * Check whether @n is divisible by @block_size. + * Check whether @n is aligned to a block boundary, i.e. if it is divisible by + * @block_size. * * @note This function assumes that block_size is a power of 2. * @@ -52,7 +53,7 @@ log2(uint64_t n) { * @returns true if @n is divisible by @block_size; false otherwise. */ constexpr bool -is_divisible(const uint64_t n, const size_t block_size) { +is_aligned(const uint64_t n, const size_t block_size) { using gkfs::utils::arithmetic::log2; assert(is_power_of_2(block_size)); return !(n & ((1u << log2(block_size)) - 1)); @@ -194,7 +195,7 @@ block_count(const uint64_t offset, const size_t size, const size_t block_size) { return (((final_block >> log2(block_size)) - (first_block >> log2(block_size)) + - !is_divisible(offset + size, block_size))) & + !is_aligned(offset + size, block_size))) & mask; } diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index e0885ca01..4238287dd 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -133,7 +133,7 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of last chunk must subtract if(target == chnk_end_target && - !is_divisible(offset + write_size, gkfs::config::rpc::chunksize)) { + !is_aligned(offset + write_size, gkfs::config::rpc::chunksize)) { total_chunk_size -= block_underrun(offset + write_size, gkfs::config::rpc::chunksize); } @@ -311,7 +311,7 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of last chunk must subtract if(target == chnk_end_target && - !is_divisible(offset + read_size, gkfs::config::rpc::chunksize)) { + !is_aligned(offset + read_size, gkfs::config::rpc::chunksize)) { total_chunk_size -= block_underrun(offset + read_size, gkfs::config::rpc::chunksize); } diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index bd94cc45c..316a00e51 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -97,7 +97,7 @@ SCENARIO(" powers of 2 can be correctly detected ", } SCENARIO(" divisibility by powers of 2 can be correctly detected ", - "[utils][numeric][is_divisible]") { + "[utils][numeric][is_aligned]") { GIVEN(" a number and a block_size ") { @@ -110,7 +110,7 @@ SCENARIO(" divisibility by powers of 2 can be correctly detected ", CAPTURE(n, block_size); bool expected = n % block_size == 0; - REQUIRE(is_divisible(n, block_size) == expected); + REQUIRE(is_aligned(n, block_size) == expected); } } -- GitLab From 0aeeddcb12a2f025daec094747f4bb5b382c77d0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 8 Dec 2020 13:25:27 +0100 Subject: [PATCH 25/25] Remove several unneeded static_casts to uint64_t --- include/global/arithmetic/arithmetic.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp index 587af0ea6..a49eabea0 100644 --- a/include/global/arithmetic/arithmetic.hpp +++ b/include/global/arithmetic/arithmetic.hpp @@ -73,7 +73,7 @@ constexpr uint64_t align_left(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return static_cast(offset) & ~(block_size - 1u); + return offset & ~(block_size - 1u); } @@ -110,7 +110,7 @@ constexpr size_t block_overrun(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return static_cast(offset) & (block_size - 1u); + return offset & (block_size - 1u); } @@ -156,8 +156,7 @@ block_index(const uint64_t offset, const size_t block_size) { // This check is automatically removed in release builds assert(is_power_of_2(block_size)); - return static_cast(align_left(offset, block_size) >> - log2(block_size)); + return align_left(offset, block_size) >> log2(block_size); } @@ -184,7 +183,7 @@ block_count(const uint64_t offset, const size_t size, const size_t block_size) { assert(is_power_of_2(block_size)); #if defined(__GNUC__) && !defined(__clang__) - assert(!__builtin_add_overflow_p(offset, size, static_cast(0))); + assert(!__builtin_add_overflow_p(offset, size, uint64_t{0})); #else assert(offset + size > offset); #endif -- GitLab