diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ed0824937097cdada6ff86238f9d97dbdb922fc..295c783b3f630ff8bc0fc5831a84b30767d8af0d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,15 @@ integration tests: paths: - "${INTEGRATION_TESTS_RUN_PATH}" +unit: + stage: test + script: + - ctest -j $(nproc) -L unit::all + artifacts: + when: on_failure + paths: + - Testing + test wr: stage: test script: diff --git a/include/client/logging.hpp b/include/client/logging.hpp index 2225f08d1e0a30e2f562b22e06c46449914f56fa..db769d648e8c291b8b6ccfbe5ec893e15e52217a 100644 --- a/include/client/logging.hpp +++ b/include/client/logging.hpp @@ -106,7 +106,7 @@ static const auto constexpr most = log_level::print_most; static const auto constexpr all = log_level::print_all; static const auto constexpr help = log_level::print_help; -static const auto constexpr level_names = util::make_array( +static const auto constexpr level_names = utils::make_array( "syscall", "syscall", // sycall_entry uses the same name as syscall "info", "critical", "error", "warning", "hermes", "mercury", "debug"); diff --git a/include/client/make_array.hpp b/include/client/make_array.hpp index 2d1618c6009519475fe7824dcf6b39ca729db874..ffd5ef3b3438c0a7bbf8f46a229922db62aee0b2 100644 --- a/include/client/make_array.hpp +++ b/include/client/make_array.hpp @@ -16,7 +16,7 @@ #include -namespace gkfs::util { +namespace gkfs::utils { template constexpr auto @@ -28,6 +28,6 @@ make_array(T&&... values) -> std::array< sizeof...(T)>{std::forward(values)...}; } -} // namespace gkfs::util +} // namespace gkfs::utils #endif // LIBGKFS_UTILS_MAKE_ARRAY_HPP diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index f42fcd979c2301cea3eed9beb890c8e41413b6c3..c8dc9fcfd570904aa000d6fd22c0bca0863c924e 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -48,7 +48,7 @@ class async_engine; extern std::unique_ptr ld_network_service; // function definitions -namespace gkfs::util { +namespace gkfs::utils { template constexpr typename std::underlying_type::type to_underlying(E e) { @@ -74,6 +74,6 @@ read_hosts_file(); void connect_to_hosts(const std::vector>& hosts); -} // namespace gkfs::util +} // namespace gkfs::utils #endif // GEKKOFS_PRELOAD_UTIL_HPP diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index e2829626ca2ffefd4aa49dea68c1c40acefd4905..3df34ea731710622bdcee940adb51b6fd73982ee 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -291,7 +291,7 @@ format_whence_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for lseek() whence arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SEEK_SET), FLAG_ENTRY(SEEK_CUR), FLAG_ENTRY(SEEK_END) @@ -314,7 +314,7 @@ format_mmap_prot_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for mmap() prot arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(PROT_NONE), FLAG_ENTRY(PROT_READ), FLAG_ENTRY(PROT_WRITE), @@ -339,7 +339,7 @@ format_mmap_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for mmap() flags arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(MAP_SHARED), FLAG_ENTRY(MAP_PRIVATE), #ifdef MAP_SHARED_VALIDATE @@ -379,7 +379,7 @@ format_clone_flags_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for clone() flags arg */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(CLONE_VM), FLAG_ENTRY(CLONE_FS), FLAG_ENTRY(CLONE_FILES), @@ -431,7 +431,7 @@ format_signum_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for signum args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SIGHUP), FLAG_ENTRY(SIGINT), FLAG_ENTRY(SIGQUIT), @@ -485,7 +485,7 @@ format_sigproc_how_arg_to(FmtBuffer& buffer, const printable_arg& parg) { /* Names for sigproc how args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(SIG_BLOCK), FLAG_ENTRY(SIG_UNBLOCK), FLAG_ENTRY(SIG_SETMASK)); @@ -572,13 +572,13 @@ format_open_flags_to(FmtBuffer& buffer, /* Names for O_ACCMODE args */ const auto flag_names = - util::make_array( + utils::make_array( FLAG_ENTRY(O_RDONLY), FLAG_ENTRY(O_WRONLY), FLAG_ENTRY(O_RDWR)); const auto extra_flag_names = - util::make_array( + utils::make_array( #ifdef O_EXEC FLAG_ENTRY(O_EXEC), #endif diff --git a/include/daemon/util.hpp b/include/daemon/util.hpp index c8845847aeac92c99dcd9ac088dae62862a03692..6e2c7cea8dce1d2e845e866bc6f1fb1aa89ac487 100644 --- a/include/daemon/util.hpp +++ b/include/daemon/util.hpp @@ -15,13 +15,13 @@ #define GEKKOFS_DAEMON_UTIL_HPP namespace gkfs { -namespace util { +namespace utils { void populate_hosts_file(); void destroy_hosts_file(); -} // namespace util +} // namespace utils } // namespace gkfs #endif // GEKKOFS_DAEMON_UTIL_HPP diff --git a/include/global/arithmetic/arithmetic.hpp b/include/global/arithmetic/arithmetic.hpp new file mode 100644 index 0000000000000000000000000000000000000000..a49eabea0ccba4633f9ba6dab1e056e58b23256d --- /dev/null +++ b/include/global/arithmetic/arithmetic.hpp @@ -0,0 +1,203 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +#ifndef GKFS_GLOBAL_ARITHMETIC_HPP +#define GKFS_GLOBAL_ARITHMETIC_HPP + +#include +#include +#include + +namespace gkfs::utils::arithmetic { + +/** + * Check whether integer `n` is a power of 2. + * + * @param [in] n the number to check. + * @returns `true` if `n` is a power of 2; `false` otherwise. + */ +constexpr bool +is_power_of_2(uint64_t n) { + return n && (!(n & (n - 1u))); +} + +/** + * Compute the base2 logarithm for 64 bit integers. + * + * @param [in] n the number from which to compute the log2. + * @returns the base 2 logarithm of `n`. + */ +constexpr std::size_t +log2(uint64_t n) { + return 8u * sizeof(uint64_t) - __builtin_clzll(n) - 1; +} + +/** + * Check whether @n is aligned to a block boundary, i.e. if it is divisible by + * @block_size. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] n the number to check. + * @param [in] block_size + * @returns true if @n is divisible by @block_size; false otherwise. + */ +constexpr bool +is_aligned(const uint64_t n, const size_t block_size) { + using gkfs::utils::arithmetic::log2; + assert(is_power_of_2(block_size)); + return !(n & ((1u << log2(block_size)) - 1)); +} + +/** + * Given a file @offset and a @block_size, align the @offset to its + * closest left-side block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset to align. + * @param [in] block_size the block size used to compute boundaries. + * @returns an offset aligned to the left-side block boundary. + */ +constexpr uint64_t +align_left(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return offset & ~(block_size - 1u); +} + + +/** + * Given a file @offset and a @block_size, align the @offset to its + * closest right-side block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset to align. + * @param [in] block_size the block size used to compute boundaries. + * @returns an offset aligned to the right-side block boundary. + */ +constexpr uint64_t +align_right(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return align_left(offset, block_size) + block_size; +} + + +/** + * Return the overrun bytes that separate @offset from the closest left side + * block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset for which the overrun distance should be + * computed. + * @param [in] block_size the block size used to compute boundaries. + * @returns the distance in bytes between the left-side boundary of @offset + */ +constexpr size_t +block_overrun(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return offset & (block_size - 1u); +} + + +/** + * Return the underrun bytes that separate @offset from the closest right side + * block boundary. + * + * @note This function assumes that block_size is a power of 2. + * + * @param [in] offset the offset for which the overrun distance should be + * computed. + * @param [in] block_size the block size used to compute boundaries. + * @returns the distance in bytes between the right-side boundary of @offset + */ +constexpr size_t +block_underrun(const uint64_t offset, const size_t block_size) { + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return align_right(offset, block_size) - offset; +} + + +/** + * Given an @offset and a @block_size, compute the block index to which @offset + * belongs. + * + * @note Block indexes are (conceptually) computed by dividing @offset + * by @block_size, with index 0 referring to block [0, block_size - 1], + * index 1 to block [block_size, 2 * block_size - 1], and so on up to + * a maximum index FILE_LENGTH / block_size. + * + * @note This function assumes that @block_size is a power of 2. + * + * @param [in] offset the offset for which the block index should be computed. + * @param [in] block_size the block_size that should be used to compute the + * index. + * @returns the index of the block containing @offset. + */ +constexpr uint64_t +block_index(const uint64_t offset, const size_t block_size) { + + using gkfs::utils::arithmetic::log2; + + // This check is automatically removed in release builds + assert(is_power_of_2(block_size)); + return align_left(offset, block_size) >> log2(block_size); +} + + +/** + * Compute the number of blocks involved in an operation affecting the + * regions from [@offset, to @offset + @count). + * + * @note This function assumes that @block_size is a power of 2. + * @note This function assumes that @offset + @count does not + * overflow. + * + * @param [in] offset the operation's initial offset. + * @param [in] size the number of bytes affected by the operation. + * @param [in] block_size the block size that should be used to compute the + * number of blocks. + * @returns the number of blocks affected by the operation. + */ +constexpr std::size_t +block_count(const uint64_t offset, const size_t size, const size_t block_size) { + + using gkfs::utils::arithmetic::log2; + + // These checks are automatically removed in release builds + assert(is_power_of_2(block_size)); + +#if defined(__GNUC__) && !defined(__clang__) + assert(!__builtin_add_overflow_p(offset, size, uint64_t{0})); +#else + assert(offset + size > offset); +#endif + + const uint64_t first_block = align_left(offset, block_size); + const uint64_t final_block = align_left(offset + size, block_size); + const size_t mask = -!!size; // this is either 0 or ~0 + + return (((final_block >> log2(block_size)) - + (first_block >> log2(block_size)) + + !is_aligned(offset + size, block_size))) & + mask; +} + +} // namespace gkfs::utils::arithmetic + +#endif // GKFS_GLOBAL_ARITHMETIC_HPP \ No newline at end of file diff --git a/include/global/chunk_calc_util.hpp b/include/global/chunk_calc_util.hpp deleted file mode 100644 index bbe14622e3a6893d56b7dbd567018208fa92f3c9..0000000000000000000000000000000000000000 --- a/include/global/chunk_calc_util.hpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain - Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany - - This software was partially supported by the - EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). - - This software was partially supported by the - ADA-FS project under the SPPEXA project funded by the DFG. - - SPDX-License-Identifier: MIT -*/ - -#ifndef GEKKOFS_CHNK_CALC_UTIL_HPP -#define GEKKOFS_CHNK_CALC_UTIL_HPP - -#include - -namespace gkfs::util { - -/** - * Compute the base2 logarithm for 64 bit integers - */ -inline int -log2(uint64_t n) { - - /* see - * http://stackoverflow.com/questions/11376288/fast-computing-of-log2-for-64-bit-integers - */ - static const int table[64] = { - 0, 58, 1, 59, 47, 53, 2, 60, 39, 48, 27, 54, 33, 42, 3, 61, - 51, 37, 40, 49, 18, 28, 20, 55, 30, 34, 11, 43, 14, 22, 4, 62, - 57, 46, 52, 38, 26, 32, 41, 50, 36, 17, 19, 29, 10, 13, 21, 56, - 45, 25, 31, 35, 16, 9, 12, 44, 24, 15, 8, 23, 7, 6, 5, 63}; - - n |= n >> 1; - n |= n >> 2; - n |= n >> 4; - n |= n >> 8; - n |= n >> 16; - n |= n >> 32; - - return table[(n * 0x03f6eaf2cd271461) >> 58]; -} - - -/** - * Align an @offset to the closest left side chunk boundary - */ -inline off64_t -chnk_lalign(const off64_t offset, const size_t chnk_size) { - return offset & ~(chnk_size - 1); -} - - -/** - * Align an @offset to the closest right side chunk boundary - */ -inline off64_t -chnk_ralign(const off64_t offset, const size_t chnk_size) { - return chnk_lalign(offset + chnk_size, chnk_size); -} - - -/** - * Return the padding (bytes) that separates the @offset from the closest - * left side chunk boundary - * - * If @offset is a boundary the resulting padding will be 0 - */ -inline size_t -chnk_lpad(const off64_t offset, const size_t chnk_size) { - return offset % chnk_size; -} - - -/** - * Return the padding (bytes) that separates the @offset from the closest - * right side chunk boundary - * - * If @offset is a boundary the resulting padding will be 0 - */ -inline size_t -chnk_rpad(const off64_t offset, const size_t chnk_size) { - return (-offset) % chnk_size; -} - - -/** - * Given an @offset calculates the chunk number to which the @offset belongs - * - * chunk_id(8,4) = 2; - * chunk_id(7,4) = 1; - * chunk_id(2,4) = 0; - * chunk_id(0,4) = 0; - */ -inline uint64_t -chnk_id_for_offset(const off64_t offset, const size_t chnk_size) { - /* - * This does not work for offsets that use the 64th bit, i.e., - * 9223372036854775808. 9223372036854775808 - 1 uses 63 bits and still - * works. `offset / chnk_size` works with the 64th bit. With this number we - * can address more than 19,300,000 exabytes of data though. Hi future me? - */ - return static_cast(chnk_lalign(offset, chnk_size) >> - log2(chnk_size)); -} - - -/** - * Return the number of chunks involved in an operation that operates - * from @offset for a certain amount of bytes (@count). - */ -inline uint64_t -chnk_count_for_offset(const off64_t offset, const size_t count, - const size_t chnk_size) { - - off64_t chnk_start = chnk_lalign(offset, chnk_size); - off64_t chnk_end = chnk_lalign(offset + count - 1, chnk_size); - - return static_cast((chnk_end >> log2(chnk_size)) - - (chnk_start >> log2(chnk_size)) + 1); -} - -} // namespace gkfs::util - -#endif diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index e6488538e90b4a3022a76b49848ec897e2acc651..3cf75472913b872902c3e03a51616359e78c7c32 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -42,7 +42,6 @@ set(PRELOAD_HEADERS ../../include/client/syscalls/syscall.hpp ../../include/client/syscalls/detail/syscall_info.h ../../include/global/cmake_configure.hpp - ../../include/global/chunk_calc_util.hpp ../../include/global/global_defs.hpp ../../include/global/path_util.hpp ../../include/global/rpc/rpc_types.hpp @@ -70,7 +69,9 @@ set(PRELOAD_INCLUDE_DIRS add_library(gkfs_intercept SHARED ${PRELOAD_SRC} ${PRELOAD_HEADERS}) -target_link_libraries(gkfs_intercept ${PRELOAD_LINK_LIBRARIES}) +target_link_libraries(gkfs_intercept + arithmetic + ${PRELOAD_LINK_LIBRARIES}) target_include_directories(gkfs_intercept PRIVATE ${PRELOAD_INCLUDE_DIRS}) @@ -115,4 +116,4 @@ if (GKFS_ENABLE_FORWARDING) ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs ) -endif () \ No newline at end of file +endif () diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 4ffb1c90a7da59eafe345829394970bbbb5abb73..1884817e33d54cd1132c4f120394a159626efe96 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -84,7 +84,7 @@ int check_parent_dir(const std::string& path) { #if CREATE_CHECK_PARENTS auto p_comp = gkfs::path::dirname(path); - auto md = gkfs::util::get_metadata(p_comp); + auto md = gkfs::utils::get_metadata(p_comp); if(!md) { if(errno == ENOENT) { LOG(DEBUG, "Parent component does not exist: '{}'", p_comp); @@ -130,7 +130,7 @@ gkfs_open(const std::string& path, mode_t mode, int flags) { } bool exists = true; - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { if(errno == ENOENT) { exists = false; @@ -253,7 +253,7 @@ gkfs_create(const std::string& path, mode_t mode) { */ int gkfs_remove(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { return -1; } @@ -276,7 +276,7 @@ gkfs_remove(const std::string& path) { */ int gkfs_access(const std::string& path, const int mask, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { errno = ENOENT; return -1; @@ -294,11 +294,11 @@ gkfs_access(const std::string& path, const int mask, bool follow_links) { */ int gkfs_stat(const string& path, struct stat* buf, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { return -1; } - gkfs::util::metadata_to_stat(path, *md, *buf); + gkfs::utils::metadata_to_stat(path, *md, *buf); return 0; } @@ -318,14 +318,14 @@ gkfs_stat(const string& path, struct stat* buf, bool follow_links) { int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, struct statx* buf, bool follow_links) { - auto md = gkfs::util::get_metadata(path, follow_links); + auto md = gkfs::utils::get_metadata(path, follow_links); if(!md) { return -1; } struct stat tmp {}; - gkfs::util::metadata_to_stat(path, *md, tmp); + gkfs::utils::metadata_to_stat(path, *md, tmp); buf->stx_mask = 0; buf->stx_blksize = tmp.st_blksize; @@ -545,7 +545,7 @@ gkfs_truncate(const std::string& path, off_t length) { return -1; } - auto md = gkfs::util::get_metadata(path, true); + auto md = gkfs::utils::get_metadata(path, true); if(!md) { return -1; } @@ -863,7 +863,7 @@ gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { int gkfs_opendir(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { return -1; } @@ -891,7 +891,7 @@ gkfs_opendir(const std::string& path) { */ int gkfs_rmdir(const std::string& path) { - auto md = gkfs::util::get_metadata(path); + auto md = gkfs::utils::get_metadata(path); if(!md) { LOG(DEBUG, "Path '{}' does not exist: ", path); errno = ENOENT; @@ -1088,7 +1088,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { * Here if the target is a directory we raise a NOTSUP error. * So that application know we don't support link to directory. */ - auto target_md = gkfs::util::get_metadata(target_path, false); + auto target_md = gkfs::utils::get_metadata(target_path, false); if(target_md != nullptr) { auto trg_mode = target_md->mode(); if(!(S_ISREG(trg_mode) || S_ISLNK(trg_mode))) { @@ -1103,7 +1103,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { return -1; } - auto link_md = gkfs::util::get_metadata(path, false); + auto link_md = gkfs::utils::get_metadata(path, false); if(link_md != nullptr) { LOG(DEBUG, "Link exists: '{}'", path); errno = EEXIST; @@ -1130,7 +1130,7 @@ gkfs_mk_symlink(const std::string& path, const std::string& target_path) { */ int gkfs_readlink(const std::string& path, char* buf, int bufsize) { - auto md = gkfs::util::get_metadata(path, false); + auto md = gkfs::utils::get_metadata(path, false); if(md == nullptr) { LOG(DEBUG, "Named link doesn't exist"); return -1; diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 0dec98e7427a0847d7c8f1cf060ec20e6f529ed8..59d98d79b56a3c27119a97b47e66794f4fa1c3e2 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -614,7 +614,7 @@ hook_chdir(const char* path) { bool internal = CTX->relativize_path(path, rel_path); if(internal) { // path falls in our namespace - auto md = gkfs::util::get_metadata(rel_path); + auto md = gkfs::utils::get_metadata(rel_path); if(md == nullptr) { LOG(ERROR, "{}() path does not exists", __func__); return -ENOENT; diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 613f4e18c8e7270f8692fb37017aaca02263cbbd..1f21f290f20f0cd8e9833ffb20bd8b5f662c0c56 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -40,7 +40,7 @@ struct opt_info { #define STR_AND_LEN(strbuf) strbuf, sizeof(strbuf) - 1 -static const auto constexpr debug_opts = util::make_array( +static const auto constexpr debug_opts = utils::make_array( opt_info{STR_AND_LEN("none"), {"don't print any messages"}, log::none}, diff --git a/src/client/open_file_map.cpp b/src/client/open_file_map.cpp index f88b3eccbb0721408b3b33486a75cd88e59eeddb..6ecec24dc11b4cfacc2a175b4764a521fdda879a 100644 --- a/src/client/open_file_map.cpp +++ b/src/client/open_file_map.cpp @@ -31,17 +31,17 @@ OpenFile::OpenFile(const string& path, const int flags, FileType type) : type_(type), path_(path) { // set flags to OpenFile if(flags & O_CREAT) - flags_[gkfs::util::to_underlying(OpenFile_flags::creat)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::creat)] = true; if(flags & O_APPEND) - flags_[gkfs::util::to_underlying(OpenFile_flags::append)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::append)] = true; if(flags & O_TRUNC) - flags_[gkfs::util::to_underlying(OpenFile_flags::trunc)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::trunc)] = true; if(flags & O_RDONLY) - flags_[gkfs::util::to_underlying(OpenFile_flags::rdonly)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::rdonly)] = true; if(flags & O_WRONLY) - flags_[gkfs::util::to_underlying(OpenFile_flags::wronly)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::wronly)] = true; if(flags & O_RDWR) - flags_[gkfs::util::to_underlying(OpenFile_flags::rdwr)] = true; + flags_[gkfs::utils::to_underlying(OpenFile_flags::rdwr)] = true; pos_ = 0; // If O_APPEND flag is used, it will be used before each write. } @@ -73,13 +73,13 @@ OpenFile::pos(unsigned long pos) { bool OpenFile::get_flag(OpenFile_flags flag) { lock_guard lock(pos_mutex_); - return flags_[gkfs::util::to_underlying(flag)]; + return flags_[gkfs::utils::to_underlying(flag)]; } void OpenFile::set_flag(OpenFile_flags flag, bool value) { lock_guard lock(flag_mutex_); - flags_[gkfs::util::to_underlying(flag)] = value; + flags_[gkfs::utils::to_underlying(flag)] = value; } FileType diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 812c54c562fe9aae52265253d0db99a5d78d8494..8792ae10be97653612540ace77f42628f76af4df 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -97,7 +97,7 @@ init_ld_environment_() { vector> hosts{}; try { LOG(INFO, "Loading peer addresses..."); - hosts = gkfs::util::read_hosts_file(); + hosts = gkfs::utils::read_hosts_file(); } catch(const std::exception& e) { exit_error_msg(EXIT_FAILURE, "Failed to load hosts addresses: "s + e.what()); @@ -111,7 +111,7 @@ init_ld_environment_() { } try { - gkfs::util::connect_to_hosts(hosts); + gkfs::utils::connect_to_hosts(hosts); } catch(const std::exception& e) { exit_error_msg(EXIT_FAILURE, "Failed to connect to hosts: "s + e.what()); @@ -120,7 +120,7 @@ init_ld_environment_() { /* Setup distributor */ #ifdef GKFS_ENABLE_FORWARDING try { - gkfs::util::load_forwarding_map(); + gkfs::utils::load_forwarding_map(); LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); } catch(std::exception& e) { @@ -160,7 +160,7 @@ forwarding_mapper(void* p) { while(forwarding_running) { try { - gkfs::util::load_forwarding_map(); + gkfs::utils::load_forwarding_map(); if(previous != CTX->fwd_host_id()) { LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index c88a10c9575713b07cdf4b2ad1754387d18958e5..d4cc99488c1048a6eed7f4bb1b02a1edfdaeace7 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -163,7 +163,7 @@ load_hostfile(const std::string& path) { } // namespace -namespace gkfs::util { +namespace gkfs::utils { /** * Retrieve metadata from daemon @@ -398,4 +398,4 @@ connect_to_hosts(const vector>& hosts) { CTX->hosts(addrs); } -} // namespace gkfs::util \ No newline at end of file +} // namespace gkfs::utils \ No newline at end of file diff --git a/src/client/rpc/forward_data.cpp b/src/client/rpc/forward_data.cpp index 2961673eaec1c4d21059005bc201856d781dc665..4238287ddf4e85e09b20869e890938c9849dbadb 100644 --- a/src/client/rpc/forward_data.cpp +++ b/src/client/rpc/forward_data.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include @@ -48,6 +48,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, const off64_t in_offset, const size_t write_size, const int64_t updated_metadentry_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(write_size > 0); // Calculate chunkid boundaries and numbers so that daemons know in @@ -55,10 +58,9 @@ forward_write(const string& path, const void* buf, const bool append_flag, off64_t offset = append_flag ? in_offset : (updated_metadentry_size - write_size); - auto chnk_start = gkfs::util::chnk_id_for_offset( - offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::util::chnk_id_for_offset( - (offset + write_size) - 1, gkfs::config::rpc::chunksize); + auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); + auto chnk_end = block_index((offset + write_size) - 1, + gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -126,13 +128,14 @@ forward_write(const string& path, const void* buf, const bool append_flag, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { total_chunk_size -= - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize); + block_overrun(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract - if(target == chnk_end_target) { - total_chunk_size -= gkfs::util::chnk_rpad( - offset + write_size, gkfs::config::rpc::chunksize); + if(target == chnk_end_target && + !is_aligned(offset + write_size, gkfs::config::rpc::chunksize)) { + total_chunk_size -= block_underrun(offset + write_size, + gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); @@ -145,8 +148,8 @@ forward_write(const string& path, const void* buf, const bool append_flag, path, // first offset in targets is the chunk with // a potential offset - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize), - target, CTX->hosts().size(), + block_overrun(offset, gkfs::config::rpc::chunksize), target, + CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), // chunk start id of this write @@ -228,12 +231,14 @@ pair forward_read(const string& path, void* buf, const off64_t offset, const size_t read_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + // Calculate chunkid boundaries and numbers so that daemons know in which // interval to look for chunks - auto chnk_start = gkfs::util::chnk_id_for_offset( - offset, gkfs::config::rpc::chunksize); - auto chnk_end = gkfs::util::chnk_id_for_offset( - (offset + read_size - 1), gkfs::config::rpc::chunksize); + auto chnk_start = block_index(offset, gkfs::config::rpc::chunksize); + auto chnk_end = + block_index((offset + read_size - 1), gkfs::config::rpc::chunksize); // Collect all chunk ids within count that have the same destination so // that those are send in one rpc bulk transfer @@ -301,13 +306,14 @@ forward_read(const string& path, void* buf, const off64_t offset, // receiver of first chunk must subtract the offset from first chunk if(target == chnk_start_target) { total_chunk_size -= - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize); + block_overrun(offset, gkfs::config::rpc::chunksize); } // receiver of last chunk must subtract - if(target == chnk_end_target) { - total_chunk_size -= gkfs::util::chnk_rpad( - offset + read_size, gkfs::config::rpc::chunksize); + if(target == chnk_end_target && + !is_aligned(offset + read_size, gkfs::config::rpc::chunksize)) { + total_chunk_size -= block_underrun(offset + read_size, + gkfs::config::rpc::chunksize); } auto endp = CTX->hosts().at(target); @@ -320,8 +326,8 @@ forward_read(const string& path, void* buf, const off64_t offset, path, // first offset in targets is the chunk with // a potential offset - gkfs::util::chnk_lpad(offset, gkfs::config::rpc::chunksize), - target, CTX->hosts().size(), + block_overrun(offset, gkfs::config::rpc::chunksize), target, + CTX->hosts().size(), // number of chunks handled by that destination target_chnks[target].size(), // chunk start id of this write @@ -400,14 +406,17 @@ int forward_truncate(const std::string& path, size_t current_size, size_t new_size) { + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(current_size > new_size); // Find out which data servers need to delete data chunks in order to // contact only them - const unsigned int chunk_start = gkfs::util::chnk_id_for_offset( - new_size, gkfs::config::rpc::chunksize); - const unsigned int chunk_end = gkfs::util::chnk_id_for_offset( - current_size - new_size - 1, gkfs::config::rpc::chunksize); + const unsigned int chunk_start = + block_index(new_size, gkfs::config::rpc::chunksize); + const unsigned int chunk_end = block_index(current_size - new_size - 1, + gkfs::config::rpc::chunksize); std::unordered_set hosts; for(unsigned int chunk_id = chunk_start; chunk_id <= chunk_end; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index b8d1201318a87af215af5a743ad092553ff453bf..f8b7f9928a89f35fbd806c2b62cf8385d0c873b3 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -258,7 +258,7 @@ init_environment() { } // setup hostfile to let clients know that a daemon is running on this host if(!GKFS_DATA->hosts_file().empty()) { - gkfs::util::populate_hosts_file(); + gkfs::utils::populate_hosts_file(); } GKFS_DATA->spdlogger()->info("Startup successful. Daemon is ready."); } @@ -301,7 +301,7 @@ destroy_enviroment() { if(!GKFS_DATA->hosts_file().empty()) { GKFS_DATA->spdlogger()->debug("{}() Removing hosts file", __func__); try { - gkfs::util::destroy_hosts_file(); + gkfs::utils::destroy_hosts_file(); } catch(const fs::filesystem_error& e) { GKFS_DATA->spdlogger()->debug("{}() hosts file not found", __func__); diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index a66d1748eced84374c42f747765ded937657a467..027955c2507e39d071e2f24d1ccf12e2908c76b2 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #ifdef GKFS_ENABLE_AGIOS #include @@ -460,7 +460,7 @@ rpc_srv_read(hg_handle_t handle) { origin_offsets[chnk_id_curr] = 0; bulk_buf_ptrs[chnk_id_curr] = chnk_ptr; chnk_sizes[chnk_id_curr] = offset_transfer_size; - // util variables + // utils variables chnk_ptr += offset_transfer_size; chnk_size_left_host -= offset_transfer_size; } else { @@ -481,7 +481,7 @@ rpc_srv_read(hg_handle_t handle) { transfer_size = chnk_size_left_host; bulk_buf_ptrs[chnk_id_curr] = chnk_ptr; chnk_sizes[chnk_id_curr] = transfer_size; - // util variables + // utils variables chnk_ptr += transfer_size; chnk_size_left_host -= transfer_size; } diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 741bd7d09ba6efc1dbfd8a6a58ca8af17fb8b028..b2fd78ae1b6774cd44161ab2d847ffac4e805a53 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -13,7 +13,7 @@ #include #include -#include +#include #include extern "C" { @@ -41,6 +41,10 @@ namespace data { */ void ChunkTruncateOperation::truncate_abt(void* _arg) { + + // import pow2-optimized arithmetic functions + using namespace gkfs::utils::arithmetic; + assert(_arg); // Unpack args auto* arg = static_cast(_arg); @@ -49,11 +53,9 @@ ChunkTruncateOperation::truncate_abt(void* _arg) { int err_response = 0; try { // get chunk from where to cut off - auto chunk_id_start = gkfs::util::chnk_id_for_offset( - size, gkfs::config::rpc::chunksize); + auto chunk_id_start = block_index(size, gkfs::config::rpc::chunksize); // do not last delete chunk if it is in the middle of a chunk - auto left_pad = - gkfs::util::chnk_lpad(size, gkfs::config::rpc::chunksize); + auto left_pad = block_overrun(size, gkfs::config::rpc::chunksize); if(left_pad != 0) { GKFS_DATA->storage()->truncate_chunk_file(path, chunk_id_start, left_pad); diff --git a/src/daemon/util.cpp b/src/daemon/util.cpp index 4e6fede35b597c5b851054b329546d521a93e5e1..6a8cc0d34ff292123212af83f6b64bf367554601 100644 --- a/src/daemon/util.cpp +++ b/src/daemon/util.cpp @@ -20,7 +20,7 @@ using namespace std; -namespace gkfs::util { +namespace gkfs::utils { void populate_hosts_file() { @@ -48,4 +48,4 @@ destroy_hosts_file() { std::remove(GKFS_DATA->hosts_file().c_str()); } -} // namespace gkfs::util +} // namespace gkfs::utils diff --git a/src/global/CMakeLists.txt b/src/global/CMakeLists.txt index 61455f69a16d90fcf562abf2c7ec37b5dd0408ef..fb611cf9a8c5c5b845d28ce9281b266312bdbdb5 100644 --- a/src/global/CMakeLists.txt +++ b/src/global/CMakeLists.txt @@ -1,3 +1,5 @@ +add_subdirectory(arithmetic) + add_library(distributor STATIC) set_property(TARGET distributor PROPERTY POSITION_INDEPENDENT_CODE ON) target_sources(distributor diff --git a/src/global/arithmetic/CMakeLists.txt b/src/global/arithmetic/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..16c35079ce4f701ab563ef522d77dda943d096a0 --- /dev/null +++ b/src/global/arithmetic/CMakeLists.txt @@ -0,0 +1,23 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany +# +# This software was partially supported by the +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). +# +# This software was partially supported by the +# ADA-FS project under the SPPEXA project funded by the DFG. +# +# SPDX-License-Identifier: MIT +################################################################################ + +add_library(arithmetic INTERFACE) +target_sources(arithmetic + INTERFACE + ${INCLUDE_DIR}/global/arithmetic/arithmetic.hpp + ) + +target_include_directories(arithmetic + INTERFACE + ${INCLUDE_DIR}/global/arithmetic/ + ) diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index b04eb2a8699ab65b16287fe9d3601515487d9b73..b82233d3cbcfc34544e9a0f753624f438332dfa4 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -41,7 +41,7 @@ init_commands(CLI::App& app) { write_validate_init(app); write_random_init(app); truncate_init(app); - // util + // utils file_compare_init(app); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 08a6f139e615e4d63da65b4f6cc21ceb82aeba13..a5568fb528f2225547677d58645a936af57eacd3 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -34,18 +34,22 @@ target_link_libraries(catch2_main add_executable(tests test_example_00.cpp test_example_01.cpp + test_utils_arithmetic.cpp ) target_link_libraries(tests catch2_main fmt::fmt + arithmetic ) # Catch2's contrib folder includes some helper functions # to auto-discover Catch tests and register them in CTest set(CMAKE_MODULE_PATH "${catch2_SOURCE_DIR}/contrib" ${CMAKE_MODULE_PATH}) include(Catch) -catch_discover_tests(tests) +catch_discover_tests(tests + PROPERTIES LABELS "unit::all" + ) if(GKFS_INSTALL_TESTS) install(TARGETS tests diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp new file mode 100644 index 0000000000000000000000000000000000000000..316a00e51418fc52e4d9560069bb5709f5552f61 --- /dev/null +++ b/tests/unit/test_utils_arithmetic.cpp @@ -0,0 +1,963 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +#include +#include +#include + +using namespace gkfs::utils::arithmetic; +constexpr auto test_reps = 200u; + +namespace { + +/** + * Check if @n is a power of two by (rather inefficiently) + * performing successive divisions by 2 in an attempt to reach 1. + * + * @param n the number to check + * @returns true if @n is a power of 2, false otherwise + */ +bool +check_power_of_2(uint64_t n) { + if(n == 0) { + return false; + } + + while(n != 1) { + if(n % 2 != 0) { + return false; + } + n /= 2; + } + + return true; +} + +} // namespace + + +SCENARIO(" powers of 2 can be correctly detected ", + "[utils][numeric][is_power_of_2]") { + + GIVEN(" a positive number ") { + + WHEN(" n is 0 ") { + + const uint64_t n = 0; + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == false); + } + } + + WHEN(" n is 1 ") { + + const uint64_t n = 1; + + THEN(" is_power_of_2(n) returns true ") { + REQUIRE(is_power_of_2(n) == true); + } + } + + WHEN(" n is neither 0 nor 1 ") { + + AND_WHEN(" n is a power of 2 ") { + + const std::size_t n = GENERATE( + filter([](uint64_t m) { return check_power_of_2(m); }, + range(0, 10000))); + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == true); + } + } + + AND_WHEN(" n is not a power of 2 ") { + + const std::size_t n = GENERATE( + filter([](uint64_t m) { return !check_power_of_2(m); }, + range(0, 10000))); + + THEN(" is_power_of_2(n) returns false ") { + REQUIRE(is_power_of_2(n) == false); + } + } + } + } +} + +SCENARIO(" divisibility by powers of 2 can be correctly detected ", + "[utils][numeric][is_aligned]") { + + GIVEN(" a number and a block_size ") { + + const uint64_t n = GENERATE(range(0, 1000), range(20000, 23000), + std::numeric_limits::max()); + const std::size_t block_size = + GENERATE(filter([](uint64_t bs) { return is_power_of_2(bs); }, + range(0, 10000))); + + CAPTURE(n, block_size); + + bool expected = n % block_size == 0; + REQUIRE(is_aligned(n, block_size) == expected); + } +} + +SCENARIO(" offsets can be left-aligned to block size boundaries ", + "[utils][numeric][align_left]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = + GENERATE(filter([](uint64_t bs) { return is_power_of_2(bs); }, + range(0, 100000))); + + WHEN(" offset is 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is 0 ") { + const uint64_t aligned_offset = align_left(offset, block_size); + REQUIRE(aligned_offset == 0); + } + } + + WHEN(" offset is smaller than block size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is 0 ") { + const uint64_t aligned_offset = align_left(offset, block_size); + REQUIRE(aligned_offset == 0); + } + } + + WHEN(" offset is larger than block size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the left-aligned offset is the left boundary of the " + "containing block ") { + const uint64_t aligned_offset = align_left(offset, block_size); + const uint64_t exp_offset = + static_cast(offset / block_size) * block_size; + REQUIRE(aligned_offset == exp_offset); + } + } + } +} + +SCENARIO(" offsets can be right-aligned to block size boundaries ", + "[utils][numeric][align_right]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is block_size ") { + const uint64_t aligned_offset = align_right(offset, block_size); + const uint64_t expected_offset = block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + + WHEN(" offset is smaller than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is 0 ") { + const uint64_t aligned_offset = align_right(offset, block_size); + const uint64_t expected_offset = block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the right-aligned offset is the right boundary of the " + "containing block ") { + const uint64_t aligned_offset = align_right(offset, block_size); + const uint64_t expected_offset = + static_cast(offset / block_size + 1) * + block_size; + REQUIRE(aligned_offset == expected_offset); + } + } + } +} + +SCENARIO(" overrun distance can be computed correctly ", + "[utils][numeric][block_overrun]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals 0 ") { + const uint64_t overrun = block_overrun(offset, block_size); + const uint64_t expected_overrun = 0; + REQUIRE(overrun == expected_overrun); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals offset ") { + const uint64_t overrun = block_overrun(offset, block_size); + const uint64_t expected_overrun = offset; + REQUIRE(overrun == expected_overrun); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals block_size - 1 ") { + const uint64_t overrun = block_overrun(offset, block_size); + const uint64_t expected_overrun = block_size - 1; + REQUIRE(overrun == expected_overrun); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals 0 ") { + const uint64_t overrun = block_overrun(offset, block_size); + const uint64_t expected_overrun = 0; + REQUIRE(overrun == expected_overrun); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the computed overrun distance equals the difference between " + "offset and its closest block's left boundary ") { + const uint64_t overrun = block_overrun(offset, block_size); + const uint64_t expected_overrun = + offset - + static_cast(offset / block_size) * block_size; + REQUIRE(overrun == expected_overrun); + } + } + } +} + +SCENARIO(" underrun distance can be computed correctly ", + "[utils][numeric][block_underrun]") { + + GIVEN(" a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size ") { + const uint64_t underrun = + block_underrun(offset, block_size); + const uint64_t expected_underrun = block_size; + REQUIRE(underrun == expected_underrun); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals offset ") { + const uint64_t underrun = + block_underrun(offset, block_size); + const uint64_t expected_underrun = block_size - offset; + REQUIRE(underrun == expected_underrun); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size - 1 ") { + const uint64_t underrun = + block_underrun(offset, block_size); + const uint64_t expected_underrun = block_size - offset; + REQUIRE(underrun == expected_underrun); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals block_size ") { + const uint64_t underrun = block_underrun(offset, block_size); + const uint64_t expected_underrun = block_size; + REQUIRE(underrun == expected_underrun); + } + } + + WHEN(" offset is larger than block_size ") { + + const uint64_t offset = GENERATE_COPY( + take(test_reps, random(block_size, block_size * 31))); + + CAPTURE(offset, block_size); + + THEN(" the computed underrun distance equals the difference between " + "offset and its closest block's right boundary ") { + const uint64_t underrun = block_underrun(offset, block_size); + const uint64_t expected_underrun = + static_cast(offset / block_size + 1) * + block_size - + offset; + REQUIRE(underrun == expected_underrun); + } + } + } +} + +SCENARIO(" chunk IDs can be computed correctly ", + "[utils][numeric][block_index]") { + + GIVEN(" an offset and a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset is smaller than block_size ") { + + AND_WHEN(" offset equals 0 ") { + + const uint64_t offset = 0; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + + AND_WHEN(" offset equals block_size - 1 ") { + + const uint64_t offset = block_size - 1; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 0 ") { + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = 0; + REQUIRE(id == expected_id); + } + } + } + + WHEN(" offset equals block_size ") { + + const uint64_t offset = block_size; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID equals 1 ") { + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = 1; + REQUIRE(id == expected_id); + } + } + + WHEN(" offset is larger than block_size ") { + + AND_WHEN(" block_size < offset < 2^63 - 1 ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, + random(block_size, + std::numeric_limits::max() / 2 - 1))); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + + // The following test specifically exercises issue #137 + AND_WHEN(" offset == 2^63 ") { + + const uint64_t offset = + std::numeric_limits::max() / 2 + 1; + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + + // The following test specifically exercises issue #137 + AND_WHEN(" offset == 2^64 - 1") { + + const uint64_t offset = std::numeric_limits::max(); + + CAPTURE(offset, block_size); + + THEN(" the computed chunk ID is equal to dividing the offset by " + "the block_size ") { + + const uint64_t id = block_index(offset, block_size); + const uint64_t expected_id = offset / block_size; + REQUIRE(id == expected_id); + } + } + } + } +} + +SCENARIO(" the number of chunks involved in an operation can be computed " + "correctly ", + "[utils][numeric][block_count]") { + + GIVEN(" an offset, an operation size, and a block size ") { + + const std::size_t block_size = GENERATE(filter( + [](uint64_t bs) { return is_power_of_2(bs); }, range(0, 100000))); + + WHEN(" offset < block_size ") { + + AND_WHEN(" offset == 0 ") { + + const uint64_t offset = 0; + + AND_WHEN(" offset + size == 0 ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 0 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" 0 < offset + size < block_size ") { + + const size_t size = GENERATE_COPY(take( + test_reps, random(std::size_t{1}, block_size))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" 0 < offset < block_size ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, random(std::size_t{0}, block_size - 1))); + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" 0 < offset + size < block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, block_size - offset))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count equals 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = block_size - offset; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == block_size - 1 ") { + + const size_t offset = block_size - 1; + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 0 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 1; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 1; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size > block_size ") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(block_size + 1, + std::numeric_limits::max()))); + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + } + + WHEN(" offset == block_size ") { + + const uint64_t offset = block_size; + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE(range(0u, test_reps)); + const size_t size = m * block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == M ") { + const std::size_t n = block_count(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY(take( + test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + WHEN(" offset > block_size ") { + + AND_WHEN(" block_size < offset < 2^63 - 1 ") { + + const uint64_t offset = GENERATE_COPY(take( + test_reps, + random(block_size, + std::numeric_limits::max() / 2 - 1))); + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE_COPY(range(2u, test_reps)); + const size_t size = m * block_size - (offset % block_size); + + CAPTURE(offset, size, block_size, m); + + THEN(" the computed block count == M ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - + offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == 2^63 ") { + + const uint64_t offset = + std::numeric_limits::max() / 2 + 1; + + AND_WHEN(" offset + size == block_size ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == M * block_size ") { + const size_t m = GENERATE(range(0u, test_reps)); + const size_t size = m * block_size; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == M ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = m; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size < 2^64 - 1") { + + const size_t size = GENERATE_COPY( + take(test_reps, + random(std::size_t{1}, + std::numeric_limits::max() - + offset))); + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 - 1") { + + const size_t size = + std::numeric_limits::max() - offset; + + THEN(" the computed block count corresponds to the number " + "of blocks involved in the operation ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = + (offset + size) / block_size - + offset / block_size + + ((offset + size) % block_size ? 1u : 0); + + REQUIRE(n == expected_n); + } + } + } + + AND_WHEN(" offset == 2^64 - 1 ") { + + const uint64_t offset = std::numeric_limits::max(); + + AND_WHEN(" offset + size == offset ") { + + const size_t size = 0; + + CAPTURE(offset, size, block_size); + + THEN(" the computed block count == 1 ") { + const std::size_t n = + block_count(offset, size, block_size); + const std::size_t expected_n = 0; + REQUIRE(n == expected_n); + } + } + + AND_WHEN(" offset + size == 2^64 ") { + // TODO: here we should check that we actually hit an + // assert(), but Catch2 does not have facilities to + // support this yet + } + } + } + } +}