From 920490f3592d127d3ba3e1d7f6b884343eaee477 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 09:48:54 +0200 Subject: [PATCH 01/22] Add more options for coverage output --- scripts/dev/coverage.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index b04d2f45a..5ee8bc288 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -170,6 +170,7 @@ def capture(args): pipeline.append([ "lcov", + "--rc", "branch_coverage=1", "--ignore-errors","inconsistent", "--capture", "--quiet" if args.verbosity > 3 else "", @@ -181,6 +182,7 @@ def capture(args): if args.exclusion_patterns: pipeline.append([ "lcov", + "--rc", "branch_coverage=1", "--ignore-errors","inconsistent", "--quiet" if args.verbosity > 3 else "", "--remove=$INPUT", @@ -219,6 +221,7 @@ def merge(args): pipeline.append([ "lcov", + "--rc", "branch_coverage=1", "--ignore-errors","inconsistent", "--quiet" if args.verbosity > 3 else "", *(f"--add-tracefile={t}" for t in tracefiles), @@ -251,9 +254,13 @@ def html_report(args): cmd = Command( cmdline=[ "genhtml", + "--rc", "genhtml_branch_coverage=1", "--quiet" if args.verbosity > 3 else "", + "--show-details", "--legend", "--frames", + "--highlight", + "--demangle-cpp", f"{args.input_tracefile}", f"--prefix={args.prefix}" if args.prefix else "", f"--output-dir={output_dir}"]) -- GitLab From 0b380ff8625fa1e2ad41ca10c6e0066b3a42cb74 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 10:20:03 +0200 Subject: [PATCH 02/22] adding mismatch --- scripts/dev/coverage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index 5ee8bc288..83fd8662d 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -171,7 +171,7 @@ def capture(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent", + "--ignore-errors","inconsistent,mismatch", "--capture", "--quiet" if args.verbosity > 3 else "", "--initial" if args.initial else "", @@ -183,7 +183,7 @@ def capture(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent", + "--ignore-errors","inconsistent,mismatch", "--quiet" if args.verbosity > 3 else "", "--remove=$INPUT", *args.exclusion_patterns, @@ -222,7 +222,7 @@ def merge(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent", + "--ignore-errors","inconsistent,mismatch", "--quiet" if args.verbosity > 3 else "", *(f"--add-tracefile={t}" for t in tracefiles), "--output-file=$OUTPUT"]) -- GitLab From dcc33fc6fe20478c9a190085d6bf13ca29c8d669 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 10:37:30 +0200 Subject: [PATCH 03/22] exclude external --- .gitlab-ci.yml | 5 +++++ scripts/dev/coverage.py | 6 +++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 24483889d..c1b6ab275 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -182,6 +182,7 @@ gkfs:integration: - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/${SUBTEST}.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps @@ -239,6 +240,7 @@ gkfwd:integration: - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/${SUBTEST}.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps @@ -286,6 +288,7 @@ gkfs:unit: - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/unit.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps @@ -326,6 +329,7 @@ gkfs:app: - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/apps.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps @@ -394,6 +398,7 @@ coverage:baseline: - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} -DCOVERAGE_ZEROCOUNT_TRACEFILE=${COVERAGE_PATH}/zerocount.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index 83fd8662d..37e52ac3d 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -171,7 +171,7 @@ def capture(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch", + "--ignore-errors","inconsistent,mismatch,negative", "--capture", "--quiet" if args.verbosity > 3 else "", "--initial" if args.initial else "", @@ -183,7 +183,7 @@ def capture(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch", + "--ignore-errors","inconsistent,mismatch,negative", "--quiet" if args.verbosity > 3 else "", "--remove=$INPUT", *args.exclusion_patterns, @@ -222,7 +222,7 @@ def merge(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch", + "--ignore-errors","inconsistent,mismatch,negative", "--quiet" if args.verbosity > 3 else "", *(f"--add-tracefile={t}" for t in tracefiles), "--output-file=$OUTPUT"]) -- GitLab From 1ab7b7d61a19bf4f664bb09ec4d3af1f07e77aba Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 10:59:54 +0200 Subject: [PATCH 04/22] demangle --- docker/0.9.5/testing/Dockerfile | 2 +- scripts/dev/coverage.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/0.9.5/testing/Dockerfile b/docker/0.9.5/testing/Dockerfile index 6084f2ae1..4f2c3fe86 100644 --- a/docker/0.9.5/testing/Dockerfile +++ b/docker/0.9.5/testing/Dockerfile @@ -8,7 +8,7 @@ RUN \ # required by lcov's genhtml libgd-perl \ libcapture-tiny-perl \ - libdatetime-perl libtimedate-perl \ + libdatetime-perl libtimedate-perl libjson-xs-perl binutils\ cppcheck && \ # install loguru # (required by several of our scripts) diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index 37e52ac3d..e5ff80cfc 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -254,13 +254,14 @@ def html_report(args): cmd = Command( cmdline=[ "genhtml", + "--rc", "genhtml_branch_coverage=1", "--quiet" if args.verbosity > 3 else "", "--show-details", "--legend", "--frames", - "--highlight", "--demangle-cpp", + "--highlight", f"{args.input_tracefile}", f"--prefix={args.prefix}" if args.prefix else "", f"--output-dir={output_dir}"]) -- GitLab From ce60bd14cfc503988665c99b5feecece6ef03d1c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 11:38:40 +0200 Subject: [PATCH 05/22] genhtml option --- scripts/dev/coverage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index e5ff80cfc..7fdfc34c1 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -254,14 +254,12 @@ def html_report(args): cmd = Command( cmdline=[ "genhtml", - - "--rc", "genhtml_branch_coverage=1", + "--rc", "branch_coverage=1", "--quiet" if args.verbosity > 3 else "", "--show-details", "--legend", - "--frames", "--demangle-cpp", - "--highlight", + "--frames", f"{args.input_tracefile}", f"--prefix={args.prefix}" if args.prefix else "", f"--output-dir={output_dir}"]) -- GitLab From 74eb8bce0a8418def0d308c00f7a3a24c0d34ee5 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 11:53:50 +0200 Subject: [PATCH 06/22] avoid errors genhtml --- scripts/dev/coverage.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index 7fdfc34c1..c81a697b0 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -257,6 +257,7 @@ def html_report(args): "--rc", "branch_coverage=1", "--quiet" if args.verbosity > 3 else "", "--show-details", + "--ignore-errors", "inconsistent,corrupt", "--legend", "--demangle-cpp", "--frames", -- GitLab From edfe493cafbbdfdcf2cd336431bf91128b52e6c9 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 14:56:39 +0200 Subject: [PATCH 07/22] added scandir and . .. --- .gitlab-ci.yml | 2 +- src/client/gkfs_functions.cpp | 8 +- src/client/gkfs_libc.cpp | 130 +++++-------- src/client/rpc/forward_metadata.cpp | 4 +- .../harness/gkfs.io/syscall_coverage.cpp | 174 +++++++++++++++++- .../operations/test_read_operations.py | 2 - 6 files changed, 233 insertions(+), 87 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1b6ab275..268d055b0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,7 +59,7 @@ gkfs: - sed -i 's/constexpr auto use_mtime = false;/constexpr auto use_mtime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" - sed -i 's/constexpr auto use_link_cnt = false;/constexpr auto use_link_cnt = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" - sed -i 's/constexpr auto use_blocks = false;/constexpr auto use_blocks = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" - #- sed -i 's/constexpr bool use_dentry_cache = false;/constexpr bool use_dentry_cache = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" + - sed -i 's/constexpr bool use_dentry_cache = false;/constexpr bool use_dentry_cache = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" #- sed -i 's/constexpr auto zero_buffer_before_read = false;/constexpr auto zero_buffer_before_read = true;/g' "${CI_PROJECT_DIR}/include/config.hpp" #- sed -i 's/constexpr auto implicit_data_removal = true;/constexpr auto implicit_data_removal = false;/g' "${CI_PROJECT_DIR}/include/config.hpp" # use ccache diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index dea59685d..be376d4d6 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1540,6 +1540,11 @@ gkfs_rmdir(const std::string& path) { LOG(DEBUG, "Error: Path '{}' err code '{}' ", path, strerror(errno)); return -1; } + if(!S_ISDIR(md->mode())) { + LOG(DEBUG, "{}() Path is not a directory", __func__); + errno = ENOTDIR; + return -1; + } auto ret = gkfs::rpc::forward_get_dirents(path); err = ret.first; if(err) { @@ -1548,7 +1553,8 @@ gkfs_rmdir(const std::string& path) { } assert(ret.second); auto open_dir = ret.second; - if(open_dir->size() != 0) { + // We use . and .. + if(open_dir->size() != 2) { errno = ENOTEMPTY; return -1; } diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index 0a9bda27f..782d47c14 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -427,6 +427,8 @@ NOLOGDLSYM_WRAPPER(char*, fgets, (char* str, int n, FILE* stream), NOLOGDLSYM_WRAPPER(int, fflush, (FILE * stream), (stream), "fflush") DLSYM_WRAPPER(int, fchown, (int fd, uid_t owner, gid_t group), (fd, owner, group), "fchown") + +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS DLSYM_WRAPPER(int, futimes, (int fd, const struct timeval times[2]), (fd, times), "futimes") DLSYM_WRAPPER(int, utimes, (const char* path, const struct timeval times[2]), @@ -439,12 +441,7 @@ DLSYM_WRAPPER(ssize_t, llistxattr, (const char* path, char* list, size_t size), DLSYM_WRAPPER(ssize_t, flistxattr, (int fd, char* list, size_t size), (fd, list, size), "flistxattr") -// This getxattr is not used, it goes directly to syscall so cannot be -// intercepted - - -// DLSYM_WRAPPER(long, syscall, (long number, char* cmd, void* arg, void* env), -// (number, cmd, arg, env), "syscall") +#endif DLSYM_WRAPPER(DIR*, fdopendir, (int fd), (fd), "fdopendir") @@ -926,6 +923,49 @@ unlink(const char* path) { } +int +statx(int dirfd, const char* path, int flags, unsigned int mask, + struct statx* statxbuf) { + initializeGekko(); + bool follow = (flags & AT_SYMLINK_NOFOLLOW) == 0; + DEBUG_INFO("[GKFS] {} -> follow {}", path, follow); + memset(statxbuf, 0, sizeof(struct statx)); + GKFS_PATH_OPERATION_DIR(statx, dirfd, path, flags, mask, statxbuf, follow) + GKFS_FALLBACK(statx, dirfd, path, flags, mask, statxbuf); +} + + +int +fstat64(int fd, struct stat64* buf) { + initializeGekko(); + if(CTX->interception_enabled() && is_gkfs_fd(fd)) { + struct stat st; + DEBUG_INFO("[GKFS] {}", CTX->file_map()->get(fd)->path()); + int res = gkfs::syscall::gkfs_stat(CTX->file_map()->get(fd)->path(), + &st, true, true); + if(res == 0) + convert(&st, buf); + return res; + } + GKFS_FALLBACK(fstat64, fd, buf); +} + +int +fstat(int fd, struct stat* buf) { + initializeGekko(); + if(CTX->interception_enabled() && is_gkfs_fd(fd)) { + DEBUG_INFO("[GKFS] {}", CTX->file_map()->get(fd)->path()); + + // The fd could be a renamed file, thus when doing gkfs_stat we will get + // a is not found. + int res = gkfs::syscall::gkfs_stat(CTX->file_map()->get(fd)->path(), + buf, true, true); + return res; + } + GKFS_FALLBACK(fstat, fd, buf); +} + +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS int __xstat(int ver, const char* path, struct stat* buf) { GKFS_PATH_OPERATION(stat, AT_FDCWD, path, buf) @@ -933,7 +973,6 @@ __xstat(int ver, const char* path, struct stat* buf) { return res; } - int __xstat64(int ver, const char* path, struct stat64* buf) { struct stat st; @@ -960,16 +999,6 @@ __xstat64(int ver, const char* path, struct stat64* buf) { } -int -statx(int dirfd, const char* path, int flags, unsigned int mask, - struct statx* statxbuf) { - initializeGekko(); - bool follow = (flags & AT_SYMLINK_NOFOLLOW) == 0; - DEBUG_INFO("[GKFS] {} -> follow {}", path, follow); - GKFS_PATH_OPERATION_DIR(statx, dirfd, path, flags, mask, statxbuf, follow) - GKFS_FALLBACK(statx, dirfd, path, flags, mask, statxbuf); -} - /** * @brief lxstat wrapper function. Should be called with resolve_last_link = * false on the CI. @@ -1058,36 +1087,6 @@ __fxstat64(int ver, int fd, struct stat64* buf) { GKFS_FALLBACK(fxstat64, ver, fd, buf); } -int -fstat64(int fd, struct stat64* buf) { - initializeGekko(); - if(CTX->interception_enabled() && is_gkfs_fd(fd)) { - struct stat st; - DEBUG_INFO("[GKFS] {}", CTX->file_map()->get(fd)->path()); - int res = gkfs::syscall::gkfs_stat(CTX->file_map()->get(fd)->path(), - &st, true, true); - if(res == 0) - convert(&st, buf); - return res; - } - GKFS_FALLBACK(fstat64, fd, buf); -} - -int -fstat(int fd, struct stat* buf) { - initializeGekko(); - if(CTX->interception_enabled() && is_gkfs_fd(fd)) { - DEBUG_INFO("[GKFS] {}", CTX->file_map()->get(fd)->path()); - - // The fd could be a renamed file, thus when doing gkfs_stat we will get - // a is not found. - int res = gkfs::syscall::gkfs_stat(CTX->file_map()->get(fd)->path(), - buf, true, true); - return res; - } - GKFS_FALLBACK(fstat, fd, buf); -} - int __fxstatat64(int ver, int dfd, const char* path, struct stat64* buf, int flags) { @@ -1112,7 +1111,7 @@ __fxstatat64(int ver, int dfd, const char* path, struct stat64* buf, } GKFS_FALLBACK(fstatat64, dfd, path, buf, flags); } - +#endif int stat(int ver, const char* path, struct stat* buf) { @@ -2100,7 +2099,7 @@ chmod(const char* path, mode_t mode) { GKFS_FALLBACK(chmod, path, mode); } -// futimes +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS int futimes(int fd, const struct timeval tv[2]) { initializeGekko(); @@ -2166,6 +2165,7 @@ llistxattr(const char* path, char* list, size_t size) { } GKFS_FALLBACK(llistxattr, path, list, size); } +#endif int lstat(const char* path, struct stat* st) { @@ -2343,35 +2343,6 @@ pwritev2(int fd, const struct iovec* iov, int iovcnt, off_t offset, int flags) { GKFS_FALLBACK(pwritev2, fd, iov, iovcnt, offset, flags) } - -// aio Operations (write) -// /* -// #include - -// struct aiocb { - - -// int aio_fildes; /* File descriptor */ -// off_t aio_offset; /* File offset */ -// volatile void *aio_buf; /* Location of buffer */ -// size_t aio_nbytes; /* Length of transfer */ -// int aio_reqprio; /* Request priority */ -// struct sigevent aio_sigevent; /* Notification method */ -// int aio_lio_opcode; /* Operation to be -// performed; -// lio_listio() only */ - -// /* Various implementation-internal fields not shown */ -// }; - -// /* Operation codes for 'aio_lio_opcode': */ - -// enum { LIO_READ, LIO_WRITE, LIO_NOP }; -// Possible values for aio_sigevent.sigev_notify are SIGEV_NONE, -// SIGEV_SIGNAL, and SIGEV_THREAD. See sigevent(3type) for further de‐ -// tails. - -// */ /* Add result to tracking list */ typedef struct ResultEntry { struct aiocb* aiocbp; @@ -2689,6 +2660,3 @@ _ZNSt10filesystem10remove_allERKNS_7__cxx114pathE( } return real_remove_all_ptr(p); } - - -// futimens \ No newline at end of file diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index 1c2688e79..ad0882cd2 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -780,7 +780,9 @@ forward_get_dirents(const string& path) { bool* bool_ptr = reinterpret_cast(base_ptr); char* names_ptr = reinterpret_cast(base_ptr) + (out.dirents_size() * sizeof(bool)); - + // Add special files like an standard fs. + open_dir->add(".", gkfs::filemap::FileType::directory); + open_dir->add("..", gkfs::filemap::FileType::directory); for(std::size_t j = 0; j < out.dirents_size(); j++) { gkfs::filemap::FileType ftype = diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index 7120f49e7..0aa98e401 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -1125,6 +1125,177 @@ test_xattr_uncovered(const std::string& base_dir) { } +// Helper function to create an empty file +bool +create_empty_file(const char* path) { + int fd = creat(path, 0644); + if(fd < 0) { + perror("creat failed in helper"); + return false; + } + close(fd); + return true; +} + +// Helper function to free memory allocated by scandir +void +cleanup_scandir_result(struct dirent** namelist, int count) { + if(namelist == nullptr || count < 0) + return; // Handle scandir error case + for(int i = 0; i < count; ++i) { + free(namelist[i]); + } + free(namelist); +} + +// Custom filter function: select entries starting with 'f' +int +filter_f_files(const struct dirent* entry) { + // Exclude . and .. implicitly if needed, but scandir often does this + // if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) + // { + // return 0; + // } + return (entry->d_name[0] == 'f'); +} + +// Custom comparison function (alphabetical sort) +int +compare_dirent_alpha(const struct dirent** a, const struct dirent** b) { + return strcmp((*a)->d_name, (*b)->d_name); +} + + +// Main test function for scandir +void +test_scandir(const std::string& base_dir) { + std::string testdir_path = make_path(base_dir, "scandir_test_dir"); + std::string file1_path = make_path(testdir_path, "file_a.txt"); + std::string file2_path = make_path(testdir_path, "file_z.txt"); + std::string file3_path = make_path(testdir_path, ".hiddenfile"); + std::string subdir_path = make_path(testdir_path, "subdir_b"); + + // 1. Setup: Create directory and populate it + errno = 0; + if(mkdir(testdir_path.c_str(), 0755) != 0 && errno != EEXIST) { + perror("Failed to create test directory"); + assert(false); + } + assert(create_empty_file(file1_path.c_str())); + assert(create_empty_file(file2_path.c_str())); + assert(create_empty_file(file3_path.c_str())); + errno = 0; + if(mkdir(subdir_path.c_str(), 0755) != 0 && errno != EEXIST) { + perror("Failed to create test subdirectory"); + assert(false); + } + + struct dirent** namelist = nullptr; + int count = -1; + std::vector expected_names = { + ".", "..", ".hiddenfile", "file_a.txt", "file_z.txt", "subdir_b"}; + + // 2. Basic Test (GekkoFS path, no filter/sort) + errno = 0; + count = scandir(testdir_path.c_str(), &namelist, NULL, NULL); + if(count < 0) { + perror("scandir (basic) failed"); + } else { + std::vector found_names; + for(int i = 0; i < count; ++i) { + std::cout << " - " << namelist[i]->d_name << std::endl; + found_names.push_back(namelist[i]->d_name); + } + // The order isn't guaranteed without a sort, just check the count + // mostly + assert(count == (int) expected_names.size()); + cleanup_scandir_result(namelist, count); + namelist = nullptr; // Reset for next test + } + + // 3. Filter Test (GekkoFS path) + errno = 0; + count = scandir(testdir_path.c_str(), &namelist, filter_f_files, NULL); + if(count < 0) { + perror("scandir (filter) failed"); + } else { + std::vector found_names; + for(int i = 0; i < count; ++i) { + std::cout << " - " << namelist[i]->d_name << std::endl; + found_names.push_back(namelist[i]->d_name); + } + // We expect "file_a.txt" and "file_z.txt" + assert(count == 2); + bool found_a = false, found_z = false; + for(const auto& name : found_names) { + if(name == "file_a.txt") + found_a = true; + if(name == "file_z.txt") + found_z = true; + } + assert(found_a && found_z); + cleanup_scandir_result(namelist, count); + namelist = nullptr; + } + + // 4. Comparator Test (GekkoFS path) + errno = 0; + count = scandir(testdir_path.c_str(), &namelist, NULL, + compare_dirent_alpha); + if(count < 0) { + perror("scandir (sort) failed"); + } else { + std::vector found_names; + for(int i = 0; i < count; ++i) { + std::cout << " - " << namelist[i]->d_name << std::endl; + found_names.push_back(namelist[i]->d_name); + } + assert(count == (int) expected_names.size()); + // Verify sorted order + std::sort(expected_names.begin(), + expected_names.end()); // Sort our expectation + assert(found_names.size() == expected_names.size()); + bool order_ok = true; + for(size_t i = 0; i < found_names.size(); ++i) { + if(found_names[i] != expected_names[i]) { + order_ok = false; + std::cerr << "Sort order mismatch at index " << i + << ": expected '" << expected_names[i] << "', got '" + << found_names[i] << "'" << std::endl; + } + } + assert(order_ok); + cleanup_scandir_result(namelist, count); + namelist = nullptr; + } + + // 5. Fallback Test (non-GekkoFS path) + const char* fallback_dir = "/tmp"; // Choose a common system directory + errno = 0; + count = scandir(fallback_dir, &namelist, NULL, NULL); + if(count < 0) { + perror("scandir (fallback) failed"); + // This might fail due to permissions on /tmp, which is acceptable for + // the test + } else { + for(int i = 0; i < count && i < 5; ++i) { // Print only a few + std::cout << " - " << namelist[i]->d_name << std::endl; + } + if(count > 5) + std::cout << " - ..." << std::endl; + cleanup_scandir_result(namelist, count); + namelist = nullptr; + } + + // 6. Cleanup + unlink(file1_path.c_str()); + unlink(file2_path.c_str()); + unlink(file3_path.c_str()); + rmdir(subdir_path.c_str()); + rmdir(testdir_path.c_str()); +} + + int libc_missing(std::string base_path) { @@ -1142,7 +1313,8 @@ libc_missing(std::string base_path) { test_creat_pread_pwrite64(base_path); test_vector_io_uncovered(base_path); // readv, preadv2, pwritev2 test_directory_ops_uncovered(base_path); // mkdirat, opendir64, readdir64, - // seekdir, telldir, lseek64 + // seekdir, telldir, lseek64 + test_scandir(base_path); test_mkstemp(base_path); // mkstemp test_stat_variants_uncovered( base_path); // *stat*64, lstat*, fstat*, fstatat*, statx diff --git a/tests/integration/operations/test_read_operations.py b/tests/integration/operations/test_read_operations.py index 335cdcddd..0ba9f3f1e 100644 --- a/tests/integration/operations/test_read_operations.py +++ b/tests/integration/operations/test_read_operations.py @@ -255,7 +255,6 @@ def test_pread_libc(gkfs_daemon, gkfs_clientLibc): assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes -@pytest.mark.skip(reason="readv not implemented in libc") def test_readv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" @@ -288,7 +287,6 @@ def test_readv_libc(gkfs_daemon, gkfs_clientLibc): assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes -@pytest.mark.skip(reason="preadv not implemented in libc") def test_preadv_libc(gkfs_daemon, gkfs_clientLibc): file = gkfs_daemon.mountdir / "file" -- GitLab From d97c185542c459f812a7beece67e3201f9be68d6 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 15:43:28 +0200 Subject: [PATCH 08/22] added . and .. in dcache --- src/client/gkfs_functions.cpp | 2 ++ src/client/gkfs_libc.cpp | 1 - .../directories/test_directories.py | 28 ++++++++++++------- .../harness/gkfs.io/syscall_coverage.cpp | 6 ++-- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index be376d4d6..7c3e3d03c 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -1489,6 +1489,8 @@ gkfs_opendir(const std::string& path) { } int cnt = 0; // Collect and process results + ret.second->add(".", gkfs::filemap::FileType::directory); + ret.second->add("..", gkfs::filemap::FileType::directory); for(auto& fut : dcache_futures) { auto res = fut.get(); // Wait for the RPC result auto& open_dir = *res.second; diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index 782d47c14..ace26ed43 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -929,7 +929,6 @@ statx(int dirfd, const char* path, int flags, unsigned int mask, initializeGekko(); bool follow = (flags & AT_SYMLINK_NOFOLLOW) == 0; DEBUG_INFO("[GKFS] {} -> follow {}", path, follow); - memset(statxbuf, 0, sizeof(struct statx)); GKFS_PATH_OPERATION_DIR(statx, dirfd, path, flags, mask, statxbuf, follow) GKFS_FALLBACK(statx, dirfd, path, flags, mask, statxbuf); } diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 018a8b864..194146359 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -92,7 +92,7 @@ def test_mkdir(gkfs_daemon, gkfs_client): ret = gkfs_client.readdir(topdir) # XXX: This might change in the future if we add '.' and '..' - assert len(ret.dirents) == 0 + assert len(ret.dirents) == 2 # close directory # TODO: disabled for now because we have no way to keep DIR* alive @@ -117,11 +117,13 @@ def test_mkdir(gkfs_daemon, gkfs_client): ret = gkfs_client.readdir(gkfs_daemon.mountdir) # XXX: This might change in the future if we add '.' and '..' - assert len(ret.dirents) == 1 - assert ret.dirents[0].d_name == 'top' - assert ret.dirents[0].d_type == 4 # DT_DIR + assert len(ret.dirents) == 3 + assert ret.dirents[2].d_name == 'top' + assert ret.dirents[2].d_type == 4 # DT_DIR expected = [ + ( ".", 4 ), # DT_DIR + ( "..", 4 ), ( dir_a.name, 4 ), # DT_DIR ( dir_b.name, 4 ), ( file_a.name, 8 ) # DT_REG @@ -147,6 +149,8 @@ def test_mkdir(gkfs_daemon, gkfs_client): assert ret.retval == 0 expected = [ + ( ".", 4 ), # DT_DIR + ( "..", 4 ), ( topdir.name, 4 ), # DT_DIR ( longer.name, 4 ), # DT_DIR ] @@ -166,6 +170,8 @@ def test_mkdir(gkfs_daemon, gkfs_client): assert ret.retval == 0 expected = [ + ( ".", 4 ), # DT_DIR + ( "..", 4 ), ( topdir.name, 4 ), # DT_DIR ( longer.name, 4 ), # DT_DIR ] @@ -178,6 +184,8 @@ def test_mkdir(gkfs_daemon, gkfs_client): assert d.d_type == e[1] expected = [ + ( ".", 4 ), # DT_DIR + ( "..", 4 ), ( subdir_a.name, 4 ), # DT_DIR ] @@ -219,19 +227,19 @@ def test_finedir(gkfs_daemon, gkfs_client): ret = gkfs_client.readdir(topdir) # XXX: This might change in the future if we add '.' and '..' - assert len(ret.dirents) == 0 + assert len(ret.dirents) == 2 # populate top directory for files in range (1,4): ret = gkfs_client.directory_validate( topdir, 1) - assert ret.retval == files + assert ret.retval == files+2 ret = gkfs_client.directory_validate( topdir, 1000) - assert ret.retval == 1000+3 + assert ret.retval == 1000+3+2 def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): @@ -321,19 +329,19 @@ def test_finedir_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_client_proxy): ret = gkfs_client_proxy.readdir(topdir) # XXX: This might change in the future if we add '.' and '..' - assert len(ret.dirents) == 0 + assert len(ret.dirents) == 2 # populate top directory for files in range (1,4): ret = gkfs_client_proxy.directory_validate( topdir, 1) - assert ret.retval == files + assert ret.retval == files+2 ret = gkfs_client_proxy.directory_validate( topdir, 1000) - assert ret.retval == 1000+3 + assert ret.retval == 1000+3+2 def test_extended_proxy(gkfs_daemon_proxy, gkfs_proxy, gkfs_shell_proxy, gkfs_client_proxy): diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index 0aa98e401..6d3624967 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -1203,7 +1203,6 @@ test_scandir(const std::string& base_dir) { } else { std::vector found_names; for(int i = 0; i < count; ++i) { - std::cout << " - " << namelist[i]->d_name << std::endl; found_names.push_back(namelist[i]->d_name); } // The order isn't guaranteed without a sort, just check the count @@ -1221,7 +1220,6 @@ test_scandir(const std::string& base_dir) { } else { std::vector found_names; for(int i = 0; i < count; ++i) { - std::cout << " - " << namelist[i]->d_name << std::endl; found_names.push_back(namelist[i]->d_name); } // We expect "file_a.txt" and "file_z.txt" @@ -1313,9 +1311,9 @@ libc_missing(std::string base_path) { test_creat_pread_pwrite64(base_path); test_vector_io_uncovered(base_path); // readv, preadv2, pwritev2 test_directory_ops_uncovered(base_path); // mkdirat, opendir64, readdir64, - // seekdir, telldir, lseek64 + // seekdir, telldir, lseek64 test_scandir(base_path); - test_mkstemp(base_path); // mkstemp + test_mkstemp(base_path); // mkstemp test_stat_variants_uncovered( base_path); // *stat*64, lstat*, fstat*, fstatat*, statx test_rename_variants_uncovered(base_path); // rename, renameat2 -- GitLab From 5e410645e624293487db95ddacd2b19e6837c0b4 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 16:10:44 +0200 Subject: [PATCH 09/22] rename test also checks . and .. --- tests/integration/rename/test_rename_operation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/rename/test_rename_operation.py b/tests/integration/rename/test_rename_operation.py index c5b6f702a..568162222 100644 --- a/tests/integration/rename/test_rename_operation.py +++ b/tests/integration/rename/test_rename_operation.py @@ -379,7 +379,7 @@ def test_rename_delete(gkfs_daemon, gkfs_client): assert ret.statbuf.st_size == len(buf) ret = gkfs_client.readdir(gkfs_daemon.mountdir) - assert len(ret.dirents) == 1 + assert len(ret.dirents) == 3 ret = gkfs_client.unlink(fileold) # Remove original file (error) assert ret.retval != 0 @@ -391,7 +391,7 @@ def test_rename_delete(gkfs_daemon, gkfs_client): assert ret.retval == -1 ret = gkfs_client.readdir(gkfs_daemon.mountdir) - assert len(ret.dirents) == 0 + assert len(ret.dirents) == 2 -- GitLab From 356b65954f5fe925708f155bddb3aee54c2d1260 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 16:24:45 +0200 Subject: [PATCH 10/22] removed some cout in tests --- tests/integration/harness/gkfs.io/syscall_coverage.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index 6d3624967..38d047462 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -1245,7 +1245,6 @@ test_scandir(const std::string& base_dir) { } else { std::vector found_names; for(int i = 0; i < count; ++i) { - std::cout << " - " << namelist[i]->d_name << std::endl; found_names.push_back(namelist[i]->d_name); } assert(count == (int) expected_names.size()); @@ -1276,11 +1275,7 @@ test_scandir(const std::string& base_dir) { // This might fail due to permissions on /tmp, which is acceptable for // the test } else { - for(int i = 0; i < count && i < 5; ++i) { // Print only a few - std::cout << " - " << namelist[i]->d_name << std::endl; - } - if(count > 5) - std::cout << " - ..." << std::endl; + cleanup_scandir_result(namelist, count); namelist = nullptr; } -- GitLab From f02253b25a61cdc09632c7b8323c29cfb366b4b4 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 19:39:39 +0200 Subject: [PATCH 11/22] Remove resolve old if GKFS_USE_LEGACY_PATH_RESOLVE is false --- include/client/path.hpp | 2 ++ src/client/path.cpp | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/include/client/path.hpp b/include/client/path.hpp index 91cf13a2a..87acc327a 100644 --- a/include/client/path.hpp +++ b/include/client/path.hpp @@ -54,10 +54,12 @@ resolve(const std::string& path, bool resolve_last_link = true); std::pair resolve_new(const std::string& path, bool resolve_last_link = true); +#ifdef GKFS_USE_LEGACY_PATH_RESOLVE [[deprecated( "Use GKFS_USE_LEGACY_PATH_RESOLVE to use old implementation")]] bool resolve(const std::string& path, std::string& resolved, bool resolve_last_link = true); +#endif std::string get_sys_cwd(); diff --git a/src/client/path.cpp b/src/client/path.cpp index b319cc8e9..910d52c14 100644 --- a/src/client/path.cpp +++ b/src/client/path.cpp @@ -82,6 +82,7 @@ static const string excluded_paths[2] = {"sys/", "proc/"}; * "head", "no"]) == 2; tot_comp == 4; * ``` */ +#ifdef GKFS_USE_LEGACY_PATH_RESOLVE unsigned int match_components(const string& path, unsigned int& path_components, const ::vector& components) { @@ -111,6 +112,7 @@ match_components(const string& path, unsigned int& path_components, path_components = processed_components; return matched; } +#endif static char* (*real_realpath)(const char* path, char* resolved_path) = nullptr; @@ -233,6 +235,7 @@ resolve_new(const string& path, bool resolve_last_link) { * returns true if the resolved path fall inside GekkoFS namespace, * and false otherwise. */ +#ifdef GKFS_USE_LEGACY_PATH_RESOLVE bool resolve(const string& path, string& resolved, bool resolve_last_link) { @@ -370,6 +373,7 @@ resolve(const string& path, string& resolved, bool resolve_last_link) { LOG(DEBUG, "external: \"{}\"", resolved); return false; } +#endif string get_sys_cwd() { -- GitLab From 159d3e6433b1b52bd5cb5fc30b031131aacb0c26 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 8 May 2025 19:46:26 +0200 Subject: [PATCH 12/22] close range shield with SYS_close_range --- tests/integration/harness/gkfs.io/syscall_coverage.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index 38d047462..c95e882ae 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -193,7 +193,7 @@ public: } }; -#ifdef close_range +#ifdef SYS_close_range void test_close_range(const fs::path& base_path) { TestDirectory dir(base_path, "close_range"); @@ -1275,7 +1275,7 @@ test_scandir(const std::string& base_dir) { // This might fail due to permissions on /tmp, which is acceptable for // the test } else { - + cleanup_scandir_result(namelist, count); namelist = nullptr; } @@ -1292,7 +1292,7 @@ test_scandir(const std::string& base_dir) { int libc_missing(std::string base_path) { -#ifdef close_range +#ifdef SYS_close_range test_close_range(base_path); #endif test_creat(base_path); -- GitLab From 675e877b8ff94f2c39369321dbdab02093240a9d Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 08:21:10 +0200 Subject: [PATCH 13/22] added locktest --- tests/apps/CMakeLists.txt | 13 ++++ tests/apps/lockfile.sh | 74 +++++++++++++++++++ .../integration/syscalls/test_malleability.py | 18 +++++ 3 files changed, 105 insertions(+) create mode 100755 tests/apps/lockfile.sh diff --git a/tests/apps/CMakeLists.txt b/tests/apps/CMakeLists.txt index 1b3ac6fc6..ecbcc9f41 100644 --- a/tests/apps/CMakeLists.txt +++ b/tests/apps/CMakeLists.txt @@ -68,6 +68,19 @@ set_tests_properties(wacomm WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) +add_test( + NAME lockfile + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/lockfile.sh +) + +set_tests_properties(lockfile + PROPERTIES + TIMEOUT 9000 + LABELS "apps::all" + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + + if(GKFS_INSTALL_TESTS) install(DIRECTORY . DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/apps diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh new file mode 100755 index 000000000..2b9e5a819 --- /dev/null +++ b/tests/apps/lockfile.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/app/lockfile/original +export MNT=$IO/tests/app/lockfile/mnt +export ROOT=$IO/tests/app/lockfile/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_lockfile.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE + +export APP=/root/S3D-IO/s3d_io.x +export PARAMS="30 30 30 1 1 1 0 F" + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + + +# Run s3d-io.x with LD_PRELOAD and capture exit code +echo "Running lockfile test..." + +# If we want to use mpi we may need : OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 mpirun -np 2 +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc $OUTSIDE + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfile/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfile/client.txt +export LIBGKFS_LOG=all +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS_LIBC $APP $PARAMS $MNT/libc & +S3D=$! +# loop 10 times and do "hexdump pressure_wave_test.0.000E+00.field.nc" +for i in {1..10}; do + sleep 1 + LIBGKFS_PROTECT_FILES_CONSUMER=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS_LIBC md5sum $MNT/libc/pressure_wave_test.0.000E+00.field.nc + # If S3D does not exist break + if ! ps -p $S3D > /dev/null; then + break + fi +done + +wait $S3D + +export LIBGKFS_LOG=none +export LD_PRELOAD=$GKFS + +rm -rf $MNT/libc +unset LD_PRELOAD + + +# Stop the daemon +kill $DAEMON_PID + +exit 0 \ No newline at end of file diff --git a/tests/integration/syscalls/test_malleability.py b/tests/integration/syscalls/test_malleability.py index 701daadc1..62a056344 100644 --- a/tests/integration/syscalls/test_malleability.py +++ b/tests/integration/syscalls/test_malleability.py @@ -80,5 +80,23 @@ def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d00.shutdown() d01.shutdown() + +def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + import time + d00 = gkfwd_daemon_factory.create() + # Add "#FS_INSTANCE_END" in the file with name d00.hostfile + + time.sleep(5) + cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + assert cmd.exit_code != 0 + + with open(d00.hostfile, 'a') as f: + f.write("#FS_INSTANCE_END\n") + + cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + assert cmd.exit_code != 0 + + d00.shutdown() + \ No newline at end of file -- GitLab From f2f55c18f8e6684da1911f84581655210df1ce22 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 08:48:46 +0200 Subject: [PATCH 14/22] whitelist 255 error codes in tests --- tests/integration/harness/gkfs.py | 4 ++-- tests/integration/syscalls/test_malleability.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index cfe81f486..9d4c3988f 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -931,7 +931,7 @@ class ShellClient: # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, - # _ok_code=list(range(0, 256)) + _ok_code=list(range(0, 256)) ) logger.debug(f"program stdout: {proc.stdout}") @@ -1083,7 +1083,7 @@ class ShellClientLibc: # _err=sys.stderr, _timeout=timeout, _timeout_signal=timeout_signal, - # _ok_code=list(range(0, 256)) + _ok_code=list(range(0, 256)) ) def run(self, cmd, *args, timeout=60, timeout_signal=signal.SIGKILL): diff --git a/tests/integration/syscalls/test_malleability.py b/tests/integration/syscalls/test_malleability.py index 62a056344..12eab555b 100644 --- a/tests/integration/syscalls/test_malleability.py +++ b/tests/integration/syscalls/test_malleability.py @@ -89,13 +89,14 @@ def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): time.sleep(5) cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) assert cmd.exit_code != 0 + assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" with open(d00.hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) assert cmd.exit_code != 0 - + assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" d00.shutdown() -- GitLab From 843f6b814900aa66c970e29318037a5c0b055570 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 09:29:44 +0200 Subject: [PATCH 15/22] reduced apps docker --- docker/0.9.5/apps/Dockerfile | 2 +- docker/0.9.5/testing/Dockerfile | 68 +++++++++++++++++++++------------ tests/apps/lockfile.sh | 4 +- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/docker/0.9.5/apps/Dockerfile b/docker/0.9.5/apps/Dockerfile index 2ec72d3f7..89317fa97 100644 --- a/docker/0.9.5/apps/Dockerfile +++ b/docker/0.9.5/apps/Dockerfile @@ -60,7 +60,7 @@ RUN mkdir -p input processed output restarts && \ cd input && \ wget -q https://api.meteo.uniparthenope.it/files/rms3/d03/history/2025/01/01/rms3_d03_20250101Z0000.nc && \ mv rms3_d03_20250101Z0000.nc input.nc && \ - ncks -7 -L 9 --ppc default=4 input.nc rms3_d03_20250101Z0000.nc && \ + ncks -7 -L 9 --qnt default=1 --ppc default=1 input.nc rms3_d03_20250101Z0000.nc && \ tar cvzf rms.tar.gz rms3_d03_20250101Z0000.nc && \ rm rms3_d03_20250101Z0000.nc input.nc diff --git a/docker/0.9.5/testing/Dockerfile b/docker/0.9.5/testing/Dockerfile index 4f2c3fe86..77a2f2172 100644 --- a/docker/0.9.5/testing/Dockerfile +++ b/docker/0.9.5/testing/Dockerfile @@ -1,36 +1,54 @@ FROM gekkofs/deps:0.9.5 -LABEL Description="Debian-based environment to test GekkoFS" +LABEL Description="Debian-based environment to test GekkoFS with LCOV, cppcheck, and reporting tools" -RUN \ - apt-get update && \ +# Set DEBIAN_FRONTEND to noninteractive to prevent apt-get from prompting +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ apt-get install -y --no-install-recommends \ - # required by lcov's genhtml + # LCOV's genhtml dependencies libgd-perl \ libcapture-tiny-perl \ - libdatetime-perl libtimedate-perl libjson-xs-perl binutils\ - cppcheck && \ - # install loguru - # (required by several of our scripts) - pip3 install --break-system-packages loguru && \ - # install lcov_cobertura - # (required to produce Cobertura XML reports) - pip3 install --break-system-packages lcov_cobertura && \ - # install lcov - # (required to produce partial coverage reports in parallel runs) - curl -OL https://github.com/linux-test-project/lcov/releases/download/v2.3.1/lcov-2.3.1.tar.gz && \ - tar xfz lcov-2.3.1.tar.gz && \ - cd lcov-2.3.1 && \ + libdatetime-perl \ + libtimedate-perl \ + libjson-xs-perl \ + # Other system tools + binutils \ + cppcheck \ + curl \ + git \ + make \ + # Python3 and pip are assumed to be in gekkofs/deps:0.9.5 + # If not, add python3 python3-pip + && \ + # Cleanup apt cache + rm -rf /var/lib/apt/lists/* && \ + apt-get clean + +# Install Python packages +RUN pip3 install --no-cache-dir --break-system-packages \ + loguru \ + lcov_cobertura + +# Install LCOV from source +WORKDIR /tmp +RUN LCOV_VERSION="2.3.1" && \ + curl -sSOL "https://github.com/linux-test-project/lcov/releases/download/v${LCOV_VERSION}/lcov-${LCOV_VERSION}.tar.gz" && \ + tar xfz "lcov-${LCOV_VERSION}.tar.gz" && \ + cd "lcov-${LCOV_VERSION}" && \ make install && \ cd .. && \ - git clone https://gitlab.com/ahogen/cppcheck-codequality.git && \ + rm -rf "lcov-${LCOV_VERSION}.tar.gz" "lcov-${LCOV_VERSION}" + +WORKDIR /opt +RUN git clone https://gitlab.com/ahogen/cppcheck-codequality.git && \ cd cppcheck-codequality && \ + git checkout 61f6ae3b150d377e9987dce9d1c2859ef08bea6f && \ sed -i 's/python = ">=3\.7,<3\.13"/python = ">=3.7,<3.14"/' pyproject.toml && \ sed -i 's/python = ">=3\.7,<3\.13"/python = ">=3.7,<3.14"/' poetry.lock && \ - pip3 install . --break-system-packages && \ - cd .. &&\ - # cleanup - rm -rf /var/lib/apt/lists/* && \ - apt-get clean && \ - apt-get autoclean && \ - rm -rf ./lcov-2.3.1.* + pip3 install --no-cache-dir --break-system-packages . && \ + cd .. && \ + rm -rf cppcheck-codequality + +WORKDIR / \ No newline at end of file diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index 2b9e5a819..f047aa4a0 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -47,12 +47,12 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfi export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS_LIBC $APP $PARAMS $MNT/libc & +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LD_PRELOAD=$GKFS_LIBC $APP $PARAMS $MNT/libc & S3D=$! # loop 10 times and do "hexdump pressure_wave_test.0.000E+00.field.nc" for i in {1..10}; do sleep 1 - LIBGKFS_PROTECT_FILES_CONSUMER=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS_LIBC md5sum $MNT/libc/pressure_wave_test.0.000E+00.field.nc + LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC md5sum $MNT/libc/pressure_wave_test.0.000E+00.field.nc # If S3D does not exist break if ! ps -p $S3D > /dev/null; then break -- GitLab From 44c9f39c6b11a5ad3e565c0d6cfae8684ea1dd9a Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 09:32:37 +0200 Subject: [PATCH 16/22] updated md5sum --- tests/apps/wacomm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apps/wacomm.sh b/tests/apps/wacomm.sh index ee49893c2..367e90a72 100755 --- a/tests/apps/wacomm.sh +++ b/tests/apps/wacomm.sh @@ -91,7 +91,7 @@ echo $LIBC_RES #ORG_SIZE=`md5sum $OUTSIDE/output/wacomm_his_20250101Z00.nc | cut -c -32` #ORG_RES=`md5sum $OUTSIDE/restarts/20250101Z01.json | cut -c -32` rm -rf $OUTSIDE -ORG_SIZE="30b3eef3ae4841875358b000fc55f77d" +ORG_SIZE="8cca7079f318340212274e675c0d93d5" ORG_RES="b5fd752891db927397b6ca59b20d8ecc" -- GitLab From e0e2f234faa8e8984ef0baa01d89d290d67d3f8e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 11:13:44 +0200 Subject: [PATCH 17/22] lockfile --- tests/apps/generator.cpp | 42 ++++++++++++++++++++++++++++++++++++++++ tests/apps/lockfile.sh | 40 +++++++++++++++++++------------------- 2 files changed, 62 insertions(+), 20 deletions(-) create mode 100644 tests/apps/generator.cpp diff --git a/tests/apps/generator.cpp b/tests/apps/generator.cpp new file mode 100644 index 000000000..d529c579e --- /dev/null +++ b/tests/apps/generator.cpp @@ -0,0 +1,42 @@ +#include +#include +#include +#include +#include +#include + +int +main(int argc, char* argv[]) { + if(argc < 2) { + std::cerr << "Usage: " << argv[0] << " \n"; + return 1; + } + + std::ofstream outfile(argv[1]); + if(!outfile.is_open()) { + std::cerr << "Error opening file: " << argv[1] << "\n"; + return 1; + } + + for(int i = 0; i < 10; ++i) { + // Get current time + auto now = std::chrono::system_clock::now(); + std::time_t now_time = std::chrono::system_clock::to_time_t(now); + std::tm now_tm = *std::localtime(&now_time); + + // Format time + char time_str[80]; + std::strftime(time_str, sizeof(time_str), "%H:%M:%S", &now_tm); + + // Write to file + outfile << "Entry " << i + 1 << " at " << time_str << std::endl; + outfile.flush(); // Ensure immediate write + + // Wait 1 second + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + + outfile.close(); + std::cout << "Finished writing to " << argv[1] << "\n"; + return 0; +} diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index f047aa4a0..15c0b82ba 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -14,8 +14,7 @@ export ROOT=$IO/tests/app/lockfile/root export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_lockfile.txt export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE -export APP=/root/S3D-IO/s3d_io.x -export PARAMS="30 30 30 1 1 1 0 F" +export APP=./generator rm -rf $MNT rm -rf $ROOT @@ -35,40 +34,41 @@ if ! ps -p $DAEMON_PID > /dev/null; then exit 1 fi - -# Run s3d-io.x with LD_PRELOAD and capture exit code -echo "Running lockfile test..." +echo "compiling generator..." +g++ generator.cpp -o generator # If we want to use mpi we may need : OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 mpirun -np 2 -LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc $OUTSIDE +LD_PRELOAD=$GKFS mkdir $MNT/syscall mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfile/ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfile/client.txt export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LD_PRELOAD=$GKFS_LIBC $APP $PARAMS $MNT/libc & -S3D=$! -# loop 10 times and do "hexdump pressure_wave_test.0.000E+00.field.nc" -for i in {1..10}; do - sleep 1 - LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC md5sum $MNT/libc/pressure_wave_test.0.000E+00.field.nc - # If S3D does not exist break - if ! ps -p $S3D > /dev/null; then - break - fi -done +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & +GENERATOR=$! + +sleep 5 + +OUTPUT=`LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC wc -l $MNT/syscall/filex` -wait $S3D +wait $GENERATOR export LIBGKFS_LOG=none export LD_PRELOAD=$GKFS -rm -rf $MNT/libc +rm -rf $MNT/syscall unset LD_PRELOAD - # Stop the daemon kill $DAEMON_PID +if [ "$OUTPUT" == "10 /builds/gitlab/hpc/gekkofs/gkfs/install/tests/app/lockfile/mnt/syscall/filex" ]; then + echo "Test passed!" + exit 0 +else + echo "Test failed! Output: $OUTPUT" + exit 1 +fi + exit 0 \ No newline at end of file -- GitLab From 051b21907891f2405c2b44b48f2dfba1bb1c2d21 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 12:29:34 +0200 Subject: [PATCH 18/22] generator.cpp path --- tests/apps/lockfile.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index 15c0b82ba..fc7c4cd18 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -35,7 +35,7 @@ if ! ps -p $DAEMON_PID > /dev/null; then fi echo "compiling generator..." -g++ generator.cpp -o generator +g++ /builds/gitlab/hpc/gekkofs/tests/apps/generator.cpp -o generator # If we want to use mpi we may need : OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 mpirun -np 2 LD_PRELOAD=$GKFS mkdir $MNT/syscall -- GitLab From 3632a7c9aa13abf23d4005014c058d175cfd3939 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 13:46:44 +0200 Subject: [PATCH 19/22] more coverage --- src/client/gkfs_libc.cpp | 11 ++++++++--- tests/apps/lockfile.sh | 4 +++- .../integration/harness/gkfs.io/syscall_coverage.cpp | 4 ++++ tests/integration/syscalls/test_error_operations.py | 11 +++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index ace26ed43..661834f83 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -661,15 +661,14 @@ open64(const char* path, int flags, ...) { GKFS_FALLBACK(open64, const_cast(path), flags, mode) } +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS int open64(const char* path, int flags, mode_t mode) { - - initializeGekko(); GKFS_PATH_OPERATION(open, AT_FDCWD, path, mode, flags) GKFS_FALLBACK(open64, const_cast(path), flags, mode) } - +#endif int openat(int dirfd, const char* path, int flags, ...) { @@ -1112,12 +1111,14 @@ __fxstatat64(int ver, int dfd, const char* path, struct stat64* buf, } #endif +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS int stat(int ver, const char* path, struct stat* buf) { initializeGekko(); GKFS_PATH_OPERATION(stat, AT_FDCWD, path, buf) GKFS_FALLBACK(stat, ver, path, buf); } +#endif int stat(const char* path, struct stat* buf) { @@ -1258,6 +1259,7 @@ vfork(void) { } */ +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS int pipe(int pipefd[2]) { initializeGekko(); @@ -1268,6 +1270,7 @@ pipe(int pipefd[2]) { // GKFS_FALLBACK(pipe, pipefd); return res; } +#endif int dup(int fd) { @@ -1303,11 +1306,13 @@ dup3(int fd, int fd2, int flags) { GKFS_FALLBACK(dup3, fd, fd2, flags); } +#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS void exit(int status) { dlsym_exit(status); __builtin_unreachable(); } +#endif // Manager API diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index fc7c4cd18..f280632bb 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -45,8 +45,10 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfi export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & GENERATOR=$! +sleep 1 +OUTPUTLS=`LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC ls $MNT/syscall/` sleep 5 diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index c95e882ae..b21d801a3 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -594,6 +594,10 @@ test_directory_ops_uncovered(const std::string& base_dir) { entry64 = readdir64(dirp); // Try reading again after seek assert(entry64 != nullptr); + rewinddir(dirp); + entry64 = readdir64(dirp); // Try reading again after seek + assert(entry64 != nullptr); + closedir(dirp); } diff --git a/tests/integration/syscalls/test_error_operations.py b/tests/integration/syscalls/test_error_operations.py index 8e310d121..51eca4a2a 100644 --- a/tests/integration/syscalls/test_error_operations.py +++ b/tests/integration/syscalls/test_error_operations.py @@ -179,3 +179,14 @@ def test_dup(gkfs_daemon, gkfs_client): assert ret.retval == 0 assert ret.errno == 0 +def test_create_error(gkfs_daemon, gkfs_client): + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, 99) + assert ret.retval != -1 + assert ret.errno == errno.EINVAL + + # S_IFSOCK + ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, 0o140000) + assert ret.retval != -1 + assert ret.errno == errno.ENOTSUP \ No newline at end of file -- GitLab From 87af4b0b416386ba6bf6cd90251d1f7924581524 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 14:29:32 +0200 Subject: [PATCH 20/22] additional --- tests/integration/syscalls/test_error_operations.py | 6 +----- tests/integration/syscalls/test_malleability.py | 6 +++--- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/integration/syscalls/test_error_operations.py b/tests/integration/syscalls/test_error_operations.py index 51eca4a2a..d87fb1c61 100644 --- a/tests/integration/syscalls/test_error_operations.py +++ b/tests/integration/syscalls/test_error_operations.py @@ -182,11 +182,7 @@ def test_dup(gkfs_daemon, gkfs_client): def test_create_error(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" - ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, 99) - assert ret.retval != -1 - assert ret.errno == errno.EINVAL - # S_IFSOCK ret = gkfs_client.open(file, os.O_CREAT | os.O_WRONLY, 0o140000) - assert ret.retval != -1 + assert ret.retval == -1 assert ret.errno == errno.ENOTSUP \ No newline at end of file diff --git a/tests/integration/syscalls/test_malleability.py b/tests/integration/syscalls/test_malleability.py index 12eab555b..e9ea2454c 100644 --- a/tests/integration/syscalls/test_malleability.py +++ b/tests/integration/syscalls/test_malleability.py @@ -45,7 +45,7 @@ def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): # Add "#FS_INSTANCE_END" in the file with name d00.hostfile - time.sleep(10) + time.sleep(5) with open(d00.hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") @@ -66,14 +66,14 @@ def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): # Create content d01 = gkfwd_daemon_factory.create() - time.sleep(10) + time.sleep(5) cmd = gkfs_shell.gkfs_malleability('expand','status') assert cmd.exit_code == 0 assert cmd.stdout.decode() == "No expansion running/finished.\n" cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) - time.sleep(20) + time.sleep(10) cmd = gkfs_shell.gkfs_malleability('expand','finalize') -- GitLab From f60f3aadcbde69e311af9465c18234453e21ac63 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 15:07:02 +0200 Subject: [PATCH 21/22] bump fmt/spdlog --- external/fmt | 2 +- external/spdlog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/fmt b/external/fmt index e69e5f977..40626af88 160000 --- a/external/fmt +++ b/external/fmt @@ -1 +1 @@ -Subproject commit e69e5f977d458f2650bb346dadf2ad30c5320281 +Subproject commit 40626af88bd7df9a5fb80be7b25ac85b122d6c21 diff --git a/external/spdlog b/external/spdlog index eb3220622..6fa36017c 160000 --- a/external/spdlog +++ b/external/spdlog @@ -1 +1 @@ -Subproject commit eb3220622e73a4889eee355ffa37972b3cac3df5 +Subproject commit 6fa36017cfd5731d617e1a934f0e5ea9c4445b13 -- GitLab From 8b79664ee2d7501ba9b8722e881121c67bce37de Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 9 May 2025 15:27:49 +0200 Subject: [PATCH 22/22] Update spdlog/fmt --- CMake/load_nlohmann_json.cmake | 8 +------- CMakeLists.txt | 4 ++-- include/client/syscalls/args.hpp | 1 + include/daemon/daemon.hpp | 15 +++++++++++++++ tests/integration/harness/gkfs.io/readdir.cpp | 1 + tests/unit/CMakeLists.txt | 10 +--------- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/CMake/load_nlohmann_json.cmake b/CMake/load_nlohmann_json.cmake index 34a5f4633..cb059db79 100644 --- a/CMake/load_nlohmann_json.cmake +++ b/CMake/load_nlohmann_json.cmake @@ -34,12 +34,8 @@ FetchContent_Declare(nlohmann_json DOWNLOAD_EXTRACT_TIMESTAMP ON URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) -FetchContent_GetProperties(nlohmann_json) -if (NOT nlohmann_json_POPULATED) - FetchContent_Populate(nlohmann_json) - message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}") - message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}") +FetchContent_MakeAvailable(nlohmann_json) # we don't really care so much about a third party library's tests to be # run from our own project's code @@ -48,5 +44,3 @@ if (NOT nlohmann_json_POPULATED) # we also don't need to install it when our main project gets installed set(JSON_Install OFF CACHE INTERNAL "") - add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR}) -endif () diff --git a/CMakeLists.txt b/CMakeLists.txt index 9540bd79f..98fc0776b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -229,7 +229,7 @@ include_from_source(fmt MESSAGE "[${PROJECT_NAME}] Searching for {fmt}" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG e69e5f977d458f2650bb346dadf2ad30c5320281 # v10.1.0 + GIT_TAG 40626af88bd7df9a5fb80be7b25ac85b122d6c21 # v11.2.0 ) # ensure that fmt is linked as PIC @@ -240,7 +240,7 @@ include_from_source(spdlog MESSAGE "[${PROJECT_NAME}] Searching for spdlog" SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/spdlog GIT_REPOSITORY https://github.com/gabime/spdlog.git - GIT_TAG eb3220622e73a4889eee355ffa37972b3cac3df5 # v1.9.2 + GIT_TAG 6fa36017cfd5731d617e1a934f0e5ea9c4445b13 # v1.15.3 ) ### CLI11: used for parsing command-line options diff --git a/include/client/syscalls/args.hpp b/include/client/syscalls/args.hpp index d53a74d0d..ea391ba97 100644 --- a/include/client/syscalls/args.hpp +++ b/include/client/syscalls/args.hpp @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include diff --git a/include/daemon/daemon.hpp b/include/daemon/daemon.hpp index 82f8286e1..dfdc0998f 100644 --- a/include/daemon/daemon.hpp +++ b/include/daemon/daemon.hpp @@ -55,6 +55,21 @@ extern "C" { #include } +#include // Must include before any spdlog headers + +// If hg_return is your custom type +namespace fmt { +template <> +struct formatter : formatter { + auto + format(const hg_return& value, format_context& ctx) const { + // Implement your formatting logic + return formatter::format( + fmt::format("hg_return({})", static_cast(value)), ctx); + } +}; +} // namespace fmt + #include #include #include diff --git a/tests/integration/harness/gkfs.io/readdir.cpp b/tests/integration/harness/gkfs.io/readdir.cpp index f05834009..ec41b92e9 100644 --- a/tests/integration/harness/gkfs.io/readdir.cpp +++ b/tests/integration/harness/gkfs.io/readdir.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 72e2d63d2..004671439 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -37,15 +37,7 @@ FetchContent_Declare(catch2 GIT_PROGRESS ON ) -FetchContent_GetProperties(catch2) - -if (NOT catch2_POPULATED) - FetchContent_Populate(catch2) - message(STATUS "[gkfs] Catch2 source dir: ${catch2_SOURCE_DIR}") - message(STATUS "[gkfs] Catch2 binary dir: ${catch2_BINARY_DIR}") - set(CATCH_BUILD_TESTING OFF CACHE INTERNAL "") - add_subdirectory(${catch2_SOURCE_DIR} ${catch2_BINARY_DIR}) -endif () +FetchContent_MakeAvailable(catch2) add_subdirectory(helpers) -- GitLab