diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 268d055b085b59f215564f5b89d144da65f847ef..cd389c722195cf939bee985e26df2de82caf50f2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -345,6 +345,88 @@ gkfs:app: junit: ${BUILD_PATH}/tests/apps/report.xml +## == java tests for gkfs ================== +gkfs:java: + stage: test + image: gekkofs/java:0.9.5 + needs: ['gkfs'] + script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin + ## run actual tests + - cd ${CI_PROJECT_DIR} + - ctest --test-dir ${BUILD_PATH} + -j 1 + -L java::all + -VV + --output-junit ${BUILD_PATH}/tests/java/report.xml + + ## capture coverage information for this test and write it to + ## $COVERAGE_PATH/java.info + - cd ${CI_PROJECT_DIR} + # use ccache + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/java.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external + ## Since the pipeline recreates the source tree, the access times for .gcno + ## files are newer than those of .gcda files. This makes gcov emit a + ## warning for each file which slows it down. Updating the timestamps + ## avoids this + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + artifacts: + expire_in: 1 day + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/java/report.xml + + +## == python tests for gkfs ================== +gkfs:python: + stage: test + image: gekkofs/testing:0.9.5 + needs: ['gkfs'] + script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin + ## run actual tests + - cd ${CI_PROJECT_DIR} + - ctest --test-dir ${BUILD_PATH} + -j 1 + -L python::all + -VV + --output-junit ${BUILD_PATH}/tests/python/report.xml + + ## capture coverage information for this test and write it to + ## $COVERAGE_PATH/java.info + - cd ${CI_PROJECT_DIR} + # use ccache + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/python.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external + ## Since the pipeline recreates the source tree, the access times for .gcno + ## files are newer than those of .gcda files. This makes gcov emit a + ## warning for each file which slows it down. Updating the timestamps + ## avoids this + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + artifacts: + expire_in: 1 day + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/python/report.xml + + ################################################################################ ## Generation of documentation ################################################################################ @@ -416,7 +498,7 @@ coverage: stage: report image: gekkofs/testing:0.9.5 #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app'] + needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] script: # use ccache - ccache --zero-stats diff --git a/CHANGELOG.md b/CHANGELOG.md index 40ab7979078334add743b4179795c14726e4eacb..5dcbf5d9efe6eea851932a8277de6333f507df67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Use PROTECT_FILES_GENERATOR=1 and PROTECT_FILES_CONSUMER=1 to enable. Generator, creates transparent .lockgekko files that blocks the open (for some seconds) of any consumer. Multiple opens / closes for generator are managed. - Basic mmap support ([!247](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/245)) - LIBC interception support, with extra tests ([!202](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/202)) - + - Fixes for vfork, java and python ([!257](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/257)) + - Adds two new testing stages, java and python. + ### Changed - Tests check ret for -1 instead of 10000 fd ([!320](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/320)) - Allow some more script tests to run as pthread_at_fork solved some issues. @@ -41,6 +43,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Java with syscalls deadlocks as it try to resolve paths (malloc) in a locking situation ([!255](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/255)) - It also solves flock missing implementation when we are ouside gekkofs - Some features in syscall_intercept still hangs if we do not lower the debug information. + - Fixes for vfork, java and python ([!257](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/257)) + - execve and chdir failed as calling the server for get_metadata fails, disabled chdir checks (GekkoFS) + - execve set reentrant_guard as true, so once it returns (vfork) the parent had syscall disabled. + - debug should be still set to debug level on syscall for vfork. + - In MN5 gkfs_libc.hpp needs to have the libc signatures. + ## [0.9.4] - 2025-03 ### New diff --git a/docker/0.9.5/java/Dockerfile b/docker/0.9.5/java/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..bf113545693efc8b1d27c1384d49dc54945b8a41 --- /dev/null +++ b/docker/0.9.5/java/Dockerfile @@ -0,0 +1,10 @@ +FROM gekkofs/testing:0.9.5 + +LABEL Description="Debian-based environment to run java" +ARG DEBIAN_FRONTEND=noninteractive + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + openjdk-25-jdk-headless \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean \ No newline at end of file diff --git a/docker/0.9.5/java/Makefile b/docker/0.9.5/java/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..f98c9b92e4d33d8c8970b07fe8882b1451093d03 --- /dev/null +++ b/docker/0.9.5/java/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/java:0.9.5 . + +aarch64: + docker build --platform aarch64 -t gekkofs/java:0.9.5 . + +all: + docker build -t gekkofs/java:0.9.5 . diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index ca04d0bd2c1b4c444be15763ca8e83395b1d2bc4..0c2721a11bced11c8d2b4bfbe80d3630872240d2 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -605,11 +605,6 @@ DLSYM_WRAPPER(int, dup2, (int fd, int fd2), (fd, fd2), "dup2") DLSYM_WRAPPER(int, dup3, (int fd, int fd2, int flags), (fd, fd2, flags), "dup3") static int (*real_fcntl)(int fd, int cmd, ...) = nullptr; // Special handling for variadic fcntl -DLSYM_WRAPPER(void, exit, (int status), (status), "exit") -#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS -DLSYM_WRAPPER(int, pipe, (int pipefd[2]), (pipefd), "pipe") -#endif - // Memory Mapping DLSYM_WRAPPER(void*, mmap, (void* addr, size_t length, int prot, int flags, int fd, @@ -2437,29 +2432,6 @@ fcntl(int fd, int cmd, ...) { } } -#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS -void -exit(int status) { - // GekkoFS cleanup might be needed before exit (e.g., flushing, - // releasing resources) This should ideally be in CTX destructor or a - // dedicated shutdown function. - DEBUG_INFO("[GKFS] exit(status={})", status); - // gkfs::PreloadContext::getInstance()->shutdown(); // Example - dlsym_exit(status); - __builtin_unreachable(); // To tell compiler exit doesn't return -} - -int -pipe(int pipefd[2]) { - gkfs_init_routine_placeholder(); - // Pipes are usually not intercepted by file system shims unless they - // are named pipes (FIFOs) and GekkoFS implements them. Standard - // anonymous pipes are kernel objects. - DEBUG_INFO("[BYPASS] pipe()"); - return dlsym_pipe(pipefd); -} -#endif - //------------------------- Memory Mapping //-----------------------------------// diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 4aae63a80273879b632dcffb69699651ffb9d911..86c75781b952d6701f8012a64d9c9d6dbdf6fd5a 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -717,7 +717,8 @@ hook_chdir(const char* path) { bool internal = CTX->relativize_path(path, rel_path); if(internal) { // path falls in our namespace - auto md = gkfs::utils::get_metadata(rel_path); + /* This get_metadata gets stuck after a forkandExec */ + /*auto md = gkfs::utils::get_metadata(rel_path); if(!md) { LOG(ERROR, "{}() path {} / {} errno {}", __func__, path, rel_path, errno); @@ -727,7 +728,7 @@ hook_chdir(const char* path) { if(!S_ISDIR(md->mode())) { LOG(ERROR, "{}() path is not a directory", __func__); return -ENOTDIR; - } + }*/ // TODO get complete path from relativize_path instead of // removing mountdir and then adding again here rel_path.insert(0, CTX->mountdir()); diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index deae776baed9331773fa2af96181635dcb46958d..1ae6e986035496477bae537954a74141a8701978 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -532,6 +532,8 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, switch(syscall_number) { case SYS_execve: + // If we do not set this to false, we are in trouble with vforks + reentrance_guard_flag = false; *result = syscall_no_intercept_wrapper( syscall_number, reinterpret_cast(arg0), reinterpret_cast(arg1), @@ -540,6 +542,8 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, #ifdef SYS_execveat case SYS_execveat: + // If we do not set this to false, we are in trouble with vforks + reentrance_guard_flag = false; *result = syscall_no_intercept_wrapper( syscall_number, arg0, reinterpret_cast(arg1), reinterpret_cast(arg2), diff --git a/src/client/syscalls/detail/syscall_info.c b/src/client/syscalls/detail/syscall_info.c index c4c9ba2593ce26eb0374a435394b777ceeeb15d8..0e463e84a83f2817bcc60e30dc89de88c1a3917b 100644 --- a/src/client/syscalls/detail/syscall_info.c +++ b/src/client/syscalls/detail/syscall_info.c @@ -642,7 +642,7 @@ SYSCALL(getpmsg, 5, S_RET(rdec), S_NARG(arg, "arg0"), SYSCALL(pidfd_open, 2, S_RET(rdec), S_NARG(dec, "pid"), S_NARG(arg, "flags")), #endif #ifdef SYS_clone3 - SYSCALL(clone3, 4, S_RET(rdec), S_NARG(clone3_args, "flags"), S_NARG(arg, "size")), + SYSCALL(clone3, 2, S_RET(rdec), S_NARG(clone3_args, "flags"), S_NARG(arg, "size")), #endif #ifdef SYS_close_range SYSCALL(close_range, 3, S_RET(rdec), S_NARG(dec, "low"), S_NARG(dec, "high"), S_NARG(arg, "flags")), diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 722764c7e62a4028e348ddcfb02f7ed185302672..2db460ca2e5144852f3fbe4a9bfccaf68079b4e7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,4 +49,8 @@ add_subdirectory(integration) # unit tests add_subdirectory(unit) +# java tests +add_subdirectory(java) +# java tests +add_subdirectory(python) \ No newline at end of file diff --git a/tests/apps/CMakeLists.txt b/tests/apps/CMakeLists.txt index e79fd67bdd439f472a2f3a83075544f8f1de0aad..83fefaeaf4184ec162109529d190c0acd1b85c8c 100644 --- a/tests/apps/CMakeLists.txt +++ b/tests/apps/CMakeLists.txt @@ -66,8 +66,8 @@ gekko_add_test(lockfile lockfile.sh) # --- Installation of Test Scripts --- if(GKFS_INSTALL_TESTS) install( - DIRECTORY . # Installs files from the current source directory - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/apps + DIRECTORY apps # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE FILES_MATCHING PATTERN "*.sh" # Only install .sh files diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index b21d801a3376b5a913fe2e340ed94d4ab5296b17..67c21b81e8ae74c5900ee69261fdbfde139c87b7 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -1554,21 +1554,21 @@ syscall_coverage_exec(const syscall_coverage_options& opts) { output("readlinkat", rv, opts); return; } - + // This won't fail as intented // chdir internal error - rv = ::chdir(opts.pathname.c_str()); - if(errno != ENOTDIR) { - output("chdir", rv, opts); - return; - } + // rv = ::chdir(opts.pathname.c_str()); + // if(errno != ENOTDIR) { + // output("chdir", rv, opts); + // return; + //} // chdir internal error - std::string nonexist = opts.pathname + "x2"; - rv = ::chdir(nonexist.c_str()); - if(rv >= 0) { - output("chdir", rv, opts); - return; - } + // std::string nonexist = opts.pathname + "x2"; + // rv = ::chdir(nonexist.c_str()); + // if(rv >= 0) { + // output("chdir", rv, opts); + // return; + //} // fchdir auto fddir = ::open(".", O_RDONLY); diff --git a/tests/integration/status/test_status.py b/tests/integration/status/test_status.py index e3b9809144e3d13239b15537ac34389f31c0b4a8..f6cc3d752546b84ce28ed6bbd5b938ef7a8c65bb 100644 --- a/tests/integration/status/test_status.py +++ b/tests/integration/status/test_status.py @@ -43,7 +43,7 @@ nonexisting = "nonexisting" -#@pytest.mark.xfail(reason="invalid errno returned on success") +@pytest.mark.xfail(reason="invalid errno returned on success") def test_statx(gkfs_daemon, gkfs_client): """Test several statx commands""" topdir = gkfs_daemon.mountdir / "top" diff --git a/tests/java/CMakeLists.txt b/tests/java/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d888f04e7a6c20021a6290722a3ff606cdc91a22 --- /dev/null +++ b/tests/java/CMakeLists.txt @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# --- Common Test Properties --- +set(DEFAULT_TEST_TIMEOUT 60) +set(DEFAULT_TEST_LABELS "java::all") +set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# --- Helper Function to Add Tests with Common Properties --- +# This function simplifies adding new tests that share common properties. +# Usage: gekko_add_test( [EXTRA_PROPERTIES ...]) +function(gekko_add_test TEST_NAME SCRIPT_FILENAME) + add_test( + NAME ${TEST_NAME} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT_FILENAME} + WORKING_DIRECTORY ${DEFAULT_TEST_WORKING_DIR} + ) + + set_tests_properties(${TEST_NAME} + PROPERTIES + TIMEOUT ${DEFAULT_TEST_TIMEOUT} + LABELS ${DEFAULT_TEST_LABELS} + ) +endfunction() + + + +gekko_add_test(java_fork java_fork.sh) +gekko_add_test(java_vfork java_vfork.sh) + + + +# --- Installation of Test Scripts --- +if(GKFS_INSTALL_TESTS) + install( + DIRECTORY java # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + FILES_MATCHING + PATTERN "*.sh" # Only install .sh files + + ) +endif() \ No newline at end of file diff --git a/tests/java/DirectoryOperations.java b/tests/java/DirectoryOperations.java new file mode 100644 index 0000000000000000000000000000000000000000..6014fccf6941b3c6330330ffbd085b48f914de8e --- /dev/null +++ b/tests/java/DirectoryOperations.java @@ -0,0 +1,262 @@ +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +public +class DirectoryOperations { + +public + static void + main(String[] args) { + if(args.length == 0) { + System.err.println( + "Usage: java DirectoryOperations "); + System.err.println( + "If the directory doesn't exist, it will be created (if possible)."); + System.exit(1); + } + + String dirPathString = args[0]; + Path directoryPath = Paths.get(dirPathString); + Path testFilePath = directoryPath.resolve( + "myTestFile.txt"); // File inside the target directory + + System.out.println( + "--- Starting I/O and Process Operations on Directory: " + + directoryPath.toAbsolutePath() + " ---\n"); + + try { + // 0. Ensure directory exists (or create it) + if(!Files.exists(directoryPath)) { + System.out.println( + "[INFO] Directory does not exist. Attempting to create: " + + directoryPath); + Files.createDirectories( + directoryPath); // Creates parent dirs if necessary + System.out.println("[SUCCESS] Directory created: " + + directoryPath.toAbsolutePath()); + } else if(!Files.isDirectory(directoryPath)) { + System.err.println( + "[ERROR] Path exists but is not a directory: " + + directoryPath.toAbsolutePath()); + System.exit(1); + } else { + System.out.println("[INFO] Directory already exists: " + + directoryPath.toAbsolutePath()); + } + + System.out.println("\n--- File Operations on: " + + testFilePath.toAbsolutePath() + " ---"); + + // 1. Create a file + System.out.println("\n1. CREATE Operation:"); + if(Files.exists(testFilePath)) { + System.out.println( + " [WARN] Test file already exists. Deleting it first."); + Files.delete(testFilePath); + } + Files.createFile(testFilePath); + System.out.println(" [SUCCESS] Created file: " + testFilePath); + + // 2. Write to the file + System.out.println("\n2. WRITE Operation:"); + String contentToWrite = + "Hello from Java I/O!\nLine 2: This is a test.\nLine 3: End of content."; + // Using try-with-resources for BufferedWriter + try(BufferedWriter writer = Files.newBufferedWriter( + testFilePath, StandardCharsets.UTF_8, + StandardOpenOption.WRITE)) { + writer.write(contentToWrite); + System.out.println( + " [SUCCESS] Wrote content to file:\n---\n" + + contentToWrite + "\n---"); + } + + // 3. Read from the file + System.out.println("\n3. READ Operation:"); + // Using Files.readString for simplicity (Java 11+) + // For older Java, use BufferedReader with Files.newBufferedReader + String readContent = + Files.readString(testFilePath, StandardCharsets.UTF_8); + System.out.println(" [SUCCESS] Read content from file:\n---\n" + + readContent + "\n---"); + if(!contentToWrite.equals(readContent)) { + System.out.println( + " [WARN] Read content does not exactly match written content (this might be due to line endings if not handled carefully)."); + } + + // 4. Stat (get file attributes) + System.out.println("\n4. STAT (Attributes) Operation:"); + BasicFileAttributes attrs = Files.readAttributes( + testFilePath, BasicFileAttributes.class); + System.out.println(" [INFO] File Size: " + attrs.size() + + " bytes"); + System.out.println(" [INFO] Is Directory: " + + attrs.isDirectory()); + System.out.println(" [INFO] Is Regular File: " + + attrs.isRegularFile()); + System.out.println(" [INFO] Is Symbolic Link: " + + attrs.isSymbolicLink()); + System.out.println(" [INFO] Last Access Time: " + + attrs.lastAccessTime()); + System.out.println(" [INFO] Last Modified Time: " + + attrs.lastModifiedTime()); + System.out.println(" [INFO] Creation Time: " + + attrs.creationTime()); + + // 5. Fork and Exec (ls or dir) + System.out.println( + "\n\n--- Fork and Exec (Listing directory contents) ---"); + executeExternalProcess(directoryPath); + + getLsCpuOutput(); + + } catch(IOException e) { + System.err.println("\n[ERROR] An I/O error occurred: " + + e.getMessage()); + e.printStackTrace(); + } catch(InterruptedException e) { + System.err.println("\n[ERROR] Process execution was interrupted: " + + e.getMessage()); + Thread.currentThread().interrupt(); // Restore interrupted status + e.printStackTrace(); + } finally { + // 6. Remove the file (cleanup) + System.out.println("\n\n--- Cleanup ---"); + System.out.println("\n6. REMOVE Operation (Cleanup):"); + try { + if(Files.exists(testFilePath)) { + Files.delete(testFilePath); + System.out.println(" [SUCCESS] Removed test file: " + + testFilePath); + } else { + System.out.println( + " [INFO] Test file was already removed or never created properly."); + } + } catch(IOException e) { + System.err.println(" [ERROR] Failed to remove test file " + + testFilePath + ": " + e.getMessage()); + } + // Optional: remove the directory if it was created by this script + // and is empty For this example, we'll leave the directory. + } + System.out.println("\n--- Operations Complete ---"); + } + +private + static void + executeExternalProcess(Path directoryToInspect) throws IOException, + InterruptedException { + List command = new ArrayList<>(); + command.add("ls"); + command.add("-la"); + command.add(directoryToInspect.toAbsolutePath() + .toString()); // `ls -la /path/to/dir` + + System.out.println(" Executing command: " + + String.join(" ", command)); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + // processBuilder.directory(directoryToInspect.toFile()); // + // Alternative: set working directory + + Process process = processBuilder.start(); + + // Capture standard output + System.out.println("\n --- Process Standard Output ---"); + try(BufferedReader stdInput = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String s; + while((s = stdInput.readLine()) != null) { + System.out.println(" " + s); + } + } + + // Capture standard error (if any) + boolean hasErrorOutput = false; + StringBuilder errorOutput = new StringBuilder(); + try(BufferedReader stdError = new BufferedReader( + new InputStreamReader(process.getErrorStream()))) { + String s; + while((s = stdError.readLine()) != null) { + if(!hasErrorOutput) { + System.out.println("\n --- Process Standard Error ---"); + hasErrorOutput = true; + } + errorOutput.append(" ").append(s).append("\n"); + } + } + if(hasErrorOutput) + System.out.print(errorOutput.toString()); + + + int exitCode = process.waitFor(); + System.out.println("\n [INFO] Process exited with code: " + exitCode); + + if(exitCode != 0 && !hasErrorOutput) { + System.out.println( + " [WARN] Process exited with non-zero status but no error output captured. Check command."); + } else if(exitCode == 0 && hasErrorOutput) { + System.out.println( + " [WARN] Process exited successfully but produced error output. This might be informational."); + } + } + + +private + static String + getLsCpuOutput() { + // ************************************************************************************** + // Get LSCPU output + // ************************************************************************************** + String cmdOutput = null; + List command = new ArrayList<>(); + command.add("lscpu"); + + ProcessBuilder pb = new ProcessBuilder(command); + try { + Process process = pb.start(); + + // Disable inputs to process + process.getOutputStream().close(); + // Wait and retrieve exit value + int exitValue = process.waitFor(); + if(exitValue != 0) { + System.out.println( + "ERROR: LSCPU command failed with exitValue " + + exitValue); + } + + // Retrieve command output + StringBuilder sb = new StringBuilder(); + try(BufferedReader br = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String line = null; + while((line = br.readLine()) != null) { + sb.append(line).append("\n"); + } + } + catch(IOException ioe) { + System.out.println( + "ERROR: Cannot retrieve LSCPU command output"); + } + cmdOutput = sb.toString(); + } catch(IOException ioe) { + System.out.println("ERROR: Cannot start LSCPU ProcessBuilder"); + } catch(InterruptedException ie) { + System.out.println("ERROR: LSCPU command interrupted"); + } + + return cmdOutput; + } +} diff --git a/tests/java/java_fork.sh b/tests/java/java_fork.sh new file mode 100755 index 0000000000000000000000000000000000000000..8ff12c85a5426aee7627a6dc47b871ac84417edb --- /dev/null +++ b/tests/java/java_fork.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/java/java_fork/original +export MNT=$IO/tests/java/java_fork/mnt +export ROOT=$IO/tests/java/java_fork/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=$IO/lib/:$LD_LIBRARY_PATH + +export APPFORK="java -Djdk.lang.Process.launchMechanism=fork DirectoryOperations" +export APPVFORK="java -Djdk.lang.Process.launchMechanism=vfork DirectoryOperations" + +REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' +REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' + + +preprocess_text() { + local text_input="$1" + local processed_text + + # 1. Remove newlines, then remove spaces + # Using Bash parameter expansion: + processed_text="${text_input//$'\n'/}" # Remove newlines + processed_text="${processed_text// /}" # Remove all spaces + + # Alternatively using tr (can be slower for large strings already in vars): + # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') + + # 2. Normalize dates on the compacted string + # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") + # The ISO date regex is unaffected as it doesn't contain spaces. + processed_text=$(echo "$processed_text" | \ + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ + sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' + ) + echo "$processed_text" +} + + +NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") +NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") + + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +echo "compiling java app..." +cd /builds/gitlab/hpc/gekkofs/tests/java/ +javac DirectoryOperations.java +echo "compiled java app..." + +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=none +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPFORK $MNT/syscall/JAVA` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_libc.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPFORK $MNT/libc/JAVA` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + + +NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") +NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") + +if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then + echo "SUCCESS: The SYSCALL (FORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The SYSCALL (FORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") +fi + +if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then + echo "SUCCESS: The LIBC (FORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The LIBC (FORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") +fi + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (FORK) command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (FORK) command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file diff --git a/tests/java/java_vfork.sh b/tests/java/java_vfork.sh new file mode 100755 index 0000000000000000000000000000000000000000..7f21a0d0fa21dd9e9233395b4ff7a9f8af638328 --- /dev/null +++ b/tests/java/java_vfork.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/java/java_vfork/original +export MNT=$IO/tests/java/java_vfork/mnt +export ROOT=$IO/tests/java/java_vfork/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=/root/wacommplusplus/build/external/lib:$IO/lib/:$LD_LIBRARY_PATH + +export APPFORK="java -Djdk.lang.Process.launchMechanism=fork DirectoryOperations" +export APPVFORK="java -Djdk.lang.Process.launchMechanism=vfork DirectoryOperations" + +REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' +REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' + + +preprocess_text() { + local text_input="$1" + local processed_text + + # 1. Remove newlines, then remove spaces + # Using Bash parameter expansion: + processed_text="${text_input//$'\n'/}" # Remove newlines + processed_text="${processed_text// /}" # Remove all spaces + + # Alternatively using tr (can be slower for large strings already in vars): + # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') + + # 2. Normalize dates on the compacted string + # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") + # The ISO date regex is unaffected as it doesn't contain spaces. + processed_text=$(echo "$processed_text" | \ + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ + sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' + ) + echo "$processed_text" +} + + +NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") +NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") + + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +echo "compiling java app..." +cd /builds/gitlab/hpc/gekkofs/tests/java/ +javac DirectoryOperations.java +echo "compiled java app..." + +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=none +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPVFORK $MNT/syscall/JAVA` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_libc.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPVFORK $MNT/libc/JAVA` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + + +NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") +NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") + +if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then + echo "SUCCESS: The SYSCALL (VFORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The SYSCALL (VFORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") +fi + +if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then + echo "SUCCESS: The LIBC (VFORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The LIBC (VFORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") +fi + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (VFORK) command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (VFORK) command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..ddb1cf4ace38fa8edf0dfb31afa35074dfb5a1b1 --- /dev/null +++ b/tests/python/CMakeLists.txt @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# --- Common Test Properties --- +set(DEFAULT_TEST_TIMEOUT 60) +set(DEFAULT_TEST_LABELS "python::all") +set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# --- Helper Function to Add Tests with Common Properties --- +# This function simplifies adding new tests that share common properties. +# Usage: gekko_add_test( [EXTRA_PROPERTIES ...]) +function(gekko_add_test TEST_NAME SCRIPT_FILENAME) + add_test( + NAME ${TEST_NAME} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT_FILENAME} + WORKING_DIRECTORY ${DEFAULT_TEST_WORKING_DIR} + ) + + set_tests_properties(${TEST_NAME} + PROPERTIES + TIMEOUT ${DEFAULT_TEST_TIMEOUT} + LABELS ${DEFAULT_TEST_LABELS} + ) +endfunction() + + + +gekko_add_test(python_exists python_exists.sh) + + + + +# --- Installation of Test Scripts --- +if(GKFS_INSTALL_TESTS) + install( + DIRECTORY python # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + FILES_MATCHING + PATTERN "*.sh" # Only install .sh files + + ) +endif() \ No newline at end of file diff --git a/tests/python/python_exists.py b/tests/python/python_exists.py new file mode 100644 index 0000000000000000000000000000000000000000..e94961559dc9e4a22cfce37df932613dd8de18f1 --- /dev/null +++ b/tests/python/python_exists.py @@ -0,0 +1,83 @@ +import os +import sys +import time # Import the time module + +def main(): + # 1. Check if a command-line argument for the file path is provided + if len(sys.argv) < 2: + print(f"Usage: python {sys.argv[0]} ") + sys.exit(1) # Exit with an error code + + file_path = sys.argv[1] + file_created_by_script = False # Flag to ensure we only delete what we create + + print(f"Target file path from argv[1]: {file_path}") + + try: + # 2. Create the file (or overwrite if it exists) + # Ensure the directory exists if the path includes new directories + directory = os.path.dirname(file_path) + if directory and not os.path.exists(directory): + print(f"Creating directory: {directory}") + os.makedirs(directory, exist_ok=True) # exist_ok=True prevents error if dir already exists + + print(f"Attempting to create file: {file_path}") + with open(file_path, 'w') as f: + f.write("This is a test file.\n") + f.write("Created to demonstrate os.path.exists().\n") + print(f"File '{file_path}' created successfully.") + file_created_by_script = True # Mark that we created it + + # 3. Use os.path.exists() to check if the file exists + if os.path.exists(file_path): + print(f"SUCCESS: os.path.exists('{file_path}') returned True.") + # You can also check if it's a file specifically + if os.path.isfile(file_path): + print(f"And os.path.isfile('{file_path}') also returned True.") + else: + print(f"Warning: os.path.isfile('{file_path}') returned False (it might be a directory).") + sys.exit(1) + else: + # This case should ideally not be hit if creation was successful + print(f"FAILURE: os.path.exists('{file_path}') returned False, even after attempting creation.") + sys.exit(1) + + except IOError as e: + print(f"Error during file operation for '{file_path}': {e}") + # Check existence even after an error, as the file might still exist or not + if os.path.exists(file_path): + print(f"INFO: os.path.exists('{file_path}') returned True (file might have existed before or other issue).") + sys.exit(1) + else: + print(f"INFO: os.path.exists('{file_path}') returned False (as expected after error).") + sys.exit(1) + except Exception as e: + print(f"An unexpected error occurred: {e}") + sys.exit(1) + + finally: + # 4. Clean up: remove the file only if this script created it + if file_created_by_script and os.path.exists(file_path): + try: + print(f"Cleaning up: Removing '{file_path}'") + os.remove(file_path) + print(f"File '{file_path}' removed.") + except OSError as e: + print(f"Error removing file '{file_path}': {e}") + sys.exit(1) + elif file_path and not file_created_by_script and os.path.exists(file_path): + print(f"Note: File '{file_path}' existed but was not created by this script run, so not removing it.") + sys.exit(1) + # Cleanup directory if it was created by this script and is now empty + if directory and os.path.exists(directory) and not os.listdir(directory) and file_created_by_script: + try: + print(f"Cleaning up empty directory: '{directory}'") + os.rmdir(directory) + print(f"Directory '{directory}' removed.") + except OSError as e: + print(f"Error removing directory '{directory}': {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/python/python_exists.sh b/tests/python/python_exists.sh new file mode 100755 index 0000000000000000000000000000000000000000..8b23373427eb822595ba22a7f3443edb25f059f9 --- /dev/null +++ b/tests/python/python_exists.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +PYTHON=PYTHON_1 +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/python/${PYTHON}/original +export MNT=$IO/tests/python/${PYTHON}/mnt +export ROOT=$IO/tests/python/${PYTHON}/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_${PYTHON}.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=$IO/lib/:$LD_LIBRARY_PATH + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +APP=/builds/gitlab/hpc/gekkofs/tests/python/python_exists.py +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_syscall.txt +# LOG should be debug for python +export LIBGKFS_LOG=debug +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create +echo "Starting python test 1..." +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS python3 $APP $MNT/syscall/${PYTHON}/file1` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_libc.txt +export LIBGKFS_LOG=all +echo "Starting python test 2..." +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC python3 $APP $MNT/libc/${PYTHON}/file2` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (FORK) command exited with non-zero status code: $SYSCALL" + echo $OUTPUT_SYSCALL +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (FORK) command exited with non-zero status code: $LIBC" + echo $OUTPUT_LIBC +fi + +# Exit with a non-zero status if either test failed +if [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file