From 3ae7d1e4fd111d808170ad7f7e957ff50c5cde7e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 23 May 2025 09:15:34 +0200 Subject: [PATCH 01/12] add java testing --- .gitlab-ci.yml | 43 +++- docker/0.9.5/java/Dockerfile | 10 + docker/0.9.5/java/Makefile | 10 + tests/integration/status/test_status.py | 2 +- tests/java/CMakeLists.txt | 66 ++++++ tests/java/DirectoryOperations.java | 262 ++++++++++++++++++++++++ tests/java/java1.sh | 68 ++++++ 7 files changed, 459 insertions(+), 2 deletions(-) create mode 100644 docker/0.9.5/java/Dockerfile create mode 100644 docker/0.9.5/java/Makefile create mode 100644 tests/java/CMakeLists.txt create mode 100644 tests/java/DirectoryOperations.java create mode 100755 tests/java/java1.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 268d055b0..91ff92917 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -345,6 +345,47 @@ gkfs:app: junit: ${BUILD_PATH}/tests/apps/report.xml +## == java tests for gkfs ================== +gkfs:java: + stage: test + image: gekkofs/java:0.9.5 + needs: ['gkfs'] + script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin + ## run actual tests + - cd ${CI_PROJECT_DIR} + - ctest --test-dir ${BUILD_PATH} + -j 1 + -L java::all + -VV + --output-junit ${BUILD_PATH}/tests/java/report.xml + + ## capture coverage information for this test and write it to + ## $COVERAGE_PATH/java.info + - cd ${CI_PROJECT_DIR} + # use ccache + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/java.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external + ## Since the pipeline recreates the source tree, the access times for .gcno + ## files are newer than those of .gcda files. This makes gcov emit a + ## warning for each file which slows it down. Updating the timestamps + ## avoids this + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + artifacts: + expire_in: 1 day + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/java/report.xml + + ################################################################################ ## Generation of documentation ################################################################################ @@ -416,7 +457,7 @@ coverage: stage: report image: gekkofs/testing:0.9.5 #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app'] + needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java'] script: # use ccache - ccache --zero-stats diff --git a/docker/0.9.5/java/Dockerfile b/docker/0.9.5/java/Dockerfile new file mode 100644 index 000000000..bf1135456 --- /dev/null +++ b/docker/0.9.5/java/Dockerfile @@ -0,0 +1,10 @@ +FROM gekkofs/testing:0.9.5 + +LABEL Description="Debian-based environment to run java" +ARG DEBIAN_FRONTEND=noninteractive + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + openjdk-25-jdk-headless \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean \ No newline at end of file diff --git a/docker/0.9.5/java/Makefile b/docker/0.9.5/java/Makefile new file mode 100644 index 000000000..f98c9b92e --- /dev/null +++ b/docker/0.9.5/java/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/java:0.9.5 . + +aarch64: + docker build --platform aarch64 -t gekkofs/java:0.9.5 . + +all: + docker build -t gekkofs/java:0.9.5 . diff --git a/tests/integration/status/test_status.py b/tests/integration/status/test_status.py index e3b980914..f6cc3d752 100644 --- a/tests/integration/status/test_status.py +++ b/tests/integration/status/test_status.py @@ -43,7 +43,7 @@ nonexisting = "nonexisting" -#@pytest.mark.xfail(reason="invalid errno returned on success") +@pytest.mark.xfail(reason="invalid errno returned on success") def test_statx(gkfs_daemon, gkfs_client): """Test several statx commands""" topdir = gkfs_daemon.mountdir / "top" diff --git a/tests/java/CMakeLists.txt b/tests/java/CMakeLists.txt new file mode 100644 index 000000000..46cba6cd8 --- /dev/null +++ b/tests/java/CMakeLists.txt @@ -0,0 +1,66 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# --- Common Test Properties --- +set(DEFAULT_TEST_TIMEOUT 9000) +set(DEFAULT_TEST_LABELS "java::all") +set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# --- Helper Function to Add Tests with Common Properties --- +# This function simplifies adding new tests that share common properties. +# Usage: gekko_add_test( [EXTRA_PROPERTIES ...]) +function(gekko_add_test TEST_NAME SCRIPT_FILENAME) + add_test( + NAME ${TEST_NAME} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT_FILENAME} + WORKING_DIRECTORY ${DEFAULT_TEST_WORKING_DIR} + ) + + set_tests_properties(${TEST_NAME} + PROPERTIES + TIMEOUT ${DEFAULT_TEST_TIMEOUT} + LABELS ${DEFAULT_TEST_LABELS} + ) +endfunction() + + + +gekko_add_test(java java1.sh) + + +# --- Installation of Test Scripts --- +if(GKFS_INSTALL_TESTS) + install( + DIRECTORY . # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/java + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + FILES_MATCHING + PATTERN "*.sh" # Only install .sh files + + ) +endif() \ No newline at end of file diff --git a/tests/java/DirectoryOperations.java b/tests/java/DirectoryOperations.java new file mode 100644 index 000000000..6014fccf6 --- /dev/null +++ b/tests/java/DirectoryOperations.java @@ -0,0 +1,262 @@ +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +public +class DirectoryOperations { + +public + static void + main(String[] args) { + if(args.length == 0) { + System.err.println( + "Usage: java DirectoryOperations "); + System.err.println( + "If the directory doesn't exist, it will be created (if possible)."); + System.exit(1); + } + + String dirPathString = args[0]; + Path directoryPath = Paths.get(dirPathString); + Path testFilePath = directoryPath.resolve( + "myTestFile.txt"); // File inside the target directory + + System.out.println( + "--- Starting I/O and Process Operations on Directory: " + + directoryPath.toAbsolutePath() + " ---\n"); + + try { + // 0. Ensure directory exists (or create it) + if(!Files.exists(directoryPath)) { + System.out.println( + "[INFO] Directory does not exist. Attempting to create: " + + directoryPath); + Files.createDirectories( + directoryPath); // Creates parent dirs if necessary + System.out.println("[SUCCESS] Directory created: " + + directoryPath.toAbsolutePath()); + } else if(!Files.isDirectory(directoryPath)) { + System.err.println( + "[ERROR] Path exists but is not a directory: " + + directoryPath.toAbsolutePath()); + System.exit(1); + } else { + System.out.println("[INFO] Directory already exists: " + + directoryPath.toAbsolutePath()); + } + + System.out.println("\n--- File Operations on: " + + testFilePath.toAbsolutePath() + " ---"); + + // 1. Create a file + System.out.println("\n1. CREATE Operation:"); + if(Files.exists(testFilePath)) { + System.out.println( + " [WARN] Test file already exists. Deleting it first."); + Files.delete(testFilePath); + } + Files.createFile(testFilePath); + System.out.println(" [SUCCESS] Created file: " + testFilePath); + + // 2. Write to the file + System.out.println("\n2. WRITE Operation:"); + String contentToWrite = + "Hello from Java I/O!\nLine 2: This is a test.\nLine 3: End of content."; + // Using try-with-resources for BufferedWriter + try(BufferedWriter writer = Files.newBufferedWriter( + testFilePath, StandardCharsets.UTF_8, + StandardOpenOption.WRITE)) { + writer.write(contentToWrite); + System.out.println( + " [SUCCESS] Wrote content to file:\n---\n" + + contentToWrite + "\n---"); + } + + // 3. Read from the file + System.out.println("\n3. READ Operation:"); + // Using Files.readString for simplicity (Java 11+) + // For older Java, use BufferedReader with Files.newBufferedReader + String readContent = + Files.readString(testFilePath, StandardCharsets.UTF_8); + System.out.println(" [SUCCESS] Read content from file:\n---\n" + + readContent + "\n---"); + if(!contentToWrite.equals(readContent)) { + System.out.println( + " [WARN] Read content does not exactly match written content (this might be due to line endings if not handled carefully)."); + } + + // 4. Stat (get file attributes) + System.out.println("\n4. STAT (Attributes) Operation:"); + BasicFileAttributes attrs = Files.readAttributes( + testFilePath, BasicFileAttributes.class); + System.out.println(" [INFO] File Size: " + attrs.size() + + " bytes"); + System.out.println(" [INFO] Is Directory: " + + attrs.isDirectory()); + System.out.println(" [INFO] Is Regular File: " + + attrs.isRegularFile()); + System.out.println(" [INFO] Is Symbolic Link: " + + attrs.isSymbolicLink()); + System.out.println(" [INFO] Last Access Time: " + + attrs.lastAccessTime()); + System.out.println(" [INFO] Last Modified Time: " + + attrs.lastModifiedTime()); + System.out.println(" [INFO] Creation Time: " + + attrs.creationTime()); + + // 5. Fork and Exec (ls or dir) + System.out.println( + "\n\n--- Fork and Exec (Listing directory contents) ---"); + executeExternalProcess(directoryPath); + + getLsCpuOutput(); + + } catch(IOException e) { + System.err.println("\n[ERROR] An I/O error occurred: " + + e.getMessage()); + e.printStackTrace(); + } catch(InterruptedException e) { + System.err.println("\n[ERROR] Process execution was interrupted: " + + e.getMessage()); + Thread.currentThread().interrupt(); // Restore interrupted status + e.printStackTrace(); + } finally { + // 6. Remove the file (cleanup) + System.out.println("\n\n--- Cleanup ---"); + System.out.println("\n6. REMOVE Operation (Cleanup):"); + try { + if(Files.exists(testFilePath)) { + Files.delete(testFilePath); + System.out.println(" [SUCCESS] Removed test file: " + + testFilePath); + } else { + System.out.println( + " [INFO] Test file was already removed or never created properly."); + } + } catch(IOException e) { + System.err.println(" [ERROR] Failed to remove test file " + + testFilePath + ": " + e.getMessage()); + } + // Optional: remove the directory if it was created by this script + // and is empty For this example, we'll leave the directory. + } + System.out.println("\n--- Operations Complete ---"); + } + +private + static void + executeExternalProcess(Path directoryToInspect) throws IOException, + InterruptedException { + List command = new ArrayList<>(); + command.add("ls"); + command.add("-la"); + command.add(directoryToInspect.toAbsolutePath() + .toString()); // `ls -la /path/to/dir` + + System.out.println(" Executing command: " + + String.join(" ", command)); + + ProcessBuilder processBuilder = new ProcessBuilder(command); + // processBuilder.directory(directoryToInspect.toFile()); // + // Alternative: set working directory + + Process process = processBuilder.start(); + + // Capture standard output + System.out.println("\n --- Process Standard Output ---"); + try(BufferedReader stdInput = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String s; + while((s = stdInput.readLine()) != null) { + System.out.println(" " + s); + } + } + + // Capture standard error (if any) + boolean hasErrorOutput = false; + StringBuilder errorOutput = new StringBuilder(); + try(BufferedReader stdError = new BufferedReader( + new InputStreamReader(process.getErrorStream()))) { + String s; + while((s = stdError.readLine()) != null) { + if(!hasErrorOutput) { + System.out.println("\n --- Process Standard Error ---"); + hasErrorOutput = true; + } + errorOutput.append(" ").append(s).append("\n"); + } + } + if(hasErrorOutput) + System.out.print(errorOutput.toString()); + + + int exitCode = process.waitFor(); + System.out.println("\n [INFO] Process exited with code: " + exitCode); + + if(exitCode != 0 && !hasErrorOutput) { + System.out.println( + " [WARN] Process exited with non-zero status but no error output captured. Check command."); + } else if(exitCode == 0 && hasErrorOutput) { + System.out.println( + " [WARN] Process exited successfully but produced error output. This might be informational."); + } + } + + +private + static String + getLsCpuOutput() { + // ************************************************************************************** + // Get LSCPU output + // ************************************************************************************** + String cmdOutput = null; + List command = new ArrayList<>(); + command.add("lscpu"); + + ProcessBuilder pb = new ProcessBuilder(command); + try { + Process process = pb.start(); + + // Disable inputs to process + process.getOutputStream().close(); + // Wait and retrieve exit value + int exitValue = process.waitFor(); + if(exitValue != 0) { + System.out.println( + "ERROR: LSCPU command failed with exitValue " + + exitValue); + } + + // Retrieve command output + StringBuilder sb = new StringBuilder(); + try(BufferedReader br = new BufferedReader( + new InputStreamReader(process.getInputStream()))) { + String line = null; + while((line = br.readLine()) != null) { + sb.append(line).append("\n"); + } + } + catch(IOException ioe) { + System.out.println( + "ERROR: Cannot retrieve LSCPU command output"); + } + cmdOutput = sb.toString(); + } catch(IOException ioe) { + System.out.println("ERROR: Cannot start LSCPU ProcessBuilder"); + } catch(InterruptedException ie) { + System.out.println("ERROR: LSCPU command interrupted"); + } + + return cmdOutput; + } +} diff --git a/tests/java/java1.sh b/tests/java/java1.sh new file mode 100755 index 000000000..c87cfa7d4 --- /dev/null +++ b/tests/java/java1.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/app/lockfile/original +export MNT=$IO/tests/app/lockfile/mnt +export ROOT=$IO/tests/app/lockfile/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_lockfile.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=/root/wacommplusplus/build/external/lib:$IO/lib/:$LD_LIBRARY_PATH + +export APP=./java DirectoryOperations + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +echo "compiling java app..." +javac /builds/gitlab/hpc/gekkofs/tests/java/DirectoryOperations.java + + +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=none +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/client_syscall.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + +echo $OUTPUT_SYSCALL +echo $SYSCALL +echo "========================================" +echo $OUTPUT_LIBC +echo $LIBC + + + +exit 0 \ No newline at end of file -- GitLab From 64d7973dbd24ca5f78547f82d339df1143705d9b Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 23 May 2025 12:30:32 +0200 Subject: [PATCH 02/12] enable java test --- tests/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 722764c7e..a032ee75a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -49,4 +49,5 @@ add_subdirectory(integration) # unit tests add_subdirectory(unit) - +# java tests +add_subdirectory(java) \ No newline at end of file -- GitLab From 51d5d6f727aa569cb85f0277d9ead30847582c74 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 23 May 2025 14:22:50 +0200 Subject: [PATCH 03/12] finish java testing --- tests/apps/CMakeLists.txt | 4 +- tests/java/CMakeLists.txt | 4 +- tests/java/java1.sh | 89 +++++++++++++++++++++++++++++++++------ 3 files changed, 79 insertions(+), 18 deletions(-) diff --git a/tests/apps/CMakeLists.txt b/tests/apps/CMakeLists.txt index e79fd67bd..83fefaeaf 100644 --- a/tests/apps/CMakeLists.txt +++ b/tests/apps/CMakeLists.txt @@ -66,8 +66,8 @@ gekko_add_test(lockfile lockfile.sh) # --- Installation of Test Scripts --- if(GKFS_INSTALL_TESTS) install( - DIRECTORY . # Installs files from the current source directory - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/apps + DIRECTORY apps # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE FILES_MATCHING PATTERN "*.sh" # Only install .sh files diff --git a/tests/java/CMakeLists.txt b/tests/java/CMakeLists.txt index 46cba6cd8..f01478639 100644 --- a/tests/java/CMakeLists.txt +++ b/tests/java/CMakeLists.txt @@ -56,8 +56,8 @@ gekko_add_test(java java1.sh) # --- Installation of Test Scripts --- if(GKFS_INSTALL_TESTS) install( - DIRECTORY . # Installs files from the current source directory - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/java + DIRECTORY java # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE FILES_MATCHING PATTERN "*.sh" # Only install .sh files diff --git a/tests/java/java1.sh b/tests/java/java1.sh index c87cfa7d4..852850ed6 100755 --- a/tests/java/java1.sh +++ b/tests/java/java1.sh @@ -8,14 +8,14 @@ export GKFS=$IO/lib/libgkfs_intercept.so export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so export DAEMON=$IO/bin/gkfs_daemon -export OUTSIDE=$IO/tests/app/lockfile/original -export MNT=$IO/tests/app/lockfile/mnt -export ROOT=$IO/tests/app/lockfile/root -export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_lockfile.txt +export OUTSIDE=$IO/tests/java/java1/original +export MNT=$IO/tests/java/java1/mnt +export ROOT=$IO/tests/java/java1/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE export LD_LIBRARY_PATH=/root/wacommplusplus/build/external/lib:$IO/lib/:$LD_LIBRARY_PATH -export APP=./java DirectoryOperations +export APP="java DirectoryOperations" rm -rf $MNT rm -rf $ROOT @@ -36,8 +36,9 @@ if ! ps -p $DAEMON_PID > /dev/null; then fi echo "compiling java app..." -javac /builds/gitlab/hpc/gekkofs/tests/java/DirectoryOperations.java - +cd /builds/gitlab/hpc/gekkofs/tests/java/ +javac DirectoryOperations.java +echo "compiled java app..." LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc @@ -47,22 +48,82 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/ export LIBGKFS_LOG=none export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/` +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/JAVA` SYSCALL=$? export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/client_syscall.txt export LIBGKFS_LOG=all -OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/` +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/JAVA` LIBC=$? # Stop the daemon kill $DAEMON_PID -echo $OUTPUT_SYSCALL -echo $SYSCALL -echo "========================================" -echo $OUTPUT_LIBC -echo $LIBC +REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' +REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' + + +preprocess_text() { + local text_input="$1" + local processed_text + + # 1. Remove newlines, then remove spaces + # Using Bash parameter expansion: + processed_text="${text_input//$'\n'/}" # Remove newlines + processed_text="${processed_text// /}" # Remove all spaces + + # Alternatively using tr (can be slower for large strings already in vars): + # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') + + # 2. Normalize dates on the compacted string + # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") + # The ISO date regex is unaffected as it doesn't contain spaces. + processed_text=$(echo "$processed_text" | \ + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ + sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' + ) + echo "$processed_text" +} + + +NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") +NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") +NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") +NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") + +if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then + echo "SUCCESS: The SYSCALL output matches the reference (ignoring dates)." +else + echo "FAILURE: The SYSCALL output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") +fi + +if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then + echo "SUCCESS: The LIBC output matches the reference (ignoring dates)." +else + echo "FAILURE: The LIBC output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") +fi + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + exit 0 \ No newline at end of file -- GitLab From 628a777cecc5c1c5f03dc495c379d80cb072db98 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 23 May 2025 14:33:38 +0200 Subject: [PATCH 04/12] separating vfork and fork execution --- tests/java/CMakeLists.txt | 4 +- tests/java/java1.sh | 129 ------------------------------------- tests/java/java_fork.sh | 131 ++++++++++++++++++++++++++++++++++++++ tests/java/java_vfork.sh | 131 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 130 deletions(-) delete mode 100755 tests/java/java1.sh create mode 100755 tests/java/java_fork.sh create mode 100755 tests/java/java_vfork.sh diff --git a/tests/java/CMakeLists.txt b/tests/java/CMakeLists.txt index f01478639..f9272cbe2 100644 --- a/tests/java/CMakeLists.txt +++ b/tests/java/CMakeLists.txt @@ -50,7 +50,9 @@ endfunction() -gekko_add_test(java java1.sh) +gekko_add_test(java_fork java_fork.sh) +gekko_add_test(java_vfork java_vfork.sh) + # --- Installation of Test Scripts --- diff --git a/tests/java/java1.sh b/tests/java/java1.sh deleted file mode 100755 index 852850ed6..000000000 --- a/tests/java/java1.sh +++ /dev/null @@ -1,129 +0,0 @@ -#!/bin/bash - -# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats - -## Environment variables -export IO=/builds/gitlab/hpc/gekkofs/gkfs/install -export GKFS=$IO/lib/libgkfs_intercept.so -export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so -export DAEMON=$IO/bin/gkfs_daemon - -export OUTSIDE=$IO/tests/java/java1/original -export MNT=$IO/tests/java/java1/mnt -export ROOT=$IO/tests/java/java1/root -export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt -export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE -export LD_LIBRARY_PATH=/root/wacommplusplus/build/external/lib:$IO/lib/:$LD_LIBRARY_PATH - -export APP="java DirectoryOperations" - -rm -rf $MNT -rm -rf $ROOT - -mkdir -p $MNT $ROOT - -# Start gkfs_daemon in the background -echo "Starting gkfs_daemon..." -$DAEMON -r $ROOT -m $MNT & -DAEMON_PID=$! - -sleep 5 - -# Check if the daemon is running -if ! ps -p $DAEMON_PID > /dev/null; then - echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" - exit 1 -fi - -echo "compiling java app..." -cd /builds/gitlab/hpc/gekkofs/tests/java/ -javac DirectoryOperations.java -echo "compiled java app..." - -LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc - -mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/ -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/client_syscall.txt -# LOG should be none for java yet -export LIBGKFS_LOG=none -export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create - -OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/JAVA` -SYSCALL=$? - -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java1/client_syscall.txt -export LIBGKFS_LOG=all -OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/JAVA` -LIBC=$? -# Stop the daemon -kill $DAEMON_PID - - - - -REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' -REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java1/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' - - -preprocess_text() { - local text_input="$1" - local processed_text - - # 1. Remove newlines, then remove spaces - # Using Bash parameter expansion: - processed_text="${text_input//$'\n'/}" # Remove newlines - processed_text="${processed_text// /}" # Remove all spaces - - # Alternatively using tr (can be slower for large strings already in vars): - # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') - - # 2. Normalize dates on the compacted string - # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") - # The ISO date regex is unaffected as it doesn't contain spaces. - processed_text=$(echo "$processed_text" | \ - sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ - sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' - ) - echo "$processed_text" -} - - -NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") -NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") -NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") -NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") - -if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then - echo "SUCCESS: The SYSCALL output matches the reference (ignoring dates)." -else - echo "FAILURE: The SYSCALL output does NOT match the reference." - echo "Differences found:" - diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") -fi - -if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then - echo "SUCCESS: The LIBC output matches the reference (ignoring dates)." -else - echo "FAILURE: The LIBC output does NOT match the reference." - echo "Differences found:" - diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") -fi - -# Inform about SYSCALL and LIBC error codes -if [ "$SYSCALL" -ne 0 ]; then - echo "FAILURE: SYSCALL command exited with non-zero status code: $SYSCALL" -fi - -if [ "$LIBC" -ne 0 ]; then - echo "FAILURE: LIBC command exited with non-zero status code: $LIBC" -fi - -# Exit with a non-zero status if either test failed -if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then - exit 1 -fi - -# If all tests passed, exit with 0 - - -exit 0 \ No newline at end of file diff --git a/tests/java/java_fork.sh b/tests/java/java_fork.sh new file mode 100755 index 000000000..e5654d4cc --- /dev/null +++ b/tests/java/java_fork.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/java/java_fork/original +export MNT=$IO/tests/java/java_fork/mnt +export ROOT=$IO/tests/java/java_fork/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=$IO/lib/:$LD_LIBRARY_PATH + +export APPFORK="java -Djdk.lang.Process.launchMechanism=fork DirectoryOperations" +export APPVFORK="java -Djdk.lang.Process.launchMechanism=vfork DirectoryOperations" + +REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' +REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_fork/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' + + +preprocess_text() { + local text_input="$1" + local processed_text + + # 1. Remove newlines, then remove spaces + # Using Bash parameter expansion: + processed_text="${text_input//$'\n'/}" # Remove newlines + processed_text="${processed_text// /}" # Remove all spaces + + # Alternatively using tr (can be slower for large strings already in vars): + # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') + + # 2. Normalize dates on the compacted string + # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") + # The ISO date regex is unaffected as it doesn't contain spaces. + processed_text=$(echo "$processed_text" | \ + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ + sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' + ) + echo "$processed_text" +} + + +NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") +NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") + + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +echo "compiling java app..." +cd /builds/gitlab/hpc/gekkofs/tests/java/ +javac DirectoryOperations.java +echo "compiled java app..." + +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=none +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPFORK $MNT/syscall/JAVA` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_syscall.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPFORK $MNT/libc/JAVA` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + + +NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") +NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") + +if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then + echo "SUCCESS: The SYSCALL (FORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The SYSCALL (FORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") +fi + +if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then + echo "SUCCESS: The LIBC (FORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The LIBC (FORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") +fi + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (FORK) command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (FORK) command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file diff --git a/tests/java/java_vfork.sh b/tests/java/java_vfork.sh new file mode 100755 index 000000000..e23790355 --- /dev/null +++ b/tests/java/java_vfork.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/java/java_vfork/original +export MNT=$IO/tests/java/java_vfork/mnt +export ROOT=$IO/tests/java/java_vfork/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_java.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=/root/wacommplusplus/build/external/lib:$IO/lib/:$LD_LIBRARY_PATH + +export APPFORK="java -Djdk.lang.Process.launchMechanism=fork DirectoryOperations" +export APPVFORK="java -Djdk.lang.Process.launchMechanism=vfork DirectoryOperations" + +REFERENCE_OUTPUT_SYSCALL='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA --- Process Standard Output --- total 0 drwxrwxrwx 0 root root 0 May 23 11:16 . drwxrwxrwx 0 root root 0 May 23 11:16 .. -rw-r--r-- 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/syscall/JAVA/myTestFile.txt --- Operations Complete ---' +REFERENCE_OUTPUT_LIBC='--- Starting I/O and Process Operations on Directory: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- [INFO] Directory does not exist. Attempting to create: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA [SUCCESS] Directory created: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- File Operations on: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt --- 1. CREATE Operation: [SUCCESS] Created file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt 2. WRITE Operation: [SUCCESS] Wrote content to file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 3. READ Operation: [SUCCESS] Read content from file: --- Hello from Java I/O! Line 2: This is a test. Line 3: End of content. --- 4. STAT (Attributes) Operation: [INFO] File Size: 68 bytes [INFO] Is Directory: false [INFO] Is Regular File: true [INFO] Is Symbolic Link: false [INFO] Last Access Time: 1970-01-01T00:00:00Z [INFO] Last Modified Time: 2025-05-23T11:16:52Z [INFO] Creation Time: 2025-05-23T11:16:52Z --- Fork and Exec (Listing directory contents) --- Executing command: ls -la /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA --- Process Standard Output --- total 0 drwxrwxrwx? 0 root root 0 May 23 11:16 . drwxrwxrwx? 0 root root 0 May 23 11:16 .. -rw-r--r--? 0 root root 68 Jan 1 1970 myTestFile.txt [INFO] Process exited with code: 0 --- Cleanup --- 6. REMOVE Operation (Cleanup): [SUCCESS] Removed test file: /builds/gitlab/hpc/gekkofs/gkfs/install/tests/java/java_vfork/mnt/libc/JAVA/myTestFile.txt --- Operations Complete ---' + + +preprocess_text() { + local text_input="$1" + local processed_text + + # 1. Remove newlines, then remove spaces + # Using Bash parameter expansion: + processed_text="${text_input//$'\n'/}" # Remove newlines + processed_text="${processed_text// /}" # Remove all spaces + + # Alternatively using tr (can be slower for large strings already in vars): + # processed_text=$(echo "$text_input" | tr -d '\n' | tr -d ' ') + + # 2. Normalize dates on the compacted string + # The regex for ls-style dates now expects no spaces (e.g., "May2311:16") + # The ISO date regex is unaffected as it doesn't contain spaces. + processed_text=$(echo "$processed_text" | \ + sed -E 's/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z//g' | \ + sed -E 's/[A-Za-z]{3}[0-9]+([0-9]{2}:[0-9]{2}|[0-9]{4})//g' + ) + echo "$processed_text" +} + + +NORMALIZED_REFERENCE_SYSCALL=$(preprocess_text "$REFERENCE_OUTPUT_SYSCALL") +NORMALIZED_REFERENCE_LIBC=$(preprocess_text "$REFERENCE_OUTPUT_LIBC") + + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +echo "compiling java app..." +cd /builds/gitlab/hpc/gekkofs/tests/java/ +javac DirectoryOperations.java +echo "compiled java app..." + +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=none +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPVFORK $MNT/syscall/JAVA` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_syscall.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPVFORK $MNT/libc/JAVA` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + + +NORMALIZED_OUTPUT_SYSCALL=$(preprocess_text "$OUTPUT_SYSCALL") +NORMALIZED_OUTPUT_LIBC=$(preprocess_text "$OUTPUT_LIBC") + +if [ "$NORMALIZED_OUTPUT_SYSCALL" = "$NORMALIZED_REFERENCE_SYSCALL" ]; then + echo "SUCCESS: The SYSCALL (VFORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The SYSCALL (VFORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_SYSCALL") <(echo "$NORMALIZED_REFERENCE_SYSCALL") +fi + +if [ "$NORMALIZED_OUTPUT_LIBC" = "$NORMALIZED_REFERENCE_LIBC" ]; then + echo "SUCCESS: The LIBC (VFORK) output matches the reference (ignoring dates)." +else + echo "FAILURE: The LIBC (VFORK) output does NOT match the reference." + echo "Differences found:" + diff -u <(echo "$NORMALIZED_OUTPUT_LIBC") <(echo "$NORMALIZED_REFERENCE_LIBC") +fi + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (VFORK) command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (VFORK) command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$NORMALIZED_OUTPUT_SYSCALL" != "$NORMALIZED_REFERENCE_SYSCALL" ] || [ "$NORMALIZED_OUTPUT_LIBC" != "$NORMALIZED_REFERENCE_LIBC" ] || [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file -- GitLab From ec0735beec63a9198a1f5d7930c42fa310563bef Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 08:06:10 +0200 Subject: [PATCH 05/12] add python testing --- .gitlab-ci.yml | 43 +++++++++++++++++++- tests/CMakeLists.txt | 5 ++- tests/java/java_fork.sh | 2 +- tests/java/java_vfork.sh | 2 +- tests/python/CMakeLists.txt | 68 +++++++++++++++++++++++++++++++ tests/python/python_1.sh | 74 ++++++++++++++++++++++++++++++++++ tests/python/python_exists.py | 75 +++++++++++++++++++++++++++++++++++ 7 files changed, 265 insertions(+), 4 deletions(-) create mode 100644 tests/python/CMakeLists.txt create mode 100755 tests/python/python_1.sh create mode 100644 tests/python/python_exists.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 91ff92917..cd389c722 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -386,6 +386,47 @@ gkfs:java: junit: ${BUILD_PATH}/tests/java/report.xml +## == python tests for gkfs ================== +gkfs:python: + stage: test + image: gekkofs/testing:0.9.5 + needs: ['gkfs'] + script: + ## Add path to mkfs.kreon + - export PATH=${PATH}:/usr/local/bin + ## run actual tests + - cd ${CI_PROJECT_DIR} + - ctest --test-dir ${BUILD_PATH} + -j 1 + -L python::all + -VV + --output-junit ${BUILD_PATH}/tests/python/report.xml + + ## capture coverage information for this test and write it to + ## $COVERAGE_PATH/java.info + - cd ${CI_PROJECT_DIR} + # use ccache + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/python.info + -DARGS_EXCLUDE_DIRECTORIES=${CI_PROJECT_DIR}/external + ## Since the pipeline recreates the source tree, the access times for .gcno + ## files are newer than those of .gcda files. This makes gcov emit a + ## warning for each file which slows it down. Updating the timestamps + ## avoids this + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + artifacts: + expire_in: 1 day + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/python/report.xml + + ################################################################################ ## Generation of documentation ################################################################################ @@ -457,7 +498,7 @@ coverage: stage: report image: gekkofs/testing:0.9.5 #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java'] + needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] script: # use ccache - ccache --zero-stats diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a032ee75a..2db460ca2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -50,4 +50,7 @@ add_subdirectory(integration) add_subdirectory(unit) # java tests -add_subdirectory(java) \ No newline at end of file +add_subdirectory(java) + +# java tests +add_subdirectory(python) \ No newline at end of file diff --git a/tests/java/java_fork.sh b/tests/java/java_fork.sh index e5654d4cc..8ff12c85a 100755 --- a/tests/java/java_fork.sh +++ b/tests/java/java_fork.sh @@ -83,7 +83,7 @@ export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPFORK $MNT/syscall/JAVA` SYSCALL=$? -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_syscall.txt +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_fork/client_libc.txt export LIBGKFS_LOG=all OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPFORK $MNT/libc/JAVA` LIBC=$? diff --git a/tests/java/java_vfork.sh b/tests/java/java_vfork.sh index e23790355..7f21a0d0f 100755 --- a/tests/java/java_vfork.sh +++ b/tests/java/java_vfork.sh @@ -83,7 +83,7 @@ export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APPVFORK $MNT/syscall/JAVA` SYSCALL=$? -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_syscall.txt +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/java_vfork/client_libc.txt export LIBGKFS_LOG=all OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APPVFORK $MNT/libc/JAVA` LIBC=$? diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt new file mode 100644 index 000000000..7f4ae21d4 --- /dev/null +++ b/tests/python/CMakeLists.txt @@ -0,0 +1,68 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# --- Common Test Properties --- +set(DEFAULT_TEST_TIMEOUT 9000) +set(DEFAULT_TEST_LABELS "python::all") +set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) + +# --- Helper Function to Add Tests with Common Properties --- +# This function simplifies adding new tests that share common properties. +# Usage: gekko_add_test( [EXTRA_PROPERTIES ...]) +function(gekko_add_test TEST_NAME SCRIPT_FILENAME) + add_test( + NAME ${TEST_NAME} + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT_FILENAME} + WORKING_DIRECTORY ${DEFAULT_TEST_WORKING_DIR} + ) + + set_tests_properties(${TEST_NAME} + PROPERTIES + TIMEOUT ${DEFAULT_TEST_TIMEOUT} + LABELS ${DEFAULT_TEST_LABELS} + ) +endfunction() + + + +gekko_add_test(python_1 python_1.sh) + + + + +# --- Installation of Test Scripts --- +if(GKFS_INSTALL_TESTS) + install( + DIRECTORY python # Installs files from the current source directory + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + FILES_MATCHING + PATTERN "*.sh" # Only install .sh files + + ) +endif() \ No newline at end of file diff --git a/tests/python/python_1.sh b/tests/python/python_1.sh new file mode 100755 index 000000000..877088752 --- /dev/null +++ b/tests/python/python_1.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Typical command line : /builds/gitlab/hpc/gekkofs/gkfs/install/bin/gkfs_daemon --mountdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/mnt --rootdir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root -l lo:22543 --metadir /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/root --dbbackend rocksdb --output-stats /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/popen-gw0/test_write_gkfs_daemon_rocksdb0/logs/stats.log --enable-collection --enable-chunkstats + +## Environment variables +PYTHON=PYTHON_1 +export IO=/builds/gitlab/hpc/gekkofs/gkfs/install +export GKFS=$IO/lib/libgkfs_intercept.so +export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so +export DAEMON=$IO/bin/gkfs_daemon + +export OUTSIDE=$IO/tests/python/$PYTHON/original +export MNT=$IO/tests/python/$PYTHON/mnt +export ROOT=$IO/tests/python/$PYTHON/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_$PYTHON.txt +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE +export LD_LIBRARY_PATH=$IO/lib/:$LD_LIBRARY_PATH + +rm -rf $MNT +rm -rf $ROOT + +mkdir -p $MNT $ROOT + +# Start gkfs_daemon in the background +echo "Starting gkfs_daemon..." +$DAEMON -r $ROOT -m $MNT & +DAEMON_PID=$! + +sleep 5 + +# Check if the daemon is running +if ! ps -p $DAEMON_PID > /dev/null; then + echo "Error: gkfs_daemon failed to start! PID: $DAEMON_PID" + exit 1 +fi + +$APP="python /builds/gitlab/hpc/gekkofs/tests/python/python_exists.py" +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc + +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/client_syscall.txt +# LOG should be none for java yet +export LIBGKFS_LOG=all +export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create + +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/$PYTHON` +SYSCALL=$? + +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/client_libc.txt +export LIBGKFS_LOG=all +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/$PYTHON` +LIBC=$? +# Stop the daemon +kill $DAEMON_PID + + +# Inform about SYSCALL and LIBC error codes +if [ "$SYSCALL" -ne 0 ]; then + echo "FAILURE: SYSCALL (FORK) command exited with non-zero status code: $SYSCALL" +fi + +if [ "$LIBC" -ne 0 ]; then + echo "FAILURE: LIBC (FORK) command exited with non-zero status code: $LIBC" +fi + +# Exit with a non-zero status if either test failed +if [ "$SYSCALL" -ne 0 ] || [ "$LIBC" -ne 0 ]; then + exit 1 +fi + +# If all tests passed, exit with 0 + + +exit 0 \ No newline at end of file diff --git a/tests/python/python_exists.py b/tests/python/python_exists.py new file mode 100644 index 000000000..8b6b53c3d --- /dev/null +++ b/tests/python/python_exists.py @@ -0,0 +1,75 @@ +import os +import sys +import time # Import the time module + +def main(): + # 1. Check if a command-line argument for the file path is provided + if len(sys.argv) < 2: + print(f"Usage: python {sys.argv[0]} ") + sys.exit(1) # Exit with an error code + + file_path = sys.argv[1] + file_created_by_script = False # Flag to ensure we only delete what we create + + print(f"Target file path from argv[1]: {file_path}") + + try: + # 2. Create the file (or overwrite if it exists) + # Ensure the directory exists if the path includes new directories + directory = os.path.dirname(file_path) + if directory and not os.path.exists(directory): + print(f"Creating directory: {directory}") + os.makedirs(directory, exist_ok=True) # exist_ok=True prevents error if dir already exists + + print(f"Attempting to create file: {file_path}") + with open(file_path, 'w') as f: + f.write("This is a test file.\n") + f.write("Created to demonstrate os.path.exists().\n") + print(f"File '{file_path}' created successfully.") + file_created_by_script = True # Mark that we created it + + # 3. Use os.path.exists() to check if the file exists + if os.path.exists(file_path): + print(f"SUCCESS: os.path.exists('{file_path}') returned True.") + # You can also check if it's a file specifically + if os.path.isfile(file_path): + print(f"And os.path.isfile('{file_path}') also returned True.") + else: + print(f"Warning: os.path.isfile('{file_path}') returned False (it might be a directory).") + else: + # This case should ideally not be hit if creation was successful + print(f"FAILURE: os.path.exists('{file_path}') returned False, even after attempting creation.") + + except IOError as e: + print(f"Error during file operation for '{file_path}': {e}") + # Check existence even after an error, as the file might still exist or not + if os.path.exists(file_path): + print(f"INFO: os.path.exists('{file_path}') returned True (file might have existed before or other issue).") + else: + print(f"INFO: os.path.exists('{file_path}') returned False (as expected after error).") + except Exception as e: + print(f"An unexpected error occurred: {e}") + + finally: + # 4. Clean up: remove the file only if this script created it + if file_created_by_script and os.path.exists(file_path): + try: + print(f"Cleaning up: Removing '{file_path}'") + os.remove(file_path) + print(f"File '{file_path}' removed.") + except OSError as e: + print(f"Error removing file '{file_path}': {e}") + elif file_path and not file_created_by_script and os.path.exists(file_path): + print(f"Note: File '{file_path}' existed but was not created by this script run, so not removing it.") + # Cleanup directory if it was created by this script and is now empty + if directory and os.path.exists(directory) and not os.listdir(directory) and file_created_by_script: + try: + print(f"Cleaning up empty directory: '{directory}'") + os.rmdir(directory) + print(f"Directory '{directory}' removed.") + except OSError as e: + print(f"Error removing directory '{directory}': {e}") + + +if __name__ == "__main__": + main() -- GitLab From 3b18a8c2918978d0319c8d70bada323177860ebb Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 08:10:27 +0200 Subject: [PATCH 06/12] renamed python tests --- tests/python/CMakeLists.txt | 2 +- tests/python/{python_1.sh => python_exists.sh} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename tests/python/{python_1.sh => python_exists.sh} (97%) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 7f4ae21d4..79951af18 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -50,7 +50,7 @@ endfunction() -gekko_add_test(python_1 python_1.sh) +gekko_add_test(python_exists python_exists.sh) diff --git a/tests/python/python_1.sh b/tests/python/python_exists.sh similarity index 97% rename from tests/python/python_1.sh rename to tests/python/python_exists.sh index 877088752..a7984ecc7 100755 --- a/tests/python/python_1.sh +++ b/tests/python/python_exists.sh @@ -34,7 +34,7 @@ if ! ps -p $DAEMON_PID > /dev/null; then exit 1 fi -$APP="python /builds/gitlab/hpc/gekkofs/tests/python/python_exists.py" +APP="python /builds/gitlab/hpc/gekkofs/tests/python/python_exists.py" LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/ -- GitLab From c3fe18f344b1e5b41245ce28b17ec9deaab3a417 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 08:38:10 +0200 Subject: [PATCH 07/12] reduce timeout --- tests/java/CMakeLists.txt | 2 +- tests/python/CMakeLists.txt | 2 +- tests/python/python_exists.sh | 20 ++++++++++---------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/java/CMakeLists.txt b/tests/java/CMakeLists.txt index f9272cbe2..d888f04e7 100644 --- a/tests/java/CMakeLists.txt +++ b/tests/java/CMakeLists.txt @@ -27,7 +27,7 @@ ################################################################################ # --- Common Test Properties --- -set(DEFAULT_TEST_TIMEOUT 9000) +set(DEFAULT_TEST_TIMEOUT 60) set(DEFAULT_TEST_LABELS "java::all") set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 79951af18..ddb1cf4ac 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -27,7 +27,7 @@ ################################################################################ # --- Common Test Properties --- -set(DEFAULT_TEST_TIMEOUT 9000) +set(DEFAULT_TEST_TIMEOUT 60) set(DEFAULT_TEST_LABELS "python::all") set(DEFAULT_TEST_WORKING_DIR ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/tests/python/python_exists.sh b/tests/python/python_exists.sh index a7984ecc7..76a0d6d4b 100755 --- a/tests/python/python_exists.sh +++ b/tests/python/python_exists.sh @@ -9,10 +9,10 @@ export GKFS=$IO/lib/libgkfs_intercept.so export GKFS_LIBC=$IO/lib/libgkfs_libc_intercept.so export DAEMON=$IO/bin/gkfs_daemon -export OUTSIDE=$IO/tests/python/$PYTHON/original -export MNT=$IO/tests/python/$PYTHON/mnt -export ROOT=$IO/tests/python/$PYTHON/root -export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_$PYTHON.txt +export OUTSIDE=$IO/tests/python/${PYTHON}/original +export MNT=$IO/tests/python/${PYTHON}/mnt +export ROOT=$IO/tests/python/${PYTHON}/root +export GKFS_HOSTS_FILE=$ROOT/gkfs_hosts_${PYTHON}.txt export LIBGKFS_HOSTS_FILE=$GKFS_HOSTS_FILE export LD_LIBRARY_PATH=$IO/lib/:$LD_LIBRARY_PATH @@ -34,21 +34,21 @@ if ! ps -p $DAEMON_PID > /dev/null; then exit 1 fi -APP="python /builds/gitlab/hpc/gekkofs/tests/python/python_exists.py" +APP=/builds/gitlab/hpc/gekkofs/tests/python/python_exists.py LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc -mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/ -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/client_syscall.txt +mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/ +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_syscall.txt # LOG should be none for java yet export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS $APP $MNT/syscall/$PYTHON` +OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS python3 $APP $MNT/syscall/${PYTHON}/file1` SYSCALL=$? -export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/$PYTHON/client_libc.txt +export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_libc.txt export LIBGKFS_LOG=all -OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC $APP $MNT/libc/$PYTHON` +OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC python3 $APP $MNT/libc/${PYTHON}/file2` LIBC=$? # Stop the daemon kill $DAEMON_PID -- GitLab From 61b44e4185da1ad4574bfe704a75bb0ea3ab1740 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 09:35:45 +0200 Subject: [PATCH 08/12] added exit in python to mark errors --- tests/python/python_exists.py | 8 ++++++++ tests/python/python_exists.sh | 11 +++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/python/python_exists.py b/tests/python/python_exists.py index 8b6b53c3d..e94961559 100644 --- a/tests/python/python_exists.py +++ b/tests/python/python_exists.py @@ -36,19 +36,24 @@ def main(): print(f"And os.path.isfile('{file_path}') also returned True.") else: print(f"Warning: os.path.isfile('{file_path}') returned False (it might be a directory).") + sys.exit(1) else: # This case should ideally not be hit if creation was successful print(f"FAILURE: os.path.exists('{file_path}') returned False, even after attempting creation.") + sys.exit(1) except IOError as e: print(f"Error during file operation for '{file_path}': {e}") # Check existence even after an error, as the file might still exist or not if os.path.exists(file_path): print(f"INFO: os.path.exists('{file_path}') returned True (file might have existed before or other issue).") + sys.exit(1) else: print(f"INFO: os.path.exists('{file_path}') returned False (as expected after error).") + sys.exit(1) except Exception as e: print(f"An unexpected error occurred: {e}") + sys.exit(1) finally: # 4. Clean up: remove the file only if this script created it @@ -59,8 +64,10 @@ def main(): print(f"File '{file_path}' removed.") except OSError as e: print(f"Error removing file '{file_path}': {e}") + sys.exit(1) elif file_path and not file_created_by_script and os.path.exists(file_path): print(f"Note: File '{file_path}' existed but was not created by this script run, so not removing it.") + sys.exit(1) # Cleanup directory if it was created by this script and is now empty if directory and os.path.exists(directory) and not os.listdir(directory) and file_created_by_script: try: @@ -69,6 +76,7 @@ def main(): print(f"Directory '{directory}' removed.") except OSError as e: print(f"Error removing directory '{directory}': {e}") + sys.exit(1) if __name__ == "__main__": diff --git a/tests/python/python_exists.sh b/tests/python/python_exists.sh index 76a0d6d4b..8b2337342 100755 --- a/tests/python/python_exists.sh +++ b/tests/python/python_exists.sh @@ -35,19 +35,20 @@ if ! ps -p $DAEMON_PID > /dev/null; then fi APP=/builds/gitlab/hpc/gekkofs/tests/python/python_exists.py -LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc +LD_PRELOAD=$GKFS mkdir $MNT/syscall $MNT/libc mkdir -p /builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_syscall.txt -# LOG should be none for java yet -export LIBGKFS_LOG=all +# LOG should be debug for python +export LIBGKFS_LOG=debug export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create - +echo "Starting python test 1..." OUTPUT_SYSCALL=`LD_PRELOAD=$GKFS python3 $APP $MNT/syscall/${PYTHON}/file1` SYSCALL=$? export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/${PYTHON}/client_libc.txt export LIBGKFS_LOG=all +echo "Starting python test 2..." OUTPUT_LIBC=`LD_PRELOAD=$GKFS_LIBC python3 $APP $MNT/libc/${PYTHON}/file2` LIBC=$? # Stop the daemon @@ -57,10 +58,12 @@ kill $DAEMON_PID # Inform about SYSCALL and LIBC error codes if [ "$SYSCALL" -ne 0 ]; then echo "FAILURE: SYSCALL (FORK) command exited with non-zero status code: $SYSCALL" + echo $OUTPUT_SYSCALL fi if [ "$LIBC" -ne 0 ]; then echo "FAILURE: LIBC (FORK) command exited with non-zero status code: $LIBC" + echo $OUTPUT_LIBC fi # Exit with a non-zero status if either test failed -- GitLab From fc505bc79899d66083ec1195e1f98d463ab19901 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 14:13:58 +0200 Subject: [PATCH 09/12] Solved execve issue, not restoring reentrant variable with syscall --- src/client/gkfs_libc.cpp | 28 ---------------------------- src/client/intercept.cpp | 4 ++++ 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/src/client/gkfs_libc.cpp b/src/client/gkfs_libc.cpp index ca04d0bd2..0c2721a11 100644 --- a/src/client/gkfs_libc.cpp +++ b/src/client/gkfs_libc.cpp @@ -605,11 +605,6 @@ DLSYM_WRAPPER(int, dup2, (int fd, int fd2), (fd, fd2), "dup2") DLSYM_WRAPPER(int, dup3, (int fd, int fd2, int flags), (fd, fd2, flags), "dup3") static int (*real_fcntl)(int fd, int cmd, ...) = nullptr; // Special handling for variadic fcntl -DLSYM_WRAPPER(void, exit, (int status), (status), "exit") -#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS -DLSYM_WRAPPER(int, pipe, (int pipefd[2]), (pipefd), "pipe") -#endif - // Memory Mapping DLSYM_WRAPPER(void*, mmap, (void* addr, size_t length, int prot, int flags, int fd, @@ -2437,29 +2432,6 @@ fcntl(int fd, int cmd, ...) { } } -#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS -void -exit(int status) { - // GekkoFS cleanup might be needed before exit (e.g., flushing, - // releasing resources) This should ideally be in CTX destructor or a - // dedicated shutdown function. - DEBUG_INFO("[GKFS] exit(status={})", status); - // gkfs::PreloadContext::getInstance()->shutdown(); // Example - dlsym_exit(status); - __builtin_unreachable(); // To tell compiler exit doesn't return -} - -int -pipe(int pipefd[2]) { - gkfs_init_routine_placeholder(); - // Pipes are usually not intercepted by file system shims unless they - // are named pipes (FIFOs) and GekkoFS implements them. Standard - // anonymous pipes are kernel objects. - DEBUG_INFO("[BYPASS] pipe()"); - return dlsym_pipe(pipefd); -} -#endif - //------------------------- Memory Mapping //-----------------------------------// diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index deae776ba..1ae6e9860 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -532,6 +532,8 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, switch(syscall_number) { case SYS_execve: + // If we do not set this to false, we are in trouble with vforks + reentrance_guard_flag = false; *result = syscall_no_intercept_wrapper( syscall_number, reinterpret_cast(arg0), reinterpret_cast(arg1), @@ -540,6 +542,8 @@ hook(long syscall_number, long arg0, long arg1, long arg2, long arg3, long arg4, #ifdef SYS_execveat case SYS_execveat: + // If we do not set this to false, we are in trouble with vforks + reentrance_guard_flag = false; *result = syscall_no_intercept_wrapper( syscall_number, arg0, reinterpret_cast(arg1), reinterpret_cast(arg2), -- GitLab From 1c618ec288781794ae29ae7ad50f70350d802eb6 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 14:19:00 +0200 Subject: [PATCH 10/12] disable chmod check for execv --- src/client/hooks.cpp | 5 ++-- .../harness/gkfs.io/syscall_coverage.cpp | 24 +++++++++---------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index 4aae63a80..86c75781b 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -717,7 +717,8 @@ hook_chdir(const char* path) { bool internal = CTX->relativize_path(path, rel_path); if(internal) { // path falls in our namespace - auto md = gkfs::utils::get_metadata(rel_path); + /* This get_metadata gets stuck after a forkandExec */ + /*auto md = gkfs::utils::get_metadata(rel_path); if(!md) { LOG(ERROR, "{}() path {} / {} errno {}", __func__, path, rel_path, errno); @@ -727,7 +728,7 @@ hook_chdir(const char* path) { if(!S_ISDIR(md->mode())) { LOG(ERROR, "{}() path is not a directory", __func__); return -ENOTDIR; - } + }*/ // TODO get complete path from relativize_path instead of // removing mountdir and then adding again here rel_path.insert(0, CTX->mountdir()); diff --git a/tests/integration/harness/gkfs.io/syscall_coverage.cpp b/tests/integration/harness/gkfs.io/syscall_coverage.cpp index b21d801a3..67c21b81e 100644 --- a/tests/integration/harness/gkfs.io/syscall_coverage.cpp +++ b/tests/integration/harness/gkfs.io/syscall_coverage.cpp @@ -1554,21 +1554,21 @@ syscall_coverage_exec(const syscall_coverage_options& opts) { output("readlinkat", rv, opts); return; } - + // This won't fail as intented // chdir internal error - rv = ::chdir(opts.pathname.c_str()); - if(errno != ENOTDIR) { - output("chdir", rv, opts); - return; - } + // rv = ::chdir(opts.pathname.c_str()); + // if(errno != ENOTDIR) { + // output("chdir", rv, opts); + // return; + //} // chdir internal error - std::string nonexist = opts.pathname + "x2"; - rv = ::chdir(nonexist.c_str()); - if(rv >= 0) { - output("chdir", rv, opts); - return; - } + // std::string nonexist = opts.pathname + "x2"; + // rv = ::chdir(nonexist.c_str()); + // if(rv >= 0) { + // output("chdir", rv, opts); + // return; + //} // fchdir auto fddir = ::open(".", O_RDONLY); -- GitLab From 10866f8023745d6a85c65b5c8e910c7cf5f429df Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 16:08:20 +0200 Subject: [PATCH 11/12] update clone params --- src/client/syscalls/detail/syscall_info.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/syscalls/detail/syscall_info.c b/src/client/syscalls/detail/syscall_info.c index c4c9ba259..0e463e84a 100644 --- a/src/client/syscalls/detail/syscall_info.c +++ b/src/client/syscalls/detail/syscall_info.c @@ -642,7 +642,7 @@ SYSCALL(getpmsg, 5, S_RET(rdec), S_NARG(arg, "arg0"), SYSCALL(pidfd_open, 2, S_RET(rdec), S_NARG(dec, "pid"), S_NARG(arg, "flags")), #endif #ifdef SYS_clone3 - SYSCALL(clone3, 4, S_RET(rdec), S_NARG(clone3_args, "flags"), S_NARG(arg, "size")), + SYSCALL(clone3, 2, S_RET(rdec), S_NARG(clone3_args, "flags"), S_NARG(arg, "size")), #endif #ifdef SYS_close_range SYSCALL(close_range, 3, S_RET(rdec), S_NARG(dec, "low"), S_NARG(dec, "high"), S_NARG(arg, "flags")), -- GitLab From 20c1fdd390f8cb31cc3357f960a4a95c18974491 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 2 Jun 2025 16:17:42 +0200 Subject: [PATCH 12/12] CHANGELOG --- CHANGELOG.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40ab79790..5dcbf5d9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Use PROTECT_FILES_GENERATOR=1 and PROTECT_FILES_CONSUMER=1 to enable. Generator, creates transparent .lockgekko files that blocks the open (for some seconds) of any consumer. Multiple opens / closes for generator are managed. - Basic mmap support ([!247](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/245)) - LIBC interception support, with extra tests ([!202](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/202)) - + - Fixes for vfork, java and python ([!257](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/257)) + - Adds two new testing stages, java and python. + ### Changed - Tests check ret for -1 instead of 10000 fd ([!320](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/320)) - Allow some more script tests to run as pthread_at_fork solved some issues. @@ -41,6 +43,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Java with syscalls deadlocks as it try to resolve paths (malloc) in a locking situation ([!255](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/255)) - It also solves flock missing implementation when we are ouside gekkofs - Some features in syscall_intercept still hangs if we do not lower the debug information. + - Fixes for vfork, java and python ([!257](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/257)) + - execve and chdir failed as calling the server for get_metadata fails, disabled chdir checks (GekkoFS) + - execve set reentrant_guard as true, so once it returns (vfork) the parent had syscall disabled. + - debug should be still set to debug level on syscall for vfork. + - In MN5 gkfs_libc.hpp needs to have the libc signatures. + ## [0.9.4] - 2025-03 ### New -- GitLab