diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake index 32500a5980f2a453871485f770387f698a544c63..b289bda359e76414fd745d3d294b662544e11046 100644 --- a/CMake/gkfs-options.cmake +++ b/CMake/gkfs-options.cmake @@ -227,6 +227,15 @@ gkfs_define_option( DEFAULT_VALUE OFF ) +# build performance tests +gkfs_define_option( + GKFS_BUILD_PERFORMANCE + HELP_TEXT "Build ${PROJECT_NAME} performance benchmarks" + DEFAULT_VALUE OFF + DESCRIPTION "Compile standalone performance benchmark tests" + FEATURE_NAME "PERF" +) + # build tools gkfs_define_option( GKFS_BUILD_TOOLS diff --git a/CMakeLists.txt b/CMakeLists.txt index 76a5607d845c249898978c754ecaa350bf3287b6..96ef55a2189e656cc3f281ee6460e469c43aeff3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -356,6 +356,11 @@ endif () ### variables. mark_variables_as_advanced(REGEX "^(FETCHCONTENT|fmt|FMT|spdlog|SPDLOG)_.*$") +if (GKFS_BUILD_PERFORMANCE) + message(STATUS "[${PROJECT_NAME}] Building performance benchmarks...") + add_subdirectory(perf_tests) +endif() + if (GKFS_BUILD_TESTS) # Boost preprocessor header-only is supplied by the Mercury installation find_package(Boost_preprocessor REQUIRED) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index b70ace750766cf321cfb0710880bad5f2809fdf4..57acfeb5d0acd73d61039a085f4bc8e9583f35bc 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -60,7 +60,7 @@ int gkfs_libcremove(const std::string& path); int -gkfs_remove(const std::string& path); +gkfs_remove(const std::string& path, bool is_rename_stub = false); // Implementation of access, // Follow links is true by default diff --git a/include/client/rpc/forward_metadata.hpp b/include/client/rpc/forward_metadata.hpp index cf7d1e5747392105c7d8f38e26c79672eeb03c7a..3acd96684c75df7ef3c05d09dbb147b0798364e0 100644 --- a/include/client/rpc/forward_metadata.hpp +++ b/include/client/rpc/forward_metadata.hpp @@ -84,6 +84,11 @@ int forward_remove(const std::string& path, bool rm_dir, const int8_t num_copies, int64_t size); +int +forward_remove(const std::string& path, bool rm_dir, const int8_t num_copies, + int64_t& size, uint32_t& mode, std::string& target_path, + bool is_rename_stub = false); + int forward_decr_size(const std::string& path, size_t length, const int copy); diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index b505cfa5caed31a9ceeef806820bb68595388428..3b15537ba41b797f2189d6222ed73328bb10f433 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -65,7 +65,7 @@ gkfs_create(const std::string& path, mode_t mode); int gkfs_libcremove(const std::string& path); int -gkfs_remove(const std::string& path); +gkfs_remove(const std::string& path, bool is_rename_stub = false); int gkfs_rmdir(const std::string& path); diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index 8fd9fdf7fbb45cb634522e74df07a36f09612ee2..34d7251de9b381cabed74f65b4240fedd73763a2 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -131,11 +131,12 @@ struct rpc_stat_out_t { struct rpc_rm_node_in_t { std::string path; bool rm_dir; + bool is_rename_stub; template void serialize(Archive& ar) { - ar(path, rm_dir); + ar(path, rm_dir, is_rename_stub); } }; @@ -143,11 +144,12 @@ struct rpc_rm_metadata_out_t { int32_t err; int64_t size; uint32_t mode; + std::string target_path; template void serialize(Archive& ar) { - ar(err, size, mode); + ar(err, size, mode, target_path); } }; diff --git a/perf_tests/CMakeLists.txt b/perf_tests/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..77e2b16108496aaf04ff8607a26840bb8b940e33 --- /dev/null +++ b/perf_tests/CMakeLists.txt @@ -0,0 +1,143 @@ +# CMakeLists.txt for GekkoFS Performance Tests +# Standalone performance benchmarks for comparing performance across changes + +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# Compiler requirements +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# Common compile flags (simple list, avoid inheriting CMAKE_CXX_FLAGS as a single string) +set(GKFS_PERF_CXX_FLAGS "-O2" "-Wall") +set(GKFS_PERF_CXX_LINKER_FLAGS "") + +# Output directory +set(GKFS_PERF_OUTPUT_DIR "${CMAKE_BINARY_DIR}/perf_tests") +file(MAKE_DIRECTORY ${GKFS_PERF_OUTPUT_DIR}) + +# ============================================================================ +# Performance test executables +# ============================================================================ + +# Metadata performance test +add_executable(perf_metadata + perf_metadata.cpp +) +target_compile_options(perf_metadata PRIVATE ${GKFS_PERF_CXX_FLAGS}) +target_link_options(perf_metadata PRIVATE ${GKFS_PERF_CXX_LINKER_FLAGS}) +target_include_directories(perf_metadata PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) +target_link_libraries(perf_metadata + stdc++fs +) +set_target_properties(perf_metadata PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} +) + +# Data performance test +add_executable(perf_data + perf_data.cpp +) +target_compile_options(perf_data PRIVATE ${GKFS_PERF_CXX_FLAGS}) +target_link_options(perf_data PRIVATE ${GKFS_PERF_CXX_LINKER_FLAGS}) +target_include_directories(perf_data PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) +set_target_properties(perf_data PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} +) + +# Delete performance test +add_executable(perf_delete + perf_delete.cpp +) +target_compile_options(perf_delete PRIVATE ${GKFS_PERF_CXX_FLAGS}) +target_link_options(perf_delete PRIVATE ${GKFS_PERF_CXX_LINKER_FLAGS}) +target_include_directories(perf_delete PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) +set_target_properties(perf_delete PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} +) + +# Find/search performance test +add_executable(perf_find + perf_find.cpp +) +target_compile_options(perf_find PRIVATE ${GKFS_PERF_CXX_FLAGS}) +target_link_options(perf_find PRIVATE ${GKFS_PERF_CXX_LINKER_FLAGS}) +target_include_directories(perf_find PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) +set_target_properties(perf_find PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} +) + +# Directory performance test +add_executable(perf_directory + perf_directory.cpp +) +target_compile_options(perf_directory PRIVATE ${GKFS_PERF_CXX_FLAGS}) +target_link_options(perf_directory PRIVATE ${GKFS_PERF_CXX_LINKER_FLAGS}) +target_include_directories(perf_directory PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) +set_target_properties(perf_directory PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} +) + +# ============================================================================ +# Custom targets for running benchmarks +# ============================================================================ + +# Run all benchmarks (requires mounted filesystem) +add_custom_target(run-all-perf-tests + COMMAND ${CMAKE_COMMAND} -E echo "Please mount your filesystem first, then run:" + COMMAND ${CMAKE_COMMAND} -E echo " ./perf_metadata --mountdir /mnt/gekkofs" + COMMAND ${CMAKE_COMMAND} -E echo " ./perf_data --mountdir /mnt/gekkofs" + COMMAND ${CMAKE_COMMAND} -E echo " ./perf_delete --mountdir /mnt/gekkofs" + COMMAND ${CMAKE_COMMAND} -E echo " ./perf_find --mountdir /mnt/gekkofs" + COMMAND ${CMAKE_COMMAND} -E echo " ./perf_directory --mountdir /mnt/gekkofs" + DEPENDS perf_metadata perf_data perf_delete perf_find perf_directory + WORKING_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} + COMMENT "Build complete. Run benchmarks manually on a mounted filesystem." +) + +# Run benchmarks with runner script +add_custom_target(run-all-perf-with-script + COMMAND ${CMAKE_COMMAND} -E echo "Running benchmarks with runner script..." + COMMAND ${CMAKE_COMMAND} -E chdir ${GKFS_PERF_OUTPUT_DIR} ./run_benchmarks.sh + DEPENDS perf_metadata perf_data perf_delete perf_find perf_directory + WORKING_DIRECTORY ${GKFS_PERF_OUTPUT_DIR} + COMMENT "Running all benchmarks with default settings." +) + +# Install targets +install(TARGETS perf_metadata perf_data perf_delete perf_find perf_directory + RUNTIME DESTINATION bin) +install(FILES run_benchmarks.sh + DESTINATION bin) +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ + DESTINATION share/gekkofs/perf_tests + FILES_MATCHING PATTERN "*.sh") diff --git a/perf_tests/README.md b/perf_tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d896dbe81604a092c02e54da3384b92c8d0b7cdb --- /dev/null +++ b/perf_tests/README.md @@ -0,0 +1,302 @@ +# GekkoFS Performance Test Suite + +Standalone performance benchmarks for comparing performance across code changes. Each test can run in isolation with a single server-client node. + +## Overview + +This directory contains standalone performance tests designed to quickly compare performance values between code changes. The tests are organized by operation type: + +| Benchmark | File | Description | +|-----------|------|-------------| +| **Metadata** | `perf_metadata.cpp` | Stat, chmod, rename, symlink operations | +| **Data I/O** | `perf_data.cpp` | Sequential/random read/write, small file operations | +| **Delete** | `perf_delete.cpp` | File delete, directory delete, batch delete, tree delete | +| **Find/Search** | `perf_find.cpp` | Tree traversal, pattern matching, directory enumeration | +| **Directory** | `perf_directory.cpp` | Directory create, list, iterate operations | + +## Building + +### Option 1: Using the main project CMake (with GEKKOFS_BUILD_PERFORMANCE) + +```bash +# From the project root +cd build_release +cmake ../ -DCMAKE_BUILD_TYPE=Release -DGKFS_BUILD_PERFORMANCE=ON +make -j$(nproc) +``` + +The benchmarks will be built in `build_release/perf_tests/`. + +### Option 2: Standalone build + +```bash +cd perf_tests +mkdir build && cd build +cmake .. -DCMAKE_BUILD_TYPE=Release +make -j$(nproc) +``` + +## Running Benchmarks + +### Prerequisites + +1. **Build GekkoFS**: + ```bash + cd build_release + make -j$(nproc) + ``` + +2. **Start the GekkoFS server**: + ```bash + ./src/daemon/gekkofsd --config /path/to/config.yaml & + ``` + +3. **Mount the filesystem** (if using FUSE mode): + ```bash + sudo gekkofs -o helpers=... /dev/sdX /mnt/gekkofs + ``` + +### Quick Start: Run All Benchmarks + +#### LD_PRELOAD Mode (recommended) + +Simply pass the GekkoFS syscall intercept library as first argument: + +```bash +# Run all benchmarks with LD_PRELOAD +./run_benchmarks.sh /path/to/gkfs_syscall_intercept.so /mnt/gekkofs + +# Or individual benchmarks +LD_PRELOAD=/path/to/gkfs_syscall_intercept.so ./perf_metadata --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +LD_PRELOAD=/path/to/gkfs_syscall_intercept.so ./perf_data --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +LD_PRELOAD=/path/to/gkfs_syscall_intercept.so ./perf_delete --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +LD_PRELOAD=/path/to/gkfs_syscall_intercept.so ./perf_find --mountdir /mnt/gekkofs --iterations 10 --warmup 2 +LD_PRELOAD=/path/to/gkfs_syscall_intercept.so ./perf_directory --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +``` + +#### FUSE Mount Mode (alternative) + +Run benchmarks directly on the FUSE mount point (no LD_PRELOAD needed): + +```bash +./perf_metadata --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +./perf_data --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +./perf_delete --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +./perf_find --mountdir /mnt/gekkofs --iterations 10 --warmup 2 +./perf_directory --mountdir /mnt/gekkofs --iterations 100 --warmup 5 +``` + +### Running Individual Benchmarks + +#### 1. Metadata Performance + +```bash +./perf_metadata --mountdir /mnt/gekkofs \ + --iterations 100 \ + --warmup 5 \ + --symlinks 50 \ + --hardlinks 50 \ + --files-per-link 10 +``` + +**Tests:** +- `stat` operations (file metadata retrieval) +- `chmod` operations (permission changes) +- `rename` operations (file renaming) +- `symlink` create + stat + readlink (symbolic link operations) + +#### 2. Data I/O Performance + +```bash +./perf_data --mountdir /mnt/gekkofs \ + --iterations 100 \ + --filesize 1048576 \ + --small-files 100 \ + --warmup 5 +``` + +**Tests:** +- Sequential write (single large file) +- Sequential read (single large file) +- Random write (4KB at random offset) +- Random read (4KB at random offset) +- Small file write (many small files) +- Small file read (many small files) + +#### 3. Delete Performance + +```bash +./perf_delete --mountdir /mnt/gekkofs \ + --iterations 100 \ + --files 100 \ + --depth 4 \ + --branches 4 \ + --leaves 4 \ + --warmup 5 +``` + +**Tests:** +- File create + delete (individual files) +- Batch delete (multiple files in one directory) +- Tree create + delete (nested directory structure) +- Rmdir performance (directory removal) + +#### 4. Find/Search Performance + +```bash +./perf_find --mountdir /mnt/gekkofs \ + --iterations 10 \ + --dir-count 50 \ + --files-per-dir 100 \ + --subdirs 5 \ + --warmup 2 +``` + +**Tests:** +- Full tree traversal +- Find by pattern (file_*.txt) +- Find directories only +- Find files only +- Single level readdir + +#### 5. Directory Operations Performance + +```bash +./perf_directory --mountdir /mnt/gekkofs \ + --iterations 100 \ + --warmup 5 \ + --dir-size 1000 \ + --depth 3 \ + --branches 4 +``` + +**Tests:** +- Directory create (single level) +- Deep directory tree create +- Directory list (readdir) +- Directory iterate (stat all entries) + +### Common Options + +All benchmarks share these common options: + +``` +--mountdir Mount point of GekkoFS (required) +--iterations Number of benchmark iterations (default varies) +--warmup Number of warmup iterations (default: 5) +--no-cleanup Don't cleanup test files after benchmark +--help Show help message +``` + +## Interpreting Results + +Each benchmark outputs timing statistics: + +``` +Metric: Value (unit) + Avg: 12.34 ms + Min: 10.12 ms (P50) + Max: 15.67 ms (P99) + P95: 14.89 ms + P99: 15.23 ms + Total: 1234.56 ms +``` + +- **Avg**: Average time across all iterations +- **Min/Max**: Minimum and maximum observed times +- **P50/P95/P99**: Percentile values (median, 95th, 99th percentile) +- **Total**: Total time for all iterations + +## Comparing Results Across Changes + +### Method 1: Manual Comparison + +1. Run benchmarks before and after your changes +2. Results are saved to `perf_tests/results/` with timestamps +3. Compare the output files manually + +```bash +# Compare two result files +diff <(grep "Avg:" results/metadata_20260119_120000.txt) \ + <(grep "Avg:" results/metadata_20260119_130000.txt) +``` + +### Method 2: Automated Comparison Script + +```bash +# Run baseline +./run_benchmarks.sh /mnt/gekkofs + +# ... make your code changes ... + +# Run after changes +./run_benchmarks.sh /mnt/gekkofs_after + +# Compare +ls -t perf_tests/results/*.txt | head -2 | xargs -I{} diff {} /dev/null +``` + +## Performance Test Structure + +Each benchmark is a standalone C++ executable that: + +1. **Creates test files/directories** in the mounted filesystem +2. **Warms up** the filesystem cache/connection +3. **Runs the benchmark** for the specified number of iterations +4. **Collects timing statistics** using high-resolution timers +5. **Cleans up** test files (unless `--no-cleanup` is specified) + +### Test Architecture + +``` +perf_tests/ +├── perf_common.hpp # Shared utilities (Timer, TimingResult, run_benchmark) +├── perf_metadata.cpp # Metadata operations benchmark +├── perf_data.cpp # Data I/O benchmark +├── perf_delete.cpp # Delete operations benchmark +├── perf_find.cpp # Find/search benchmark +├── perf_directory.cpp # Directory operations benchmark +├── run_benchmarks.sh # Runner script for all benchmarks +├── CMakeLists.txt # Build configuration +└── README.md # This file +``` + +## Example: Quick Performance Check + +```bash +# 1. Mount the filesystem +sudo gekkofs /dev/sdX /mnt/gekkofs + +# 2. Run a quick metadata test +./perf_metadata --mountdir /mnt/gekkofs --iterations 50 --warmup 3 + +# 3. Make code changes and rebuild + +# 4. Run the same test +./perf_metadata --mountdir /mnt/gekkofs --iterations 50 --warmup 3 + +# 5. Compare the Avg values to see performance impact +``` + +## Tips for Accurate Results + +1. **Run multiple times**: Take the median of 3 runs +2. **Minimize system load**: Close other applications during testing +3. **Use consistent settings**: Keep `--iterations` and `--warmup` constant +4. **Mount options**: Ensure mount options are identical between runs +5. **Cold vs warm cache**: Use `--no-cleanup` to control this + +## Troubleshooting + +### "Mount point does not exist" +Ensure the filesystem is mounted at the specified path. + +### "Permission denied" +Check that you have write permissions to the mount point. + +### "Benchmark executable not found" +Build the benchmarks first using cmake/make. + +### Slow results +- Increase `--warmup` to allow cache warming +- Ensure the filesystem is healthy and responsive \ No newline at end of file diff --git a/perf_tests/perf_common.hpp b/perf_tests/perf_common.hpp new file mode 100644 index 0000000000000000000000000000000000000000..36ed347c28602c8ed205e9a3e27f19f1d465993f --- /dev/null +++ b/perf_tests/perf_common.hpp @@ -0,0 +1,171 @@ +// Performance test common utilities for GekkoFS +// Standalone benchmarking framework for comparing performance across changes + +#ifndef GKFS_PERF_COMMON_HPP +#define GKFS_PERF_COMMON_HPP + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// ============================================================================ +// Timing utilities +// ============================================================================ + +struct TimingResult { + double min_ms = 0; + double max_ms = 0; + double mean_ms = 0; + double median_ms = 0; + double std_dev_ms = 0; + double p50_ms = 0; + double p90_ms = 0; + double p95_ms = 0; + double p99_ms = 0; + double total_ms = 0; + size_t iterations = 0; +}; + +class Timer { +public: + static std::chrono::steady_clock::time_point now() { + return std::chrono::steady_clock::now(); + } + + static double elapsed_ms(std::chrono::steady_clock::time_point start) { + return std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + } +}; + +// Finalize helper for TimingResult +inline TimingResult finalize_timing(const std::vector& intervals_ms) { + TimingResult result; + if (intervals_ms.empty()) return result; + + result.iterations = intervals_ms.size(); + result.min_ms = *std::min_element(intervals_ms.begin(), intervals_ms.end()); + result.max_ms = *std::max_element(intervals_ms.begin(), intervals_ms.end()); + + double sum = std::accumulate(intervals_ms.begin(), intervals_ms.end(), 0.0); + result.mean_ms = sum / intervals_ms.size(); + result.total_ms = sum; + + // Median + std::vector sorted_intervals = intervals_ms; + std::sort(sorted_intervals.begin(), sorted_intervals.end()); + size_t n = sorted_intervals.size(); + result.median_ms = (n % 2 == 0) + ? (sorted_intervals[n/2 - 1] + sorted_intervals[n/2]) / 2.0 + : sorted_intervals[n/2]; + + // Percentiles + result.p50_ms = sorted_intervals[size_t(n * 0.50)]; + result.p90_ms = sorted_intervals[size_t(n * 0.90)]; + result.p95_ms = sorted_intervals[size_t(n * 0.95)]; + result.p99_ms = sorted_intervals[size_t(n * 0.99)]; + + // Standard deviation + double sq_sum = std::accumulate(intervals_ms.begin(), intervals_ms.end(), 0.0, + [mean = result.mean_ms](double acc, double val) { + return acc + (val - mean) * (val - mean); + }); + result.std_dev_ms = std::sqrt(sq_sum / n); + + return result; +} + +// ============================================================================ +// Results formatting +// ============================================================================ + +inline std::string format_timing(const TimingResult& result, const std::string& op_name, + size_t total_ops = 0) { + std::ostringstream oss; + oss << "\n" << std::string(60, '=') << "\n"; + oss << " " << op_name << "\n"; + oss << std::string(60, '-') << "\n"; + oss << std::fixed << std::setprecision(3); + oss << " Iterations: " << result.iterations << "\n"; + if (total_ops > 0) { + oss << " Total ops: " << total_ops << "\n"; + oss << " Throughput: " << std::setprecision(2) + << (total_ops / (result.total_ms / 1000.0)) << " ops/sec\n"; + } + oss << std::fixed << std::setprecision(3); + oss << " Total time: " << result.total_ms << " ms\n"; + oss << " Min: " << result.min_ms << " ms\n"; + oss << " Max: " << result.max_ms << " ms\n"; + oss << " Mean: " << result.mean_ms << " ms\n"; + oss << " Median: " << result.median_ms << " ms\n"; + oss << " Std dev: " << result.std_dev_ms << " ms\n"; + oss << " p50: " << result.p50_ms << " ms\n"; + oss << " p90: " << result.p90_ms << " ms\n"; + oss << " p95: " << result.p95_ms << " ms\n"; + oss << " p99: " << result.p99_ms << " ms\n"; + oss << std::string(60, '=') << "\n"; + return oss.str(); +} + +// ============================================================================ +// Test runner helper +// ============================================================================ + +inline TimingResult run_benchmark(const std::function& benchmark_fn, + size_t num_warmup = 0, + size_t num_iterations = 100) { + // Warmup + for (size_t i = 0; i < num_warmup; ++i) { + benchmark_fn(); + } + + // Benchmark + std::vector intervals_ms; + intervals_ms.reserve(num_iterations); + + for (size_t i = 0; i < num_iterations; ++i) { + auto start = Timer::now(); + benchmark_fn(); + double elapsed = Timer::elapsed_ms(start); + intervals_ms.push_back(elapsed); + } + + return finalize_timing(intervals_ms); +} + +// ============================================================================ +// File helper utilities +// ============================================================================ + +inline void write_file(const std::string& path, size_t size_bytes, char fill_char = 'A') { + std::ofstream f(path, std::ios::binary); + std::vector buffer(1024 * 1024, fill_char); // 1MB chunks + size_t written = 0; + while (written < size_bytes) { + size_t to_write = std::min(buffer.size(), size_bytes - written); + f.write(buffer.data(), to_write); + written += to_write; + } + f.flush(); + f.close(); +} + +inline std::string read_file(const std::string& path) { + std::ifstream f(path, std::ios::binary); + return std::string((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); +} + +#endif // GKFS_PERF_COMMON_HPP \ No newline at end of file diff --git a/perf_tests/perf_data.cpp b/perf_tests/perf_data.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a1eb5a14a5a1a75ca7b1b61a75970fddf060da37 --- /dev/null +++ b/perf_tests/perf_data.cpp @@ -0,0 +1,569 @@ +// Performance test: Data operations (read/write) +// Tests: sequential write, random write, sequential read, random read, append +// Usage: ./perf_data --mountdir /path/to/mount --iterations 100 --filesize 1048576 + +#include "perf_common.hpp" + +#include +#include +#include +#include +#include +#include + +// ============================================================================ +// Benchmark functions +// ============================================================================ + +// --- Sequential Write benchmark --- +struct SeqWriteBenchmark { + std::string base_dir; + size_t file_size; // bytes + int fd; + + SeqWriteBenchmark(const std::string& dir, size_t sz) : base_dir(dir), file_size(sz), fd(-1) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + if (fd >= 0) { close(fd); fd = -1; } + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + const size_t buf_size = 1024 * 1024; // 1MB write buffer + std::vector write_buf(buf_size, 'A'); + + for (size_t i = 0; i < iterations; ++i) { + std::string path = base_dir + "/seq_write_" + std::to_string(i); + fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) break; + + auto start = Timer::now(); + size_t written = 0; + while (written < file_size) { + size_t to_write = std::min(buf_size, file_size - written); + int ret = write(fd, write_buf.data(), to_write); + if (ret <= 0) break; + written += ret; + } + fsync(fd); + close(fd); + fd = -1; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } + + // Sequential write with append (reuse file) + TimingResult append_run(size_t iterations) { + std::string path = base_dir + "/seq_append"; + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + const size_t buf_size = 1024 * 1024; + std::vector write_buf(buf_size, 'B'); + + // Create file once + fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) return TimingResult{}; + + for (size_t i = 0; i < iterations; ++i) { + lseek(fd, 0, SEEK_SET); + auto start = Timer::now(); + + size_t written = 0; + while (written < file_size) { + size_t to_write = std::min(buf_size, file_size - written); + int ret = write(fd, write_buf.data(), to_write); + if (ret <= 0) break; + written += ret; + } + fsync(fd); + + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + close(fd); + fd = -1; + return finalize_timing(intervals_ms); + } +}; + +// --- Sequential Read benchmark --- +struct SeqReadBenchmark { + std::string base_dir; + size_t file_size; + int fd; + + SeqReadBenchmark(const std::string& dir, size_t sz) : base_dir(dir), file_size(sz), fd(-1) {} + + void setup() { + fs::create_directories(base_dir); + // Create a file of the specified size + std::string fpath = base_dir + "/seq_read_file"; + fd = open(fpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) return; + + const size_t buf_size = 1024 * 1024; + std::vector buf(buf_size, 'C'); + size_t written = 0; + while (written < file_size) { + size_t to_write = std::min(buf_size, file_size - written); + write(fd, buf.data(), to_write); + written += to_write; + } + fsync(fd); + close(fd); + fd = -1; + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + const size_t buf_size = 1024 * 1024; + std::vector read_buf(buf_size); + + for (size_t i = 0; i < iterations; ++i) { + std::string fpath = base_dir + "/seq_read_file"; + fd = open(fpath.c_str(), O_RDONLY); + if (fd < 0) break; + + auto start = Timer::now(); + size_t total_read = 0; + while (true) { + int ret = read(fd, read_buf.data(), buf_size); + if (ret <= 0) break; + total_read += ret; + } + close(fd); + fd = -1; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// --- Random Write benchmark (seek + write) --- +struct RandWriteBenchmark { + std::string base_dir; + size_t file_size; + int fd; + std::mt19937 rng; + + RandWriteBenchmark(const std::string& dir, size_t sz) + : base_dir(dir), file_size(sz), fd(-1), rng(42) {} + + void setup() { + fs::create_directories(base_dir); + std::string fpath = base_dir + "/rand_write_file"; + fd = open(fpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) return; + close(fd); + fd = -1; + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + std::uniform_int_distribution dist(0, file_size - 4096); + + for (size_t i = 0; i < iterations; ++i) { + std::string fpath = base_dir + "/rand_write_file"; + fd = open(fpath.c_str(), O_RDWR); + if (fd < 0) break; + + auto start = Timer::now(); + + // Random seek and write + off_t pos = dist(rng); + lseek(fd, pos, SEEK_SET); + std::vector buf(4096, 'D'); + write(fd, buf.data(), 4096); + fsync(fd); + + close(fd); + fd = -1; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// --- Random Read benchmark (seek + read) --- +struct RandReadBenchmark { + std::string base_dir; + size_t file_size; + int fd; + std::mt19937 rng; + + RandReadBenchmark(const std::string& dir, size_t sz) + : base_dir(dir), file_size(sz), fd(-1), rng(42) {} + + void setup() { + fs::create_directories(base_dir); + std::string fpath = base_dir + "/rand_read_file"; + fd = open(fpath.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) return; + + const size_t buf_size = 1024 * 1024; + std::vector buf(buf_size, 'E'); + size_t written = 0; + while (written < file_size) { + size_t to_write = std::min(buf_size, file_size - written); + write(fd, buf.data(), to_write); + written += to_write; + } + fsync(fd); + close(fd); + fd = -1; + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + std::uniform_int_distribution dist(0, file_size - 4096); + + for (size_t i = 0; i < iterations; ++i) { + std::string fpath = base_dir + "/rand_read_file"; + fd = open(fpath.c_str(), O_RDONLY); + if (fd < 0) break; + + auto start = Timer::now(); + + off_t pos = dist(rng); + lseek(fd, pos, SEEK_SET); + char buf[4096]; + read(fd, buf, sizeof(buf)); + + close(fd); + fd = -1; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// --- Small file write/read (many small files) --- +struct SmallFileBenchmark { + std::string base_dir; + int num_files; + + SmallFileBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult write(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + for (int j = 0; j < num_files; ++j) { + std::string fpath = base_dir + "/small_" + std::to_string(j) + "_" + std::to_string(i); + std::ofstream f(fpath); + f << "small file test data for iteration " << i << " file " << j << "\n"; + f.flush(); + } + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } + + TimingResult read(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + // Create files once + for (int j = 0; j < num_files; ++j) { + std::string fpath = base_dir + "/small_" + std::to_string(j) + "_ref"; + std::ofstream f(fpath); + f << "reference small file data"; + } + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + for (int j = 0; j < num_files; ++j) { + std::string fpath = base_dir + "/small_" + std::to_string(j) + "_ref"; + std::ifstream f(fpath); + std::string content((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); + } + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// ============================================================================ +// Usage and help +// ============================================================================ + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n" + << "\nOptions:\n" + << " --mountdir Mount point of GekkoFS (required)\n" + << " --iterations Number of benchmark iterations (default: 100)\n" + << " --filesize File size for single-file tests (default: 1048576 = 1MB)\n" + << " --small-files Number of small files (default: 100)\n" + << " --warmup Number of warmup iterations (default: 5)\n" + << " --no-cleanup Don't cleanup test files after benchmark\n" + << " --help Show this help message\n"; +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char* argv[]) { + std::string mountdir = "/mnt/gekkofs"; + int iterations = 100; + size_t file_size = 1024 * 1024; // 1MB + int small_files = 100; + int warmup = 5; + bool cleanup = true; + + static struct option long_options[] = { + {"mountdir", required_argument, 0, 'm'}, + {"iterations", required_argument, 0, 'i'}, + {"filesize", required_argument, 0, 'f'}, + {"small-files", required_argument, 0, 's'}, + {"warmup", required_argument, 0, 'w'}, + {"no-cleanup", no_argument, 0, 'n'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int opt; + while ((opt = getopt_long(argc, argv, "m:i:f:s:w:nh", long_options, nullptr)) != -1) { + switch (opt) { + case 'm': mountdir = optarg; break; + case 'i': iterations = std::atoi(optarg); break; + case 'f': file_size = static_cast(std::atol(optarg)); break; + case 's': small_files = std::atoi(optarg); break; + case 'w': warmup = std::atoi(optarg); break; + case 'n': cleanup = false; break; + case 'h': print_usage(argv[0]); return 0; + default: print_usage(argv[0]); return 1; + } + } + + std::cout << "\n=== GekkoFS Data Performance Test ===\n"; + std::cout << "Mount directory: " << mountdir << "\n"; + std::cout << "Iterations: " << iterations << "\n"; + std::cout << "File size: " << file_size << " bytes (" << (file_size / (1024*1024)) << " MB)\n"; + std::cout << "Small files: " << small_files << "\n"; + std::cout << "Warmup: " << warmup << "\n\n"; + + std::string test_dir = mountdir + "/perf_data_test_XXXXXX"; + char* tmp = mkdtemp(const_cast(test_dir.c_str())); + if (!tmp) { + std::cerr << "Error: Failed to create temp directory\n"; + return 1; + } + + std::cout << "Test directory: " << tmp << "\n\n"; + + // ---- 1. Sequential Write benchmark ---- + { + std::cout << "--- Sequential Write (large file) ---\n"; + SeqWriteBenchmark bench(tmp, file_size); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + std::string path = std::string(tmp) + "/seq_write_iter"; + int fd = open(path.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) return; + + const size_t buf_size = 1024 * 1024; + std::vector write_buf(buf_size, 'A'); + size_t written = 0; + while (written < file_size) { + size_t to_write = std::min(buf_size, file_size - written); + write(fd, write_buf.data(), to_write); + written += to_write; + } + fsync(fd); + close(fd); + }, + warmup, iterations + ); + std::cout << format_timing(result, "Seq Write (single file, " + std::to_string(file_size / (1024*1024)) + " MB)", iterations) << "\n"; + + bench.cleanup(); + } + + // ---- 2. Sequential Read benchmark ---- + { + std::cout << "\n--- Sequential Read (large file) ---\n"; + SeqReadBenchmark bench(tmp, file_size); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + std::string fpath = std::string(tmp) + "/seq_read_file"; + int fd = open(fpath.c_str(), O_RDONLY); + if (fd < 0) return; + + const size_t buf_size = 1024 * 1024; + std::vector read_buf(buf_size); + while (true) { + int ret = read(fd, read_buf.data(), buf_size); + if (ret <= 0) break; + } + close(fd); + }, + warmup, iterations + ); + std::cout << format_timing(result, "Seq Read (single file, " + std::to_string(file_size / (1024*1024)) + " MB)", iterations) << "\n"; + + bench.cleanup(); + } + + // ---- 3. Random Write benchmark ---- + { + std::cout << "\n--- Random Write (seek + write 4KB) ---\n"; + RandWriteBenchmark bench(tmp, 10 * 1024 * 1024); // 10MB file + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + std::string fpath = std::string(tmp) + "/rand_write_file"; + int fd = open(fpath.c_str(), O_RDWR); + if (fd < 0) return; + + std::vector buf(4096, 'D'); + write(fd, buf.data(), 4096); + fsync(fd); + close(fd); + }, + warmup, iterations + ); + std::cout << format_timing(result, "Rand Write (4KB at random offset)", iterations) << "\n"; + + bench.cleanup(); + } + + // ---- 4. Random Read benchmark ---- + { + std::cout << "\n--- Random Read (seek + read 4KB) ---\n"; + RandReadBenchmark bench(tmp, 10 * 1024 * 1024); // 10MB file + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + std::string fpath = std::string(tmp) + "/rand_read_file"; + int fd = open(fpath.c_str(), O_RDONLY); + if (fd < 0) return; + + int rand_val = std::rand() % static_cast(9 * 1024 * 1024); + char buf[4096]; + lseek(fd, static_cast(rand_val), SEEK_SET); + read(fd, buf, sizeof(buf)); + close(fd); + }, + warmup, iterations + ); + std::cout << format_timing(result, "Rand Read (4KB at random offset)", iterations) << "\n"; + + bench.cleanup(); + } + + // ---- 5. Small file write benchmark ---- + { + std::cout << "\n--- Small File Write (many small files) ---\n"; + SmallFileBenchmark bench(tmp, small_files); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + for (int j = 0; j < small_files; ++j) { + std::string path = std::string(tmp) + "/small_write_" + std::to_string(std::rand()); + std::ofstream f(path); + f << "small file data " << j << "\n"; + f.flush(); + close(open(path.c_str(), O_WRONLY)); + } + }, + warmup, iterations + ); + std::cout << format_timing(result, "Small file write (" + std::to_string(small_files) + " files)", iterations) << "\n"; + + bench.cleanup(); + } + + // ---- 6. Small file read benchmark ---- + { + std::cout << "\n--- Small File Read (many small files) ---\n"; + SmallFileBenchmark bench(tmp, small_files); + + // Create reference files + for (int j = 0; j < small_files; ++j) { + std::string path = std::string(tmp) + "/small_ref_" + std::to_string(j); + std::ofstream f(path); + f << "reference data " << j; + } + + TimingResult result = run_benchmark( + [&]() { + for (int j = 0; j < small_files; ++j) { + std::string path = std::string(tmp) + "/small_ref_" + std::to_string(j); + int fd = open(path.c_str(), O_RDONLY); + if (fd >= 0) { + char buf[4096]; + read(fd, buf, sizeof(buf)); + close(fd); + } + } + }, + warmup, iterations + ); + std::cout << format_timing(result, "Small file read (" + std::to_string(small_files) + " files)", iterations) << "\n"; + + bench.cleanup(); + } + + // Cleanup + if (cleanup) { + fs::remove_all(tmp); + std::cout << "\nTest files cleaned up.\n"; + } else { + std::cout << "\nTest files kept in: " << tmp << "\n"; + } + + std::cout << "\n=== Data benchmark complete ===\n"; + return 0; +} \ No newline at end of file diff --git a/perf_tests/perf_delete.cpp b/perf_tests/perf_delete.cpp new file mode 100644 index 0000000000000000000000000000000000000000..26fc09b12cdf6b0392e72d7c608e8cb58cc9467c --- /dev/null +++ b/perf_tests/perf_delete.cpp @@ -0,0 +1,299 @@ +// Performance test: Delete operations +// Tests: file delete, directory delete, batch delete, deep tree delete +// Usage: ./perf_delete --mountdir /path/to/mount --iterations 100 --files 100 + +#include "perf_common.hpp" + +#include +#include +#include + +// ============================================================================ +// Benchmark functions +// ============================================================================ + +// --- Single file create+delete benchmark --- +struct FileCreateDeleteBenchmark { + std::string base_dir; + int num_files; + + FileCreateDeleteBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Batch delete benchmark --- +struct BatchDeleteBenchmark { + std::string base_dir; + int num_files; + + BatchDeleteBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Deep directory tree delete benchmark --- +struct TreeDeleteBenchmark { + std::string base_dir; + int depth; + int branches; + int leaves; + + TreeDeleteBenchmark(const std::string& dir, int d, int b, int l) + : base_dir(dir), depth(d), branches(b), leaves(l) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Rmdir benchmark --- +struct RmdirBenchmark { + std::string base_dir; + int num_dirs; + + RmdirBenchmark(const std::string& dir, int n) : base_dir(dir), num_dirs(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// ============================================================================ +// Usage and help +// ============================================================================ + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n" + << "\nOptions:\n" + << " --mountdir Mount point of GekkoFS (required)\n" + << " --iterations Number of benchmark iterations (default: 100)\n" + << " --files Number of files per test (default: 100)\n" + << " --depth Depth of tree (default: 4)\n" + << " --branches Branches per level (default: 4)\n" + << " --leaves Files per dir (default: 4)\n" + << " --warmup Number of warmup iterations (default: 5)\n" + << " --no-cleanup Don't cleanup test files after benchmark\n" + << " --help Show this help message\n"; +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char* argv[]) { + std::string mountdir = "/mnt/gekkofs"; + int iterations = 100; + int num_files = 100; + int tree_depth = 4; + int tree_branches = 4; + int tree_leaves = 4; + int warmup = 5; + bool cleanup = true; + + static struct option long_options[] = { + {"mountdir", required_argument, 0, 'm'}, + {"iterations", required_argument, 0, 'i'}, + {"files", required_argument, 0, 'f'}, + {"depth", required_argument, 0, 'd'}, + {"branches", required_argument, 0, 'b'}, + {"leaves", required_argument, 0, 'l'}, + {"warmup", required_argument, 0, 'w'}, + {"no-cleanup", no_argument, 0, 'n'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int opt; + while ((opt = getopt_long(argc, argv, "m:i:f:d:b:l:w:nh", long_options, nullptr)) != -1) { + switch (opt) { + case 'm': mountdir = optarg; break; + case 'i': iterations = static_cast(std::atoi(optarg)); break; + case 'f': num_files = static_cast(std::atoi(optarg)); break; + case 'd': tree_depth = static_cast(std::atoi(optarg)); break; + case 'b': tree_branches = static_cast(std::atoi(optarg)); break; + case 'l': tree_leaves = static_cast(std::atoi(optarg)); break; + case 'w': warmup = static_cast(std::atoi(optarg)); break; + case 'n': cleanup = false; break; + case 'h': print_usage(argv[0]); return 0; + default: print_usage(argv[0]); return 1; + } + } + + std::cout << "\n=== GekkoFS Delete Performance Test ===\n"; + std::cout << "Mount directory: " << mountdir << "\n"; + std::cout << "Iterations: " << iterations << "\n"; + std::cout << "Files per test: " << num_files << "\n"; + std::cout << "Tree config: depth=" << tree_depth << " branches=" << tree_branches + << " leaves=" << tree_leaves << "\n"; + std::cout << "Warmup: " << warmup << "\n\n"; + + std::string test_dir = mountdir + "/perf_delete_test_XXXXXX"; + char* tmp = mkdtemp(const_cast(test_dir.c_str())); + if (!tmp) { + std::cerr << "Error: Failed to create temp directory\n"; + return 1; + } + + std::cout << "Test directory: " << tmp << "\n\n"; + + // ---- 1. Single file create+delete benchmark ---- + { + std::cout << "--- File Create + Delete ---\n"; + FileCreateDeleteBenchmark bench(tmp, num_files); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + int iter_num2 = static_cast(rand()); + std::string iter_dir = std::string(tmp) + "/iter_" + std::to_string(iter_num2); + fs::create_directories(iter_dir); + for (int i = 0; i < num_files; ++i) { + std::string fpath = iter_dir + "/file_" + std::to_string(i); + std::ofstream f(fpath); + f << "test data " << i; + f.flush(); + } + for (int i = 0; i < num_files; ++i) { + std::string fpath = iter_dir + "/file_" + std::to_string(i); + remove(fpath.c_str()); + } + fs::remove(iter_dir); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "File create + delete (" + std::to_string(num_files) + " files)", static_cast(iterations)) << "\n"; + + bench.cleanup(); + } + + // ---- 2. Batch delete benchmark ---- + { + std::cout << "\n--- Batch Delete ---\n"; + BatchDeleteBenchmark bench(tmp, num_files); + + TimingResult result = run_benchmark( + [&]() { + int batch_num2 = static_cast(rand()); + std::string batch_dir = std::string(tmp) + "/batch_" + std::to_string(batch_num2); + fs::create_directories(batch_dir); + for (int i = 0; i < num_files; ++i) { + std::string fpath = batch_dir + "/file_" + std::to_string(i); + std::ofstream f(fpath); + f << "batch data " << i; + } + for (int i = 0; i < num_files; ++i) { + std::string fpath = batch_dir + "/file_" + std::to_string(i); + remove(fpath.c_str()); + } + fs::remove(batch_dir); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Batch delete (" + std::to_string(num_files) + " files)", static_cast(iterations)) << "\n"; + } + + // ---- 3. Tree delete benchmark ---- + { + std::cout << "\n--- Tree Create + Delete ---\n"; + TreeDeleteBenchmark bench(tmp, tree_depth, tree_branches, tree_leaves); + + int tree_num2 = static_cast(rand()); + std::string tree_name = "tree_" + std::to_string(tree_num2); + std::string tree_path = std::string(tmp) + "/" + tree_name; + fs::create_directories(tree_path); + + // Setup tree once + std::string current = tree_path; + for (int d = 0; d < tree_depth; ++d) { + for (int b = 0; b < tree_branches; ++b) { + current += "/dir_" + std::to_string(d) + "_" + std::to_string(b); + fs::create_directories(current); + for (int l = 0; l < tree_leaves; ++l) { + std::string fpath = current + "/file_" + std::to_string(l); + std::ofstream f(fpath); + f << "tree data"; + } + } + } + + TimingResult result = run_benchmark( + [&]() { + // Delete existing tree + fs::remove_all(tree_path); + // Recreate + fs::create_directories(tree_path); + current = tree_path; + for (int d = 0; d < tree_depth; ++d) { + for (int b = 0; b < tree_branches; ++b) { + current += "/dir_" + std::to_string(d) + "_" + std::to_string(b); + fs::create_directories(current); + for (int l = 0; l < tree_leaves; ++l) { + std::string fpath = current + "/file_" + std::to_string(l); + std::ofstream f(fpath); + f << "tree data"; + } + } + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Tree create+delete (depth=" + std::to_string(tree_depth) + + " branches=" + std::to_string(tree_branches) + " leaves=" + std::to_string(tree_leaves) + ")", + static_cast(iterations)) << "\n"; + } + + // ---- 4. Rmdir benchmark ---- + { + std::cout << "\n--- Rmdir Performance ---\n"; + RmdirBenchmark bench(tmp, num_files); + + TimingResult result = run_benchmark( + [&]() { + for (int i = 0; i < num_files; ++i) { + int rmdir_num2 = static_cast(rand()); + std::string dpath = std::string(tmp) + "/rmdir_" + std::to_string(rmdir_num2) + "_" + std::to_string(i); + mkdir(dpath.c_str(), 0755); + } + for (int i = 0; i < num_files; ++i) { + int rmdir_num3_val = static_cast(std::rand()); + std::string dpath = std::string(tmp) + "/rmdir_" + std::to_string(rmdir_num3_val) + "_" + std::to_string(i); + rmdir(dpath.c_str()); + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Rmdir (" + std::to_string(num_files) + " dirs)", static_cast(iterations)) << "\n"; + } + + // Cleanup + if (cleanup) { + fs::remove_all(tmp); + std::cout << "\nTest files cleaned up.\n"; + } else { + std::cout << "\nTest files kept in: " << tmp << "\n"; + } + + std::cout << "\n=== Delete benchmark complete ===\n"; + return 0; +} \ No newline at end of file diff --git a/perf_tests/perf_directory.cpp b/perf_tests/perf_directory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bf2196dc92b1989b90907f41c73b5b56e618c75e --- /dev/null +++ b/perf_tests/perf_directory.cpp @@ -0,0 +1,325 @@ +// Performance test: Directory operations +// Tests: directory create, list, iterate, deep tree create +// Usage: ./perf_directory --mountdir /path/to/mount --iterations 100 --warmup 5 + +#include "perf_common.hpp" + +#include +#include +#include +#include +#include + +// ============================================================================ +// Benchmark functions +// ============================================================================ + +// --- Directory create benchmark --- +struct DirCreateBenchmark { + std::string base_dir; + int num_dirs; + + DirCreateBenchmark(const std::string& dir, int n) : base_dir(dir), num_dirs(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Deep directory tree benchmark --- +struct DeepTreeBenchmark { + std::string base_dir; + int depth; + int branches; + + DeepTreeBenchmark(const std::string& dir, int d, int b) + : base_dir(dir), depth(d), branches(b) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Directory list benchmark --- +struct DirListBenchmark { + std::string base_dir; + int num_entries; + + DirListBenchmark(const std::string& dir, int n) : base_dir(dir), num_entries(n) {} + + void setup() { + fs::create_directories(base_dir); + // Create mixed file/directory entries + for (int i = 0; i < num_entries / 2; ++i) { + std::string dirpath = base_dir + "/dir_" + std::to_string(i); + fs::create_directories(dirpath); + + std::string filepath = base_dir + "/file_" + std::to_string(i); + std::ofstream f(filepath); + f << "test data"; + } + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// --- Directory iterate benchmark --- +struct DirIterateBenchmark { + std::string base_dir; + int num_dirs; + + DirIterateBenchmark(const std::string& dir, int n) : base_dir(dir), num_dirs(n) {} + + void setup() { + fs::create_directories(base_dir); + for (int i = 0; i < num_dirs; ++i) { + std::string dirpath = base_dir + "/dir_" + std::to_string(i); + fs::create_directories(dirpath); + } + } + + void cleanup() { + fs::remove_all(base_dir); + } +}; + +// ============================================================================ +// Usage and help +// ============================================================================ + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n" + << "\nOptions:\n" + << " --mountdir Mount point of GekkoFS (required)\n" + << " --iterations Number of benchmark iterations (default: 100)\n" + << " --dir-size Number of directories per test (default: 500)\n" + << " --depth Depth of tree (default: 5)\n" + << " --branches Branches per level (default: 4)\n" + << " --warmup Number of warmup iterations (default: 5)\n" + << " --no-cleanup Don't cleanup test files after benchmark\n" + << " --help Show this help message\n"; +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char* argv[]) { + std::string mountdir = "/mnt/gekkofs"; + int iterations = 100; + int dir_size = 500; + int tree_depth = 5; + int tree_branches = 4; + int warmup = 5; + bool cleanup = true; + + static struct option long_options[] = { + {"mountdir", required_argument, 0, 'm'}, + {"iterations", required_argument, 0, 'i'}, + {"dir-size", required_argument, 0, 'd'}, + {"depth", required_argument, 0, 'p'}, + {"branches", required_argument, 0, 'b'}, + {"warmup", required_argument, 0, 'w'}, + {"no-cleanup", no_argument, 0, 'n'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int opt; + while ((opt = getopt_long(argc, argv, "m:i:d:p:b:w:nh", long_options, nullptr)) != -1) { + switch (opt) { + case 'm': mountdir = optarg; break; + case 'i': iterations = static_cast(std::atoi(optarg)); break; + case 'd': dir_size = static_cast(std::atoi(optarg)); break; + case 'p': tree_depth = static_cast(std::atoi(optarg)); break; + case 'b': tree_branches = static_cast(std::atoi(optarg)); break; + case 'w': warmup = static_cast(std::atoi(optarg)); break; + case 'n': cleanup = false; break; + case 'h': print_usage(argv[0]); return 0; + default: print_usage(argv[0]); return 1; + } + } + + std::cout << "\n=== GekkoFS Directory Performance Test ===\n"; + std::cout << "Mount directory: " << mountdir << "\n"; + std::cout << "Iterations: " << iterations << "\n"; + std::cout << "Dir size: " << dir_size << "\n"; + std::cout << "Tree: depth=" << tree_depth << " branches=" << tree_branches << "\n"; + std::cout << "Warmup: " << warmup << "\n\n"; + + std::string test_dir = mountdir + "/perf_dir_test_XXXXXX"; + char* tmp = mkdtemp(const_cast(test_dir.c_str())); + if (!tmp) { + std::cerr << "Error: Failed to create temp directory\n"; + return 1; + } + + std::cout << "Test directory: " << tmp << "\n\n"; + + // ---- 1. Directory create benchmark ---- + { + std::cout << "--- Directory Create (single level) ---\n"; + DirCreateBenchmark bench(tmp, dir_size); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + int rand_val1 = static_cast(std::rand()); + std::string dname = std::string(tmp) + "/newdir_" + std::to_string(rand_val1); + mkdir(dname.c_str(), 0755); + rmdir(dname.c_str()); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Dir create (single)", static_cast(iterations)) << "\n"; + } + + // ---- 2. Deep tree create benchmark ---- + { + std::cout << "\n--- Deep Tree Create ---\n"; + DeepTreeBenchmark bench(tmp, tree_depth, tree_branches); + + int rand_val2 = static_cast(std::rand()); + std::string tree_name = "tree_" + std::to_string(rand_val2); + std::string tree_path = std::string(tmp) + "/" + tree_name; + fs::create_directories(tree_path); + + TimingResult result = run_benchmark( + [&]() { + // Delete existing + fs::remove_all(tree_path); + // Recreate + fs::create_directories(tree_path); + std::string current = tree_path; + for (int d = 0; d < tree_depth; ++d) { + for (int b = 0; b < tree_branches; ++b) { + current += "/dir_" + std::to_string(d) + "_" + std::to_string(b); + fs::create_directories(current); + } + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Deep tree create (depth=" + std::to_string(tree_depth) + + " branches=" + std::to_string(tree_branches) + ")", + static_cast(iterations)) << "\n"; + } + + // ---- 3. Directory list benchmark ---- + { + std::cout << "\n--- Directory List (readdir) ---\n"; + DirListBenchmark bench(tmp, dir_size / 2); + bench.setup(); + + // Count entries + int entry_count = 0; + DIR* dir = opendir(tmp); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { + entry_count++; + } + } + closedir(dir); + } + + TimingResult result = run_benchmark( + [&]() { + DIR* d = opendir(tmp); + if (d) { + struct dirent* entry; + int count = 0; + while ((entry = readdir(d)) != nullptr) { + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { + count++; + } + } + closedir(d); + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Dir list (" + std::to_string(entry_count) + " entries)", + static_cast(iterations)) << "\n"; + + bench.cleanup(); + } + + // ---- 4. Directory iterate benchmark ---- + { + std::cout << "\n--- Directory Iterate (stat all) ---\n"; + DirIterateBenchmark bench(tmp, dir_size); + bench.setup(); + + TimingResult result = run_benchmark( + [&]() { + DIR* d = opendir(tmp); + if (d) { + struct dirent* entry; + while ((entry = readdir(d)) != nullptr) { + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { + std::string fullpath = std::string(tmp) + "/" + entry->d_name; + struct stat st; + stat(fullpath.c_str(), &st); + } + } + closedir(d); + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Dir iterate + stat (" + std::to_string(dir_size) + " dirs)", + static_cast(iterations)) << "\n"; + + bench.cleanup(); + } + + // ---- 5. Nested directory create benchmark ---- + { + std::cout << "\n--- Nested Directory Create ---\n"; + + std::string nested_base = std::string(tmp) + "/nested_" + std::to_string(static_cast(std::rand())); + fs::create_directories(nested_base); + + TimingResult result = run_benchmark( + [&]() { + // Create nested structure + std::string nested = nested_base; + for (int i = 0; i < tree_depth; ++i) { + std::string sub = nested + "/level_" + std::to_string(i); + fs::create_directories(sub); + nested = sub; + } + // Cleanup + fs::remove_all(nested_base); + // Recreate + fs::create_directories(nested_base); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Nested create+delete (depth=" + std::to_string(tree_depth) + ")", + static_cast(iterations)) << "\n"; + } + + // Cleanup + if (cleanup) { + fs::remove_all(tmp); + std::cout << "\nTest files cleaned up.\n"; + } else { + std::cout << "\nTest files kept in: " << tmp << "\n"; + } + + std::cout << "\n=== Directory benchmark complete ===\n"; + return 0; +} \ No newline at end of file diff --git a/perf_tests/perf_find.cpp b/perf_tests/perf_find.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b5833cac57a125c805964e32bd953656a1df228c --- /dev/null +++ b/perf_tests/perf_find.cpp @@ -0,0 +1,313 @@ +// Performance test: Find/search operations +// Tests: find all, find by name, find by type (file/dir) +// Usage: ./perf_find --mountdir /path/to/mount --iterations 10 --dir-count 100 --files-per-dir 100 + +#include "perf_common.hpp" + +#include +#include +#include +#include +#include + +// Recursive directory walker +struct DirWalker { + std::function callback; + + void walk(const std::string& path) { + DIR* dir = opendir(path.c_str()); + if (!dir) return; + + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { + continue; + } + callback(path, entry, entry->d_type == DT_DIR); + } + closedir(dir); + } + + void walk_recursive(const std::string& path, int max_depth, int current_depth, + std::function cb) { + DIR* dir = opendir(path.c_str()); + if (!dir) return; + + std::vector subdirs; + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) { + continue; + } + std::string fullpath = path + "/" + entry->d_name; + cb(path, entry, entry->d_type == DT_DIR); + if (entry->d_type == DT_DIR && current_depth < max_depth) { + subdirs.push_back(fullpath); + } + } + closedir(dir); + + for (const auto& subdir : subdirs) { + walk_recursive(subdir, max_depth, current_depth + 1, cb); + } + } +}; + +// Benchmark setup helper +struct TestFileSystem { + std::string base_dir; + int num_dirs; + int dirs_deep; + int files_per_dir; + + TestFileSystem(const std::string& dir, int nd, int depth, int fpd) + : base_dir(dir), num_dirs(nd), dirs_deep(depth), files_per_dir(fpd) {} + + void create() { + fs::create_directories(base_dir); + // Create directory tree + for (int d = 0; d < num_dirs; ++d) { + std::string dirpath = base_dir + "/dir_" + std::to_string(d); + fs::create_directories(dirpath); + + // Create files in each directory + for (int f = 0; f < files_per_dir; ++f) { + std::string filepath = dirpath + "/file_" + std::to_string(f) + ".txt"; + std::ofstream fstream(filepath); + fstream << "test content for file " << f << " in dir " << d; + } + + // Create subdirs + for (int s = 0; s < dirs_deep; ++s) { + std::string subpath = dirpath + "/subdir_" + std::to_string(s); + fs::create_directories(subpath); + for (int f = 0; f < files_per_dir; ++f) { + std::string filepath = subpath + "/file_" + std::to_string(f) + ".txt"; + std::ofstream fstream(filepath); + fstream << "test content for file " << f << " in subdir " << s; + } + } + } + } + + void cleanup() { + fs::remove_all(base_dir); + } + + size_t count_entries() { + size_t count = 0; + DirWalker walker; + walker.walk_recursive(base_dir, 10, 0, [&count](const std::string&, const dirent*, bool) { + count++; + }); + return count; + } +}; + +// ============================================================================ +// Usage and help +// ============================================================================ + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n" + << "\nOptions:\n" + << " --mountdir Mount point of GekkoFS (required)\n" + << " --iterations Number of benchmark iterations (default: 10)\n" + << " --dir-count Number of top-level directories (default: 50)\n" + << " --files-per-dir Files per directory (default: 100)\n" + << " --subdirs Subdirectories per dir (default: 5)\n" + << " --warmup Number of warmup iterations (default: 2)\n" + << " --no-cleanup Don't cleanup test files after benchmark\n" + << " --help Show this help message\n"; +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char* argv[]) { + std::string mountdir = "/mnt/gekkofs"; + int iterations = 10; + int num_dirs = 50; + int files_per_dir = 100; + int subdirs = 5; + int warmup = 2; + bool cleanup = true; + + static struct option long_options[] = { + {"mountdir", required_argument, 0, 'm'}, + {"iterations", required_argument, 0, 'i'}, + {"dir-count", required_argument, 0, 'd'}, + {"files-per-dir", required_argument, 0, 'f'}, + {"subdirs", required_argument, 0, 's'}, + {"warmup", required_argument, 0, 'w'}, + {"no-cleanup", no_argument, 0, 'n'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int opt; + while ((opt = getopt_long(argc, argv, "m:i:d:f:s:w:nh", long_options, nullptr)) != -1) { + switch (opt) { + case 'm': mountdir = optarg; break; + case 'i': iterations = static_cast(std::atoi(optarg)); break; + case 'd': num_dirs = static_cast(std::atoi(optarg)); break; + case 'f': files_per_dir = static_cast(std::atoi(optarg)); break; + case 's': subdirs = static_cast(std::atoi(optarg)); break; + case 'w': warmup = static_cast(std::atoi(optarg)); break; + case 'n': cleanup = false; break; + case 'h': print_usage(argv[0]); return 0; + default: print_usage(argv[0]); return 1; + } + } + + std::cout << "\n=== GekkoFS Find/Search Performance Test ===\n"; + std::cout << "Mount directory: " << mountdir << "\n"; + std::cout << "Iterations: " << iterations << "\n"; + std::cout << "Directories: " << num_dirs << ", Subdirs: " << subdirs << "\n"; + std::cout << "Files per directory: " << files_per_dir << "\n"; + std::cout << "Warmup: " << warmup << "\n\n"; + + std::string test_dir = mountdir + "/perf_find_test_XXXXXX"; + char* tmp = mkdtemp(const_cast(test_dir.c_str())); + if (!tmp) { + std::cerr << "Error: Failed to create temp directory\n"; + return 1; + } + + std::cout << "Test directory: " << tmp << "\n\n"; + + // Setup test filesystem + TestFileSystem tfs(tmp, num_dirs, subdirs, files_per_dir); + tfs.create(); + + size_t total_entries = tfs.count_entries(); + std::cout << "Total entries created: " << total_entries << "\n\n"; + + // ---- 1. Full directory tree traversal ---- + { + std::cout << "--- Full Tree Traversal ---\n"; + + TimingResult result = run_benchmark( + [&]() { + DirWalker walker; + std::size_t count = 0; + walker.walk_recursive(tmp, 10, 0, [&count](const std::string&, const dirent*, bool) { + count++; + }); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Full tree traversal (" + std::to_string(total_entries) + " entries)", + static_cast(iterations)) << "\n"; + } + + // ---- 2. Find by pattern (simple name search) ---- + { + std::cout << "\n--- Find by Pattern (file_*.txt) ---\n"; + + TimingResult result = run_benchmark( + [&]() { + DirWalker walker; + std::size_t count = 0; + walker.walk_recursive(tmp, 10, 0, [&count](const std::string&, const dirent* entry, bool) { + std::string name = entry->d_name; + if (name.find("file_") == 0 && name.find(".txt") != std::string::npos) { + count++; + } + }); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Find by pattern (file_*.txt)", + static_cast(iterations)) << "\n"; + } + + // ---- 3. Find directories only ---- + { + std::cout << "\n--- Find Directories Only ---\n"; + + TimingResult result = run_benchmark( + [&]() { + DirWalker walker; + std::size_t count = 0; + walker.walk_recursive(tmp, 10, 0, [&count](const std::string&, const dirent* entry, bool is_dir) { + if (is_dir) { + count++; + } + }); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Find directories only", + static_cast(iterations)) << "\n"; + } + + // ---- 4. Find files only ---- + { + std::cout << "\n--- Find Files Only ---\n"; + + TimingResult result = run_benchmark( + [&]() { + DirWalker walker; + std::size_t count = 0; + walker.walk_recursive(tmp, 10, 0, [&count](const std::string&, const dirent* entry, bool is_dir) { + if (!is_dir) { + count++; + } + }); + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Find files only", + static_cast(iterations)) << "\n"; + } + + // ---- 5. Shallow traversal (single level readdir) ---- + { + std::cout << "\n--- Single Level Readdir ---\n"; + + // Count entries in first level + int first_level_count = 0; + DIR* dir = opendir(tmp); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) { + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { + first_level_count++; + } + } + closedir(dir); + } + + TimingResult result = run_benchmark( + [&]() { + DIR* d = opendir(tmp); + if (d) { + struct dirent* entry; + int count = 0; + while ((entry = readdir(d)) != nullptr) { + if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) { + count++; + } + } + closedir(d); + } + }, + static_cast(warmup), static_cast(iterations) + ); + std::cout << format_timing(result, "Single level readdir (" + std::to_string(first_level_count) + " entries)", + static_cast(iterations)) << "\n"; + } + + // Cleanup + if (cleanup) { + tfs.cleanup(); + std::cout << "\nTest files cleaned up.\n"; + } else { + std::cout << "\nTest files kept in: " << tmp << "\n"; + } + + std::cout << "\n=== Find benchmark complete ===\n"; + return 0; +} \ No newline at end of file diff --git a/perf_tests/perf_metadata.cpp b/perf_tests/perf_metadata.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bfc1fc1a3eba47c0e4163ccf33613c487ceea510 --- /dev/null +++ b/perf_tests/perf_metadata.cpp @@ -0,0 +1,467 @@ +// Performance test: Metadata operations +// Tests: mkdir, stat, list, symlink, rename performance +// Usage: ./perf_metadata --mountdir /path/to/mount --iterations 1000 --files 100 + +#include "perf_common.hpp" + +#include +#include +#include + +#include +#include +#include + +// ============================================================================ +// Benchmark functions +// ============================================================================ + +// --- mkdir benchmark --- +struct MkdirBenchmark { + std::string base_dir; + int num_files; + + MkdirBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + auto benchmark = [&]() { + std::string dir = base_dir + "/" + std::to_string(num_files); + int ret = mkdir(dir.c_str(), 0755); + (void)ret; + }; + + // Each iteration creates 1 dir + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + benchmark(); + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } + + TimingResult batch_run(size_t batch_count) { + // Create batch_count directories in each iteration + auto benchmark = [&]() { + for (int i = 0; i < num_files; ++i) { + std::string dir = base_dir + "/batch_" + std::to_string(i); + int ret = mkdir(dir.c_str(), 0755); + (void)ret; + } + }; + + std::vector intervals_ms; + intervals_ms.reserve(batch_count); + + for (size_t i = 0; i < batch_count; ++i) { + auto start = Timer::now(); + benchmark(); + intervals_ms.push_back(Timer::elapsed_ms(start)); + // Cleanup after each batch + for (int j = 0; j < num_files; ++j) { + fs::remove_all(base_dir + "/batch_" + std::to_string(j)); + } + } + + return finalize_timing(intervals_ms); + } +}; + +// --- stat benchmark --- +struct StatBenchmark { + std::string base_dir; + int num_files; + + StatBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + for (int i = 0; i < num_files; ++i) { + std::string path = base_dir + "/file_" + std::to_string(i); + std::ofstream f(path); + f << "test data for stat benchmark " << i; + } + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t iter = 0; iter < iterations; ++iter) { + for (int i = 0; i < num_files; ++i) { + auto start = Timer::now(); + struct stat st; + std::string path = base_dir + "/file_" + std::to_string(i); + int ret = stat(path.c_str(), &st); + (void)ret; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + } + + return finalize_timing(intervals_ms); + } +}; + +// --- readdir benchmark --- +struct ReaddirBenchmark { + std::string base_dir; + int num_files; + + ReaddirBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + for (int i = 0; i < num_files; ++i) { + std::string path = base_dir + "/file_" + std::to_string(i); + std::ofstream f(path); + f << "test data for readdir benchmark " << i; + } + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + DIR* dir = opendir(base_dir.c_str()); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) {} + closedir(dir); + } + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// --- symlink benchmark --- +struct SymlinkBenchmark { + std::string base_dir; + int num_files; + + SymlinkBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + for (int i = 0; i < num_files; ++i) { + std::string path = base_dir + "/target_" + std::to_string(i); + std::ofstream f(path); + f << "target data"; + } + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult create(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + std::string link = base_dir + "/link_" + std::to_string(i % num_files); + int ret = symlink((base_dir + "/target_" + std::to_string(i % num_files)).c_str(), link.c_str()); + (void)ret; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } + + TimingResult resolve(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + char buf[4096]; + ssize_t len = readlink((base_dir + "/link_" + std::to_string(i % num_files)).c_str(), buf, sizeof(buf) - 1); + (void)len; + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } + + TimingResult batch_create(size_t batch_count) { + std::vector intervals_ms; + intervals_ms.reserve(batch_count); + + for (size_t b = 0; b < batch_count; ++b) { + auto start = Timer::now(); + for (int i = 0; i < num_files; ++i) { + std::string link = base_dir + "/batch_link_" + std::to_string(b) + "_" + std::to_string(i); + int ret = symlink((base_dir + "/target_" + std::to_string(i)).c_str(), link.c_str()); + (void)ret; + } + intervals_ms.push_back(Timer::elapsed_ms(start)); + // Cleanup + for (int i = 0; i < num_files; ++i) { + fs::remove(base_dir + "/batch_link_" + std::to_string(b) + "_" + std::to_string(i)); + } + } + + return finalize_timing(intervals_ms); + } +}; + +// --- rename benchmark --- +struct RenameBenchmark { + std::string base_dir; + int num_files; + + RenameBenchmark(const std::string& dir, int n) : base_dir(dir), num_files(n) {} + + void setup() { + fs::create_directories(base_dir); + for (int i = 0; i < num_files; ++i) { + std::string path = base_dir + "/orig_" + std::to_string(i); + std::ofstream f(path); + f << "rename test data"; + } + } + + void cleanup() { + fs::remove_all(base_dir); + } + + TimingResult run(size_t iterations) { + std::vector intervals_ms; + intervals_ms.reserve(iterations); + + for (size_t i = 0; i < iterations; ++i) { + auto start = Timer::now(); + // Rename back and forth + std::string temp = base_dir + "/temp_" + std::to_string(i); + int ret = rename((base_dir + "/orig_" + std::to_string(i % num_files)).c_str(), temp.c_str()); + if (ret == 0) { + rename(temp.c_str(), (base_dir + "/orig_" + std::to_string(i % num_files)).c_str()); + } + intervals_ms.push_back(Timer::elapsed_ms(start)); + } + + return finalize_timing(intervals_ms); + } +}; + +// ============================================================================ +// Usage and help +// ============================================================================ + +void print_usage(const char* prog) { + std::cout << "Usage: " << prog << " [options]\n" + << "\nOptions:\n" + << " --mountdir Mount point of GekkoFS (required)\n" + << " --iterations Number of benchmark iterations (default: 500)\n" + << " --files Number of files to create (default: 100)\n" + << " --warmup Number of warmup iterations (default: 10)\n" + << " --batch Use batch mode for mkdir test\n" + << " --no-cleanup Don't cleanup test files after benchmark\n" + << " --help Show this help message\n"; +} + +// ============================================================================ +// Main +// ============================================================================ + +int main(int argc, char* argv[]) { + std::string mountdir = "/mnt/gekkofs"; + int iterations = 500; + int num_files = 100; + int warmup = 10; + bool batch_mode = false; + bool cleanup = true; + + static struct option long_options[] = { + {"mountdir", required_argument, 0, 'm'}, + {"iterations", required_argument, 0, 'i'}, + {"files", required_argument, 0, 'f'}, + {"warmup", required_argument, 0, 'w'}, + {"batch", no_argument, 0, 'b'}, + {"no-cleanup", no_argument, 0, 'n'}, + {"help", no_argument, 0, 'h'}, + {0, 0, 0, 0} + }; + + int opt; + while ((opt = getopt_long(argc, argv, "m:i:f:w:bnh", long_options, nullptr)) != -1) { + switch (opt) { + case 'm': mountdir = optarg; break; + case 'i': iterations = std::atoi(optarg); break; + case 'f': num_files = std::atoi(optarg); break; + case 'w': warmup = std::atoi(optarg); break; + case 'b': batch_mode = true; break; + case 'n': cleanup = false; break; + case 'h': print_usage(argv[0]); return 0; + default: print_usage(argv[0]); return 1; + } + } + + std::cout << "\n=== GekkoFS Metadata Performance Test ===\n"; + std::cout << "Mount directory: " << mountdir << "\n"; + std::cout << "Iterations: " << iterations << "\n"; + std::cout << "Files per test: " << num_files << "\n"; + std::cout << "Warmup: " << warmup << "\n\n"; + + // Create temp test directory + std::string test_dir = mountdir + "/perf_metadata_test_XXXXXX"; + char* tmp = mkdtemp(const_cast(test_dir.c_str())); + if (!tmp) { + std::cerr << "Error: Failed to create temp directory\n"; + return 1; + } + + std::cout << "Test directory: " << tmp << "\n\n"; + + // ---- 1. Mkdir benchmark ---- + { + MkdirBenchmark mkdir_bench(tmp, num_files); + mkdir_bench.setup(); + + std::cout << "--- Mkdir Performance ---\n"; + TimingResult single = run_benchmark( + [&]() { + std::string d = std::string(tmp) + "/single_" + std::to_string(rand()); + mkdir(d.c_str(), 0755); + }, + warmup, iterations + ); + std::cout << format_timing(single, "Single mkdir (individual)", iterations) << "\n"; + + if (batch_mode) { + TimingResult batch = mkdir_bench.batch_run(iterations); + std::cout << format_timing(batch, "Batch mkdir (" + std::to_string(num_files) + " files/batch)", iterations) << "\n"; + } + + mkdir_bench.cleanup(); + } + + // ---- 2. Stat benchmark ---- + { + StatBenchmark stat_bench(tmp, num_files); + stat_bench.setup(); + + std::cout << "--- Stat Performance ---\n"; + TimingResult result = run_benchmark( + [&]() { + for (int i = 0; i < num_files; ++i) { + struct stat st; + stat((std::string(tmp) + "/file_" + std::to_string(i)).c_str(), &st); + } + }, + warmup, iterations + ); + std::cout << format_timing(result, "Stat all files (" + std::to_string(num_files) + " files)", + iterations * num_files) << "\n"; + + stat_bench.cleanup(); + } + + // ---- 3. Readdir benchmark ---- + { + ReaddirBenchmark readdir_bench(tmp, num_files); + readdir_bench.setup(); + + std::cout << "--- Readdir Performance ---\n"; + TimingResult result = run_benchmark( + [&]() { + DIR* dir = opendir(tmp); + if (dir) { + struct dirent* entry; + while ((entry = readdir(dir)) != nullptr) {} + closedir(dir); + } + }, + warmup, iterations + ); + std::cout << format_timing(result, "Readdir (" + std::to_string(num_files) + " entries)", iterations) << "\n"; + + readdir_bench.cleanup(); + } + + // ---- 4. Symlink benchmark ---- + { + SymlinkBenchmark symlink_bench(tmp, num_files); + symlink_bench.setup(); + + std::cout << "--- Symlink Create Performance ---\n"; + TimingResult create_result = run_benchmark( + [&]() { + std::string link = std::string(tmp) + "/link_" + std::to_string(rand()); + symlink((std::string(tmp) + "/target_0").c_str(), link.c_str()); + }, + warmup, iterations + ); + std::cout << format_timing(create_result, "Symlink create (individual)", iterations) << "\n"; + + if (batch_mode) { + TimingResult batch = symlink_bench.batch_create(iterations); + std::cout << format_timing(batch, "Symlink batch create (" + std::to_string(num_files) + " files/batch)", iterations) << "\n"; + } + + std::cout << "\n--- Symlink Resolve Performance ---\n"; + TimingResult resolve_result = run_benchmark( + [&]() { + char buf[4096]; + readlink((std::string(tmp) + "/link_0").c_str(), buf, sizeof(buf) - 1); + }, + warmup, iterations + ); + std::cout << format_timing(resolve_result, "Symlink resolve (readlink)", iterations) << "\n"; + + symlink_bench.cleanup(); + } + + // ---- 5. Rename benchmark ---- + { + RenameBenchmark rename_bench(tmp, num_files); + rename_bench.setup(); + + std::cout << "--- Rename Performance ---\n"; + TimingResult result = run_benchmark( + [&]() { + std::string temp = std::string(tmp) + "/temp_" + std::to_string(rand()); + rename((std::string(tmp) + "/orig_0").c_str(), temp.c_str()); + rename(temp.c_str(), (std::string(tmp) + "/orig_0").c_str()); + }, + warmup, iterations + ); + std::cout << format_timing(result, "Rename (back and forth)", iterations) << "\n"; + + rename_bench.cleanup(); + } + + // Cleanup + if (cleanup) { + fs::remove_all(tmp); + std::cout << "\nTest files cleaned up.\n"; + } else { + std::cout << "\nTest files kept in: " << tmp << "\n"; + } + + std::cout << "\n=== Metadata benchmark complete ===\n"; + return 0; +} \ No newline at end of file diff --git a/perf_tests/run_benchmarks.sh b/perf_tests/run_benchmarks.sh new file mode 100755 index 0000000000000000000000000000000000000000..fcedc780ebad8e2063002af4bdb73153fbc0f774 --- /dev/null +++ b/perf_tests/run_benchmarks.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# Run all GekkoFS performance benchmarks +# Usage: +# LD_PRELOAD mode: ./run_benchmarks.sh /path/to/gkfs_syscall_intercept.so [mount_point] +# FUSE mode: ./run_benchmarks.sh [mount_point] + +set -e + +# Configuration +if [ $# -ge 2 ] && [[ "$1" == *.so ]]; then + LD_PRELOAD_PATH="$1" + MOUNT_POINT="${2:-/mnt/gekkofs}" + USE_LD_PRELOAD=1 +elif [ $# -ge 1 ]; then + MOUNT_POINT="$1" + USE_LD_PRELOAD=0 +else + MOUNT_POINT="/mnt/gekkofs" + USE_LD_PRELOAD=0 +fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Output directory for results +RESULTS_DIR="${SCRIPT_DIR}/results" +mkdir -p "${RESULTS_DIR}" + +echo -e "${BLUE}============================================${NC}" +echo -e "${BLUE} GekkoFS Performance Benchmark Suite${NC}" +echo -e "${BLUE}============================================${NC}" +echo "" +echo -e "Mode: ${YELLOW}$([ "$USE_LD_PRELOAD" == "1" ] && echo "LD_PRELOAD" || echo "FUSE")${NC}" +echo -e "Mount point: ${YELLOW}${MOUNT_POINT}${NC}" +echo -e "Results directory: ${YELLOW}${RESULTS_DIR}${NC}" +[ "$USE_LD_PRELOAD" == "1" ] && echo -e "LD_PRELOAD: ${YELLOW}${LD_PRELOAD_PATH}${NC}" +echo "" + +# Check if mount point exists +if [ ! -d "${MOUNT_POINT}" ]; then + echo -e "${RED}Error: Mount point ${MOUNT_POINT} does not exist${NC}" + echo "Usage: $0 [gkfs_syscall_intercept.so] [mount_point]" + exit 1 +fi + +# Check if writable +if ! touch "${MOUNT_POINT}/.perf_test_write" 2>/dev/null; then + echo -e "${RED}Error: Mount point ${MOUNT_POINT} is not writable${NC}" + exit 1 +fi +rm -f "${MOUNT_POINT}/.perf_test_write" + +echo -e "${GREEN}✓ Mount point is accessible and writable${NC}" +echo "" + +# Function to run a benchmark and save results +run_benchmark() { + local bench_name="$1" + local bench_cmd="$2" + local extra_args="${3:-}" + local timestamp + timestamp=$(date +%Y%m%d_%H%M%S) + local result_file="${RESULTS_DIR}/${bench_name}_${timestamp}.txt" + + echo -e "${BLUE}--------------------------------------------${NC}" + echo -e "${GREEN}Running: ${bench_name}${NC}" + echo -e "${BLUE}--------------------------------------------${NC}" + + if command -v "${bench_cmd}" >/dev/null 2>&1; then + echo "Running benchmark... Output saved to: ${result_file}" + echo "" + + if [ "$USE_LD_PRELOAD" == "1" ]; then + LD_PRELOAD="${LD_PRELOAD_PATH}" "${bench_cmd}" --mountdir "${MOUNT_POINT}" ${extra_args} 2>&1 | tee "${result_file}" + else + "${bench_cmd}" --mountdir "${MOUNT_POINT}" ${extra_args} 2>&1 | tee "${result_file}" + fi + + echo "" + echo -e "${GREEN}✓ Benchmark complete. Results saved to: ${result_file}${NC}" + else + echo -e "${RED}Error: Benchmark executable not found in PATH: ${bench_cmd}${NC}" + return 1 + fi + echo "" +} + +# Run each benchmark +echo -e "${YELLOW}Starting benchmarks...${NC}" +echo "" + +# 1. Metadata benchmarks +run_benchmark "metadata" "perf_metadata" "--iterations 50 --warmup 3" + +# 2. Data benchmarks +run_benchmark "data" "perf_data" "--iterations 50 --warmup 3 --filesize 1048576" + +# 3. Delete benchmarks +run_benchmark "delete" "perf_delete" "--iterations 50 --warmup 3 --files 100" + +# 4. Find benchmarks +run_benchmark "find" "perf_find" "--iterations 5 --warmup 2 --dir-count 50 --files-per-dir 100 --subdirs 5" + +# 5. Directory benchmarks +run_benchmark "directory" "perf_directory" "--iterations 50 --warmup 3" + +echo "" +echo -e "${BLUE}============================================${NC}" +echo -e "${GREEN} All benchmarks complete!${NC}" +echo -e "${BLUE}============================================${NC}" +echo "" +echo -e "Results saved to: ${YELLOW}${RESULTS_DIR}${NC}" +echo "" + +# Show latest results summary +LATEST_FILE=$(ls -t "${RESULTS_DIR}"/*.txt 2>/dev/null | head -1) +if [ -n "${LATEST_FILE}" ]; then + echo -e "${YELLOW}Latest results:${NC}" + cat "${LATEST_FILE}" +fi \ No newline at end of file diff --git a/scripts/ci/.coverage-exclusions b/scripts/ci/.coverage-exclusions index 4519dbd03f5f038366b20d8de0027a28eaa69f9e..d1877443b602ef59e0f54b5b6ba2dae2f8a7d013 100644 --- a/scripts/ci/.coverage-exclusions +++ b/scripts/ci/.coverage-exclusions @@ -1,4 +1,5 @@ /usr/include/* [[ROOT_DIR]]/external/* [[ROOT_DIR]]/tests/* +[[ROOT_DIR]]/perf_tests/* [[BUILD_DIR]]/* diff --git a/scripts/dev/coverage.py b/scripts/dev/coverage.py index c81a697b0efd6c9c7a38b4d7deca4b5030083e1e..34afec62c257b2594045ba7bd0bd9672d2d0d608 100755 --- a/scripts/dev/coverage.py +++ b/scripts/dev/coverage.py @@ -166,12 +166,16 @@ def capture(args): sources_directory = args.sources_directory.resolve() output_file = args.output_file.resolve() + if args.exclusion_patterns is None: + args.exclusion_patterns = [] + args.exclusion_patterns.extend(["*/*.hpp", "*/*.h", f"{root_directory}/*"]) + pipeline = CommandPipeline() pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch,negative", + "--ignore-errors","inconsistent,inconsistent,corrupt,corrupt,mismatch,mismatch,negative,negative,version,version,empty,empty,unused,unused,source,source,missing,missing,count,count,gcov,gcov", "--capture", "--quiet" if args.verbosity > 3 else "", "--initial" if args.initial else "", @@ -183,7 +187,7 @@ def capture(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch,negative", + "--ignore-errors","inconsistent,inconsistent,corrupt,corrupt,mismatch,mismatch,negative,negative,version,version,empty,empty,unused,unused,source,source,missing,missing,count,count,gcov,gcov", "--quiet" if args.verbosity > 3 else "", "--remove=$INPUT", *args.exclusion_patterns, @@ -222,7 +226,7 @@ def merge(args): pipeline.append([ "lcov", "--rc", "branch_coverage=1", - "--ignore-errors","inconsistent,mismatch,negative", + "--ignore-errors","inconsistent,inconsistent,corrupt,corrupt,mismatch,mismatch,negative,negative,version,version,empty,empty,unused,unused,source,source,missing,missing,count,count,gcov,gcov", "--quiet" if args.verbosity > 3 else "", *(f"--add-tracefile={t}" for t in tracefiles), "--output-file=$OUTPUT"]) @@ -257,7 +261,8 @@ def html_report(args): "--rc", "branch_coverage=1", "--quiet" if args.verbosity > 3 else "", "--show-details", - "--ignore-errors", "inconsistent,corrupt", + "--filter", "missing", + "--ignore-errors", "inconsistent,inconsistent,corrupt,corrupt,mismatch,mismatch,negative,negative,version,version,empty,empty,unused,unused,source,source,missing,missing,count,count,gcov,gcov", "--legend", "--demangle-cpp", "--frames", diff --git a/src/client/gkfs_metadata.cpp b/src/client/gkfs_metadata.cpp index 3287c6365976c716d0b8f02b86c2dd8e70490774..2e1209f4f68fbc0efd68cd077388a54263d61da8 100644 --- a/src/client/gkfs_metadata.cpp +++ b/src/client/gkfs_metadata.cpp @@ -470,61 +470,33 @@ gkfs_libcremove(const std::string& path) { * @return 0 on success, -1 on failure */ int -gkfs_remove(const std::string& path) { - auto md = gkfs::utils::get_metadata(path); - if(!md) { - return -1; - } - - if(gkfs::config::metadata::rename_support) { - - if(S_ISDIR(md->mode())) { - LOG(ERROR, "Cannot remove directory '{}'", path); - errno = EISDIR; - return -1; - } - - if(md.value().blocks() == -1) { - errno = ENOENT; - return -1; - } else { - if(!md->is_link()) { - if(!md.value().target_path().empty()) { - auto md_ = - gkfs::utils::get_metadata(md.value().target_path()); - std::string new_path = md.value().target_path(); - while(!md.value().target_path().empty() and - md.value().blocks() != -1) { - new_path = md.value().target_path(); - md = gkfs::utils::get_metadata(md.value().target_path(), - false); - if(!md) { - return -1; - } - } - auto err = gkfs::rpc::forward_remove( - new_path, false, CTX->get_replicas(), md->size()); - if(err) { - errno = err; - return -1; - } - } - } - } - } - +gkfs_remove(const std::string& path, bool is_rename_stub) { int err = 0; + int64_t size = 0; + uint32_t mode = 0; + std::string target_path; + if(gkfs::config::proxy::fwd_remove && CTX->use_proxy()) { err = gkfs::rpc::forward_remove_proxy(path, false); } else { - err = gkfs::rpc::forward_remove(path, false, CTX->get_replicas(), - md->size()); + err = gkfs::rpc::forward_remove(path, false, CTX->get_replicas(), size, + mode, target_path, is_rename_stub); } if(err) { errno = err; return -1; } + if(gkfs::config::metadata::rename_support && !is_rename_stub) { + if(!target_path.empty() && !S_ISLNK(mode)) { + // It was a rename link! We recursively remove the target path. + auto rm_err = gkfs_remove(target_path, true); + if(rm_err) { + return -1; + } + } + } + return 0; } diff --git a/src/client/logging.cpp b/src/client/logging.cpp index 284ca8085909e32081ab187e1d4d4335aa9768b6..f690d1e0a9aadc9305e9c1ca28829218ff7083ce 100644 --- a/src/client/logging.cpp +++ b/src/client/logging.cpp @@ -176,7 +176,7 @@ process_log_options(const std::string& gkfs_debug) { #ifndef GKFS_ENABLE_LOGGING (void) gkfs_debug; - logger::log_message(stdout, "warning: logging options ignored: " + logger::log_message(stderr, "warning: logging options ignored: " "logging support was disabled in this build"); return log::none; @@ -204,7 +204,7 @@ process_log_options(const std::string& gkfs_debug) { } if(!is_known) { - logger::log_message(stdout, + logger::log_message(stderr, "warning: logging option '{}' unknown; " "try {}=help", t, gkfs::env::LOG); @@ -212,7 +212,7 @@ process_log_options(const std::string& gkfs_debug) { } if(!!(dm & log::help)) { - logger::log_message(stdout, + logger::log_message(stderr, "Valid options for the {} " "environment variable are:\n", gkfs::env::LOG); @@ -221,21 +221,21 @@ process_log_options(const std::string& gkfs_debug) { for(const auto& opt : debug_opts) { const auto padding = max_debug_opt_length - opt.length_ + 2; - logger::log_message(stdout, " {}{:>{}}{}", opt.name_, "", padding, + logger::log_message(stderr, " {}{:>{}}{}", opt.name_, "", padding, opt.help_text_[0]); for(auto i = 1lu; i < max_help_text_rows; ++i) { if(opt.help_text_[i][0] != 0) { - logger::log_message(stdout, " {:>{}}{}", "", + logger::log_message(stderr, " {:>{}}{}", "", max_debug_opt_length + 2, opt.help_text_[i]); } } - logger::log_message(stdout, ""); + logger::log_message(stderr, ""); } - logger::log_message(stdout, + logger::log_message(stderr, "\n" "To direct the logging output into a file " "instead of standard output\n" @@ -265,7 +265,7 @@ process_log_filter(const std::string& log_filter) { const auto sc = syscall::lookup_by_name(t); if(std::strcmp(sc.name(), "unknown_syscall") == 0) { - logger::log_message(stdout, + logger::log_message(stderr, "warning: system call '{}' unknown; " "will not filter", t); diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index 36ab01620097390426a7f69a5007aa899c0f2827..baa69d8f60933bdac1aff84ba549535c07f9653b 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -173,8 +173,18 @@ forward_stat(const std::string& path, string& attr, string& inline_data, } int -forward_remove(const std::string& path, bool rm_dir, int8_t num_copies, +forward_remove(const std::string& path, bool rm_dir, const int8_t num_copies, int64_t size) { + uint32_t mode = 0; + std::string target_path; + return forward_remove(path, rm_dir, num_copies, size, mode, target_path, + false); +} + +int +forward_remove(const std::string& path, bool rm_dir, const int8_t num_copies, + int64_t& size, uint32_t& mode, std::string& target_path, + bool is_rename_stub) { int err = 0; // Step 1: Remove metadata from the responsible server(s) @@ -186,6 +196,7 @@ forward_remove(const std::string& path, bool rm_dir, int8_t num_copies, gkfs::rpc::rpc_rm_node_in_t in; in.path = path; in.rm_dir = rm_dir; + in.is_rename_stub = is_rename_stub; auto out = gkfs::rpc::forward_call( CTX->rpc_engine(), endp, gkfs::rpc::tag::remove_metadata, in, @@ -193,6 +204,10 @@ forward_remove(const std::string& path, bool rm_dir, int8_t num_copies, if(out.err != 0) { err = out.err; + } else { + size = out.size; + mode = out.mode; + target_path = out.target_path; } } @@ -217,6 +232,7 @@ forward_remove(const std::string& path, bool rm_dir, int8_t num_copies, rm_in.path = path; rm_in.rm_dir = true; // tells daemon to remove the whole chunk directory + rm_in.is_rename_stub = false; auto chunk_size = gkfs::config::rpc::chunksize; auto num_chunks = size / chunk_size; diff --git a/src/client/rpc/forward_metadata_proxy.cpp b/src/client/rpc/forward_metadata_proxy.cpp index 4833a6bf7356661c9d3fd3c977ae29016826cef9..b4148db723879d638090ca2d9865e319bd789db6 100644 --- a/src/client/rpc/forward_metadata_proxy.cpp +++ b/src/client/rpc/forward_metadata_proxy.cpp @@ -79,6 +79,7 @@ forward_remove_proxy(const std::string& path, bool rm_dir) { gkfs::rpc::rpc_rm_node_in_t in; in.path = path; in.rm_dir = rm_dir; + in.is_rename_stub = false; auto out = gkfs::rpc::forward_call( CTX->ipc_engine(), endp, gkfs::rpc::tag::client_proxy_remove, in, diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 471145fe025a787903a61133938308114e0c819b..c1cc8124f96102890a3a2b427af07ca2ffb0e8a8 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -49,6 +49,7 @@ #include #include #include +#include #include #include #include @@ -435,7 +436,8 @@ std::vector> RocksDBBackend::get_dirents_impl(const std::string& dir) const { auto root_path = dir; rocksdb::ReadOptions ropts; - auto it = db_->NewIterator(ropts); + ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer + std::unique_ptr it(db_->NewIterator(ropts)); std::vector> entries; for(it->Seek(root_path); it->Valid() && it->key().starts_with(root_path); @@ -499,7 +501,8 @@ RocksDBBackend::get_dirents_extended_impl(const std::string& dir, size_t max_entries) const { auto root_path = dir; rocksdb::ReadOptions ropts; - auto it = db_->NewIterator(ropts); + ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer + std::unique_ptr it(db_->NewIterator(ropts)); std::vector> entries; @@ -586,7 +589,8 @@ RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, size_t max_entries) const { auto root_path = dir; rocksdb::ReadOptions ropts; - auto it = db_->NewIterator(ropts); + ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer + std::unique_ptr it(db_->NewIterator(ropts)); std::vector> entries; @@ -676,7 +680,8 @@ RocksDBBackend::get_dirents_filtered_impl( const int64_t filter_ctime, bool count_only, size_t max_entries) const { auto root_path = dir; rocksdb::ReadOptions ropts; - auto it = db_->NewIterator(ropts); + ropts.readahead_size = 2 * 1024 * 1024; // 2MB prefetch buffer + std::unique_ptr it(db_->NewIterator(ropts)); std::vector> entries; std::string last_scanned_key; diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index cf042e47fb817022acf87cebcfd940d44c73954e..98e05c2a2fc001b8c3e95589131de1042715312f 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -284,6 +284,11 @@ rpc_srv_remove_metadata(const tl::request& req, [](const gkfs::rpc::rpc_rm_node_in_t& in, gkfs::rpc::rpc_rm_metadata_out_t& out) { auto md = gkfs::metadata::get(in.path); + if(gkfs::config::metadata::rename_support && + md.blocks() == -1 && !in.is_rename_stub) { + out.err = ENOENT; + return; + } if(S_ISDIR(md.mode()) && !in.rm_dir) { out.err = EISDIR; return; @@ -295,6 +300,7 @@ rpc_srv_remove_metadata(const tl::request& req, out.err = 0; out.mode = md.mode(); out.size = S_ISDIR(md.mode()) ? 0 : md.size(); + out.target_path = md.target_path(); // if file, remove metadata and also return mode and size if constexpr(gkfs::config::metadata::implicit_data_removal) { if(S_ISREG(md.mode()) && (md.size() != 0)) diff --git a/src/proxy/rpc/forward_metadata.cpp b/src/proxy/rpc/forward_metadata.cpp index a4f296742afff6f5a6501436c5ee16061a5a0710..05cd5551e542ade6888c1d9a5180fcfa9b90d281 100644 --- a/src/proxy/rpc/forward_metadata.cpp +++ b/src/proxy/rpc/forward_metadata.cpp @@ -40,6 +40,7 @@ remove_metadata(const std::string& path, bool rm_dir) { rpc_rm_metadata_out_t daemon_out{}; daemon_in.path = path; daemon_in.rm_dir = rm_dir; + daemon_in.is_rename_stub = false; auto endp = PROXY_DATA->rpc_endpoints().at( PROXY_DATA->distributor()->locate_file_metadata(path, 0)); @@ -68,6 +69,7 @@ remove_data(const std::string& path) { rpc_rm_node_in_t rpc_in{}; rpc_in.path = path; rpc_in.rm_dir = true; + rpc_in.is_rename_stub = false; try { diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 068e105e172208095c3829cec7210e346fce1128..f68962804a0c9f155aa48d6a35a1c171e8c7da99 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -32,7 +32,7 @@ include(FetchContent) set(FETCHCONTENT_QUIET OFF) FetchContent_Declare(catch2 GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v2.13.9 + GIT_TAG v3.15.1 GIT_SHALLOW ON GIT_PROGRESS ON ) @@ -43,11 +43,11 @@ add_subdirectory(helpers) # create a convenience library with Catch2's main # to speed up test compilation -add_library(catch2_main STATIC) -target_sources(catch2_main PRIVATE catch_main.cpp) -target_link_libraries(catch2_main - Catch2::Catch2 -) +#add_library(catch2_main STATIC) +#target_sources(catch2_main PRIVATE catch_main.cpp) +#target_link_libraries(catch2_main +# Catch2::Catch2 +#) # define executables for tests and make them depend on the convenience # library (and Catch2 transitively) and fmt @@ -67,7 +67,7 @@ endif () if (GKFS_BUILD_USER_LIB) target_link_libraries(unit_tests PRIVATE - catch2_main + Catch2WithMain fmt::fmt path_util helpers @@ -78,7 +78,7 @@ target_link_libraries(unit_tests else () target_link_libraries(unit_tests PRIVATE - catch2_main + Catch2WithMain fmt::fmt path_util helpers diff --git a/tests/unit/catch_main.cpp b/tests/unit/catch_main.cpp index 8e99c082ecbd4e4bbbd3158cf1226844c2132028..376780649c078c76e41e6fdf52cb1f5c0ddca3f7 100644 --- a/tests/unit/catch_main.cpp +++ b/tests/unit/catch_main.cpp @@ -37,4 +37,4 @@ */ #define CATCH_CONFIG_MAIN -#include +#include diff --git a/tests/unit/test_common_path.cpp b/tests/unit/test_common_path.cpp index 68984a609ab9b1db969e57befd381a9eedffd35a..b8edf78b56e501974cd9e473bf5af4dcb150fd4b 100644 --- a/tests/unit/test_common_path.cpp +++ b/tests/unit/test_common_path.cpp @@ -26,7 +26,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include TEST_CASE("Path utils", "[common][path_util]") { diff --git a/tests/unit/test_distributor.cpp b/tests/unit/test_distributor.cpp index 902c8d2bf905911ba8d8c8671c9c7a85acd52956..f7e69150c115820be36ab6867e662c98f1a78e5e 100644 --- a/tests/unit/test_distributor.cpp +++ b/tests/unit/test_distributor.cpp @@ -26,7 +26,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include TEST_CASE("SimpleHashDistributor", "[common][distributor]") { diff --git a/tests/unit/test_example_00.cpp b/tests/unit/test_example_00.cpp index 94b299605666140d1a2f5ab87afde1d73a24fe2d..718cfea38e419fa1041d0fc7a157fe2a87d84181 100644 --- a/tests/unit/test_example_00.cpp +++ b/tests/unit/test_example_00.cpp @@ -36,7 +36,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include unsigned int diff --git a/tests/unit/test_example_01.cpp b/tests/unit/test_example_01.cpp index e3e6a8745c46299bfa9db493f14727ca05098522..eaf4c600da2f37ba05db7872c8444cfcfae91fe1 100644 --- a/tests/unit/test_example_01.cpp +++ b/tests/unit/test_example_01.cpp @@ -36,7 +36,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include SCENARIO("vectors can be sized and resized", "[vector]") { diff --git a/tests/unit/test_guided_distributor.cpp b/tests/unit/test_guided_distributor.cpp index ebdbc02fb36a309a0a0ef53dd9553c3d7d8308ed..af0ef8e2dedbefee24026bfb30d7352829df8596 100644 --- a/tests/unit/test_guided_distributor.cpp +++ b/tests/unit/test_guided_distributor.cpp @@ -21,7 +21,7 @@ SPDX-License-Identifier: MIT */ -#include +#include #include #include #include diff --git a/tests/unit/test_helpers.cpp b/tests/unit/test_helpers.cpp index 2bdf35b8051ceb3baa1be35280c4b3686b41419c..3207978b81ee99a348566341542014c892db62c2 100644 --- a/tests/unit/test_helpers.cpp +++ b/tests/unit/test_helpers.cpp @@ -36,7 +36,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include "helpers/helpers.hpp" SCENARIO("random strings can be generated", "[test_helpers][random_string]") { diff --git a/tests/unit/test_path.cpp b/tests/unit/test_path.cpp index 80a685844978291755bf61c0c7e8361862e88492..1160c2b400f9befddf87bcfe0ba2928a34b39afd 100644 --- a/tests/unit/test_path.cpp +++ b/tests/unit/test_path.cpp @@ -36,7 +36,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include #include diff --git a/tests/unit/test_utils_arithmetic.cpp b/tests/unit/test_utils_arithmetic.cpp index a550caf4286e50b92d8b66429e40c9ca1de31a01..697af7ad5c55391453ae654ff0daf972d49d38bd 100644 --- a/tests/unit/test_utils_arithmetic.cpp +++ b/tests/unit/test_utils_arithmetic.cpp @@ -36,7 +36,7 @@ SPDX-License-Identifier: GPL-3.0-or-later */ -#include +#include #include #include