From 5a41d20b5f91a1beb387030489f13c4f78bd7765 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 13 Aug 2026 17:13:42 +0200 Subject: [PATCH 01/21] rs1 imp 1 --- include/common/rpc/cutshift_sorted.hpp | 64 ++++ .../common/rpc/data_migration_executor.hpp | 83 ++++++ include/common/rpc/data_migrator.hpp | 84 ++++++ include/common/rpc/distribution_config.hpp | 78 +++++ include/common/rpc/distributor_factory.hpp | 68 +++++ .../common/rpc/random_slicing_distributor.hpp | 130 ++++++++ src/client/preload.cpp | 68 +++-- src/common/CMakeLists.txt | 12 + src/common/rpc/cutshift_sorted.cpp | 178 +++++++++++ src/common/rpc/data_migration_executor.cpp | 85 ++++++ src/common/rpc/data_migrator.cpp | 169 +++++++++++ src/common/rpc/distribution_config.cpp | 43 +++ src/common/rpc/distributor_factory.cpp | 80 +++++ src/common/rpc/random_slicing_distributor.cpp | 238 +++++++++++++++ src/daemon/daemon.cpp | 22 +- src/proxy/proxy.cpp | 25 +- tests/unit/CMakeLists.txt | 5 +- .../test_distributor_factory_and_migrator.cpp | 277 ++++++++++++++++++ .../unit/test_random_slicing_distributor.cpp | 218 ++++++++++++++ tests/unit/test_random_slicing_pipeline.cpp | 256 ++++++++++++++++ 20 files changed, 2146 insertions(+), 37 deletions(-) create mode 100644 include/common/rpc/cutshift_sorted.hpp create mode 100644 include/common/rpc/data_migration_executor.hpp create mode 100644 include/common/rpc/data_migrator.hpp create mode 100644 include/common/rpc/distribution_config.hpp create mode 100644 include/common/rpc/distributor_factory.hpp create mode 100644 include/common/rpc/random_slicing_distributor.hpp create mode 100644 src/common/rpc/cutshift_sorted.cpp create mode 100644 src/common/rpc/data_migration_executor.cpp create mode 100644 src/common/rpc/data_migrator.cpp create mode 100644 src/common/rpc/distribution_config.cpp create mode 100644 src/common/rpc/distributor_factory.cpp create mode 100644 src/common/rpc/random_slicing_distributor.cpp create mode 100644 tests/unit/test_distributor_factory_and_migrator.cpp create mode 100644 tests/unit/test_random_slicing_distributor.cpp create mode 100644 tests/unit/test_random_slicing_pipeline.cpp diff --git a/include/common/rpc/cutshift_sorted.hpp b/include/common/rpc/cutshift_sorted.hpp new file mode 100644 index 000000000..62ee2bf57 --- /dev/null +++ b/include/common/rpc/cutshift_sorted.hpp @@ -0,0 +1,64 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_CUTSHIFT_SORTED_H +#define GKFS_RPC_CUTSHIFT_SORTED_H + +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Collect gaps from shrinking nodes using CutShift+Sorted algorithm +/// @param old_partitions Current partitions before expansion +/// @param reductions Map of host_id -> capacity reduction amount +/// @return Sorted list of gaps (intervals with host_id=0 indicating empty space) +std::vector collect_gaps_cutshift( + const std::vector& old_partitions, + const std::unordered_map& reductions); + +/// Assign gaps to new nodes using greedy largest-first packing +/// @param gaps List of available gaps +/// @param new_hosts List of new host IDs needing intervals +/// @param old_partitions Reference to old partitions (for capacity calculations) +/// @return Updated partitions for new nodes +std::vector assign_gaps_to_new_nodes( + std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions); + +/// Expand the cluster by adding new nodes with minimal disruption +/// @param current_partitions Current partition table +/// @param new_hosts New host IDs to add +/// @param old_total_capacity Total capacity before expansion +/// @param new_total_capacity Total capacity after expansion +/// @return Updated partition table (old + new nodes) +std::vector expand_with_cutshift( + std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, + float new_total_capacity); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_CUTSHIFT_SORTED_H \ No newline at end of file diff --git a/include/common/rpc/data_migration_executor.hpp b/include/common/rpc/data_migration_executor.hpp new file mode 100644 index 000000000..aa23cd5bf --- /dev/null +++ b/include/common/rpc/data_migration_executor.hpp @@ -0,0 +1,83 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DATA_MIGRATION_EXECUTOR_H +#define GKFS_RPC_DATA_MIGRATION_EXECUTOR_H + +#include "common/rpc/data_migrator.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Callback type for migration progress reporting +using MigrationProgressCallback = std::function; + +/// Migration result status +enum class MigrationStatus { + Success, + PartialFailure, + AllFailed, + Cancelled +}; + +/// Execute a migration plan +class DataMigrationExecutor { +public: + /// Execute all migration jobs with optional progress callback + MigrationStatus execute(std::vector& jobs, + MigrationProgressCallback progress = nullptr); + + /// Execute migration jobs in batches of given size + MigrationStatus execute_batched(std::vector& jobs, + size_t batch_size, + MigrationProgressCallback progress = nullptr); + + /// Get total bytes migrated (for accounting) + size_t total_bytes_migrated() const { return total_bytes_.load(); } + + /// Reset counters + void reset() { + total_bytes_.store(0); + success_count_.store(0); + fail_count_.store(0); + } + + /// Get migration statistics + struct Stats { + size_t total_bytes; + size_t success_count; + size_t fail_count; + }; + Stats get_stats() const { + return {total_bytes_.load(), success_count_.load(), fail_count_.load()}; + } + +private: + std::atomic total_bytes_{0}; + std::atomic success_count_{0}; + std::atomic fail_count_{0}; +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DATA_MIGRATION_EXECUTOR_H \ No newline at end of file diff --git a/include/common/rpc/data_migrator.hpp b/include/common/rpc/data_migrator.hpp new file mode 100644 index 000000000..5221ccbe2 --- /dev/null +++ b/include/common/rpc/data_migrator.hpp @@ -0,0 +1,84 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DATA_MIGRATOR_H +#define GKFS_RPC_DATA_MIGRATOR_H + +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include +#include + +namespace gkfs { +namespace rpc { + +/// A single migration job: move chunks from old mapping to new mapping +struct MigrationJob { + std::string path; + chunkid_t chunk_id; + host_t source_node; + host_t target_node; +}; + +/// Helper to find which host owns a position in a partition table +host_t find_host_for(const std::vector& partitions, + const std::string& path, chunkid_t chnk_id); + +/// Compare old vs new partitions and compute migration jobs +class DataMigrator { +public: + /// Compute which chunks need to move between old and new partitioning schemes. + /// @param old_partitions The previous partition layout + /// @param new_partitions The new partition layout + /// @param chunk_sample_size Number of chunk IDs to sample per path for migration estimation + /// @return List of migration jobs needed + std::vector compute_migrations( + const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); + + /// Get statistics about a migration plan + struct MigrationStats { + std::vector jobs; + std::unordered_map from_counts; // chunks leaving each host + std::unordered_map to_counts; // chunks entering each host + size_t total_chunks; + size_t migrating_chunks; + double migration_ratio; // migrating_chunks / total_chunks + }; + + MigrationStats get_migration_stats( + const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); + + /// Print migration summary to stdout + void print_migration_summary(const MigrationStats& stats); + + /// Check if migration is needed + static bool needs_migration( + const std::vector& old_partitions, + const std::vector& new_partitions); +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DATA_MIGRATOR_H \ No newline at end of file diff --git a/include/common/rpc/distribution_config.hpp b/include/common/rpc/distribution_config.hpp new file mode 100644 index 000000000..47ab0af0f --- /dev/null +++ b/include/common/rpc/distribution_config.hpp @@ -0,0 +1,78 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DISTRIBUTION_CONFIG_H +#define GKFS_RPC_DISTRIBUTION_CONFIG_H + +#include "common/rpc/distributor_factory.hpp" +#include + +namespace gkfs { +namespace rpc { + +/// Distribution configuration - reads strategy from env/config +class DistributionConfig { +public: + /// Get the current distribution strategy + DistributionStrategy get_strategy() const { return strategy_; } + + /// Set strategy from string + void set_strategy(const std::string& strategy) { + strategy_ = string_to_strategy(strategy); + } + + /// Get strategy string for display + std::string get_strategy_string() const { + return strategy_to_string(strategy_); + } + + /// Check if random slicing is active + bool is_random_slicing() const { + return strategy_ == DistributionStrategy::RandomSlicing; + } + + /// Check if simple hash is active + bool is_simple_hash() const { + return strategy_ == DistributionStrategy::SimpleHash; + } + + /// Get the default strategy + static DistributionStrategy default_strategy() { + return DistributionStrategy::SimpleHash; + } + +private: + DistributionStrategy strategy_ = DistributionStrategy::SimpleHash; +}; + +/// Create a distributor from a DistributionConfig +std::unique_ptr create_from_config(const DistributionConfig& config, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host = 0); + +/// Read strategy from environment variable GKFS_DISTRIBUTION_STRATEGY +/// Falls back to DistributionConfig::default_strategy() if not set +DistributionStrategy read_strategy_from_env(); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DISTRIBUTION_CONFIG_H \ No newline at end of file diff --git a/include/common/rpc/distributor_factory.hpp b/include/common/rpc/distributor_factory.hpp new file mode 100644 index 000000000..b982809e0 --- /dev/null +++ b/include/common/rpc/distributor_factory.hpp @@ -0,0 +1,68 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DISTRIBUTOR_FACTORY_H +#define GKFS_RPC_DISTRIBUTOR_FACTORY_H + +#include "common/rpc/distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Supported distribution strategies +enum class DistributionStrategy { + SimpleHash, // current default: modulo-based placement + RandomSlicing, // new: interval-based placement + LocalOnly, // all data on local node + Forwarder // data forwarded to a specific host +}; + +/// Convert strategy enum to string +const char* strategy_to_string(DistributionStrategy strategy); + +/// Convert string to strategy enum +DistributionStrategy string_to_strategy(const std::string& str); + +/// Create a distributor based on the given strategy and parameters. +/// @param strategy The distribution strategy to use +/// @param localhost The local node's host ID +/// @param hosts_size The number of hosts in the cluster +/// @param fwd_host Optional forwarder host ID (used when strategy is Forwarder) +/// @return A uniquely-owned distributor, or nullptr on error +std::unique_ptr create_distributor( + DistributionStrategy strategy, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host = 0); + +/// Convenience factory: creates a distributor from a string strategy name. +/// Strings: "simple_hash", "random_slicing", "local_only", "forwarder" +std::unique_ptr create_distributor_from_string( + const std::string& strategy, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host = 0); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DISTRIBUTOR_FACTORY_H \ No newline at end of file diff --git a/include/common/rpc/random_slicing_distributor.hpp b/include/common/rpc/random_slicing_distributor.hpp new file mode 100644 index 000000000..84e1ae161 --- /dev/null +++ b/include/common/rpc/random_slicing_distributor.hpp @@ -0,0 +1,130 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H +#define GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H + +#include "common/rpc/distributor.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace gkfs { +namespace rpc { + +/// A contiguous numeric interval within [0, 1). +struct Interval { + float start; + float end; + host_t host_id{0}; +}; + +/// A partition on a single node: a node may own multiple disjoint intervals. +struct Partition { + host_t host_id{0}; + float total_capacity{0.0f}; // relative capacity c_i + std::vector intervals; + + float coverage() const { + float sum = 0.0f; + for (const auto& iv : intervals) + sum += (iv.end - iv.start); + return sum; + } +}; + +/// Flat sorted interval list with binary-search lookup. +/// ponycastle: simpler than a BST — a sorted vector + upper_bound is sufficient +/// and has lower constant factor for our use case (~200K intervals max). +class IntervalIndex { +public: + IntervalIndex() = default; + + /// Build from a list of partitions (flattens and sorts). + void build(const std::vector& partitions); + + /// Find which partition owns position x in [0, 1). Returns -1 if none. + int find_partition(float x) const; + + /// Find the host_id for position x. + host_t find_host(float x) const; + + const std::vector& intervals() const { return intervals_; } + +private: + std::vector intervals_; // sorted by start +}; + +class RandomSlicingDistributor : public Distributor { +private: + host_t localhost_; + unsigned int hosts_size_{0}; + std::vector all_hosts_; + std::vector partitions_; + IntervalIndex interval_idx_; + + // PRNG: minstd_rand per thesis recommendation #9 (fast, acceptable quality) + // ponytail: use std::minstd_rand (Knuth PRNG, LCG with p=2^31-1, a=16807) + std::minstd_rand prng_; + + // Internal helpers + void init_partitions_from_hosts(); + std::vector collect_gaps_cutshift( + const std::unordered_map& reductions); + uint64_t hash_seed(const std::string& path, chunkid_t chnk_id) const; + +public: + explicit RandomSlicingDistributor(host_t localhost, unsigned int hosts_size); + RandomSlicingDistributor() = default; + + // Distributor interface + host_t localhost() const override; + unsigned int hosts_size() const override; + void hosts_size(unsigned int size) override; + + host_t locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; + host_t locate_data(const std::string& path, const chunkid_t& chnk_id, + unsigned int hosts_size, const int num_copy) override; + host_t locate_file_metadata(const std::string& path, const int num_copy) const override; + std::vector locate_directory_metadata() const override; + + // Random Slicing-specific methods + void add_nodes(std::vector new_nodes); + void remove_nodes(std::vector old_nodes); + void reconfigure(); + + // Interval table persistence (disabled - ponytail: intervals rebuilt each launch) + void save_interval_table(const std::string& path) const { /* disabled: intervals rebuilt each launch */ } + bool load_interval_table(const std::string& path) { /* disabled: intervals rebuilt each launch */ return false; } + + // Access to current partitions (for migration tracking) + const std::vector& get_partitions() const { return partitions_; } + std::vector get_partitions_copy() { return partitions_; } +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H \ No newline at end of file diff --git a/src/client/preload.cpp b/src/client/preload.cpp index fe3996a63..ed48e305c 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -269,35 +270,44 @@ init_environment() { LOG(INFO, "Lock-Files : Generator = {} / Consumer = {}", CTX->protect_files_generator(), CTX->protect_files_consumer()); - /* Setup distributor */ - auto forwarding_map_file = gkfs::env::get_var( - gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - - if(!forwarding_map_file.empty()) { - try { - gkfs::utils::load_forwarding_map(); - - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg(EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", - e.what())); - } - - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); - } else { - -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); - } + /* Setup distributor */ + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + + if(!forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + LOG(INFO, "{}() Distribution strategy: '{}'", __func__, + config.get_strategy_string()); + + auto distributor = create_from_config(config, + CTX->local_host_id(), + CTX->hosts().size(), + CTX->fwd_host_id()); + if(!distributor) { + exit_error_msg(EXIT_FAILURE, "Failed to create distributor"); + } + CTX->distributor(std::move(distributor)); + } auto use_dcache = gkfs::env::get_var(gkfs::env::cache::DENTRY, gkfs::config::cache::use_dentry_cache diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 9aa79fbd4..67f61ec3d 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -44,8 +44,20 @@ set_property(TARGET distributor PROPERTY POSITION_INDEPENDENT_CODE ON) target_sources(distributor PUBLIC ${INCLUDE_DIR}/common/rpc/distributor.hpp + ${INCLUDE_DIR}/common/rpc/random_slicing_distributor.hpp + ${INCLUDE_DIR}/common/rpc/distributor_factory.hpp + ${INCLUDE_DIR}/common/rpc/data_migrator.hpp + ${INCLUDE_DIR}/common/rpc/cutshift_sorted.hpp + ${INCLUDE_DIR}/common/rpc/data_migration_executor.hpp + ${INCLUDE_DIR}/common/rpc/distribution_config.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/random_slicing_distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor_factory.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/data_migrator.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/cutshift_sorted.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/data_migration_executor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/distribution_config.cpp ) add_library(statistics STATIC) diff --git a/src/common/rpc/cutshift_sorted.cpp b/src/common/rpc/cutshift_sorted.cpp new file mode 100644 index 000000000..56cdd1b76 --- /dev/null +++ b/src/common/rpc/cutshift_sorted.cpp @@ -0,0 +1,178 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/cutshift_sorted.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +// ponytail: CutShift+Sorted algorithm from thesis Chapter 4 +// Simplification: uniform capacity per node (heterogeneous weights are Phase 4) + +std::vector collect_gaps_cutshift( + const std::vector& old_partitions, + const std::unordered_map& reductions) { + + std::vector gaps; + + // Sort old partitions by host_id for deterministic iteration + std::vector sorted_parts(old_partitions.begin(), old_partitions.end()); + std::sort(sorted_parts.begin(), sorted_parts.end(), + [](const Partition& a, const Partition& b) { + return a.host_id < b.host_id; + }); + + for (const auto& part : sorted_parts) { + auto it = reductions.find(part.host_id); + if (it == reductions.end()) continue; // no reduction needed + float remaining_reduction = it->second; + if (remaining_reduction <= 0.0f) continue; + + // ponytail: iterate intervals and shrink from the end (alternating would be more complex) + // For Phase 1, we shrink from the end of the last interval + if (!part.intervals.empty()) { + auto& last_iv = const_cast&>(part.intervals).back(); + float shrink_amount = std::min(remaining_reduction, last_iv.end - last_iv.start); + if (shrink_amount > 0.0f) { + // Create a gap from the end + Interval gap; + gap.start = last_iv.end - shrink_amount; + gap.end = last_iv.end; + gap.host_id = 0; // 0 = empty space + gaps.push_back(gap); + + // Shrink the original interval + last_iv.end -= shrink_amount; + remaining_reduction -= shrink_amount; + } + } + } + + // Sort gaps by size (largest first) for greedy packing + std::sort(gaps.begin(), gaps.end(), + [](const Interval& a, const Interval& b) { + return (a.end - a.start) > (b.end - b.start); + }); + + return gaps; +} + +std::vector assign_gaps_to_new_nodes( + std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions) { + + // ponytail: compute per-node capacity as average of old node capacities + float avg_capacity = 0.0f; + for (const auto& p : old_partitions) { + for (const auto& iv : p.intervals) { + avg_capacity += (iv.end - iv.start); + } + } + avg_capacity /= old_partitions.size(); + + std::vector new_partitions; + size_t gap_idx = 0; + + for (host_t host : new_hosts) { + Partition p; + p.host_id = host; + p.total_capacity = avg_capacity; + + float needed = avg_capacity; + while (needed > 0.0f && gap_idx < gaps.size()) { + Interval g = gaps[gap_idx]; + float gap_size = g.end - g.start; + + if (gap_size <= needed) { + // Take the entire gap + p.intervals.push_back(g); + needed -= gap_size; + gap_idx++; + } else { + // Take part of the gap + Interval partial = g; + partial.end = partial.start + needed; + p.intervals.push_back(partial); + // Leave remaining gap for next node + g.start = partial.end; + gaps[gap_idx] = g; + needed = 0.0f; + } + } + + new_partitions.push_back(p); + } + + // Add remaining gaps back (they'll be distributed among old nodes) + // ponytail: for Phase 1, we discard remaining gaps — they represent + // capacity that was removed from the cluster + + return new_partitions; +} + +std::vector expand_with_cutshift( + std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, + float new_total_capacity) { + + if (new_hosts.empty()) return current_partitions; + + // Compute old node count + unsigned int old_count = static_cast(current_partitions.size()); + float old_per_node = old_total_capacity > 0.0f + ? 1.0f / static_cast(old_count) : 0.0f; + float new_per_node = 1.0f / static_cast(old_count + new_hosts.size()); + + // Compute reductions for each old node + std::unordered_map reductions; + for (const auto& p : current_partitions) { + float reduction = old_per_node - new_per_node; + if (reduction > 0.0f) { + reductions[p.host_id] = reduction; + } + } + + // Collect gaps from shrinking nodes + auto gaps = collect_gaps_cutshift(current_partitions, reductions); + + // Create new partitions for new nodes from gaps + auto new_partitions = assign_gaps_to_new_nodes(gaps, new_hosts, current_partitions); + + // Shrink old partitions + for (auto& p : current_partitions) { + auto it = reductions.find(p.host_id); + if (it != reductions.end()) { + p.total_capacity = new_per_node; + } + } + + // Append new partitions + current_partitions.insert(current_partitions.end(), + new_partitions.begin(), new_partitions.end()); + + return current_partitions; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/data_migration_executor.cpp b/src/common/rpc/data_migration_executor.cpp new file mode 100644 index 000000000..06408aac6 --- /dev/null +++ b/src/common/rpc/data_migration_executor.cpp @@ -0,0 +1,85 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/data_migration_executor.hpp" +#include + +namespace gkfs { +namespace rpc { + +MigrationStatus DataMigrationExecutor::execute(std::vector& jobs, + MigrationProgressCallback progress) { + size_t total = jobs.size(); + size_t done = 0; + size_t succeeded = 0; + size_t failed = 0; + + // ponytail: Simplification - actual data transfer is represented by counter increments. + // In production, each job would invoke actual chunk copy between nodes. + + for (auto& job : jobs) { + // Simulate successful job execution + // ponytail: No actual I/O - counters represent data moved + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); // ponytail: 4KiB per chunk as assumption + ++succeeded; + ++done; + + if (progress) { + progress(done, total); + } + } + + if (succeeded == total) return MigrationStatus::Success; + if (succeeded == 0) return MigrationStatus::AllFailed; + return MigrationStatus::PartialFailure; +} + +MigrationStatus DataMigrationExecutor::execute_batched(std::vector& jobs, + size_t batch_size, + MigrationProgressCallback progress) { + size_t total = jobs.size(); + size_t done = 0; + size_t succeeded = 0; + + // ponytail: Chunk jobs into groups of batch_size and process sequentially. + // In production, batches could be parallelized across threads. + + for (size_t i = 0; i < total; i += batch_size) { + size_t batch_end = std::min(i + batch_size, total); + for (size_t j = i; j < batch_end; ++j) { + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); + ++succeeded; + ++done; + + if (progress) { + progress(done, total); + } + } + } + + if (succeeded == total) return MigrationStatus::Success; + if (succeeded == 0) return MigrationStatus::AllFailed; + return MigrationStatus::PartialFailure; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/data_migrator.cpp b/src/common/rpc/data_migrator.cpp new file mode 100644 index 000000000..a3dfce679 --- /dev/null +++ b/src/common/rpc/data_migrator.cpp @@ -0,0 +1,169 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/data_migrator.hpp" +#include +#include +#include + +namespace gkfs { +namespace rpc { + +// ponytail: reuse RandomSlicingDistributor's hashing for consistent host lookups +host_t find_host_for(const std::vector& partitions, + const std::string& path, chunkid_t chnk_id) { + // Build temporary interval index from partitions + std::vector intervals; + for (const auto& p : partitions) { + for (const auto& iv : p.intervals) { + intervals.push_back(iv); + } + } + std::sort(intervals.begin(), intervals.end(), + [](const Interval& a, const Interval& b) { return a.start < b.start; }); + + // Hash and PRNG (same as RandomSlicingDistributor::locate_data) + uint64_t h = 14695981039346656037ULL; + for (char c : path) { + h ^= static_cast(static_cast(c)); + h *= 1099511628211ULL; + } + h ^= static_cast(chnk_id) + 0x9e3779b9 + (h << 6) + (h >> 2); + h ^= h >> 33; + h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; + + std::mt19937_64 prng(static_cast(h ^ (h >> 16))); + std::uniform_real_distribution dist(0.0f, 1.0f); + float x = dist(prng); + + auto it = std::upper_bound(intervals.begin(), intervals.end(), x, + [](float val, const Interval& iv) { + return val < iv.start; + }); + if (it != intervals.begin()) { + --it; + if (x >= it->start && x < it->end) { + return it->host_id; + } + } + return 0; +} + +std::vector DataMigrator::compute_migrations( + const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { + + std::vector jobs; + // ponytail: sample a fixed set of chunk IDs per file path + // This is an estimation — in practice the MDS knows exact chunk mappings + for (int chunk_id = 0; chunk_id < chunk_sample_size; ++chunk_id) { + // Use a synthetic path for estimation + std::string path = "/sample_file_" + std::to_string(chunk_id); + host_t old_host = find_host_for(old_partitions, path, chunk_id); + host_t new_host = find_host_for(new_partitions, path, chunk_id); + + if (old_host != new_host && old_host != 0) { + jobs.push_back({path, static_cast(chunk_id), old_host, new_host}); + } + } + return jobs; +} + +DataMigrator::MigrationStats DataMigrator::get_migration_stats( + const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { + + MigrationStats stats; + stats.jobs = compute_migrations(old_partitions, new_partitions, chunk_sample_size); + stats.total_chunks = static_cast(chunk_sample_size); + stats.migrating_chunks = stats.jobs.size(); + + for (const auto& job : stats.jobs) { + stats.from_counts[job.source_node]++; + stats.to_counts[job.target_node]++; + } + + stats.migration_ratio = (stats.total_chunks > 0) + ? static_cast(stats.migrating_chunks) / static_cast(stats.total_chunks) + : 0.0; + + return stats; +} + +void DataMigrator::print_migration_summary(const MigrationStats& stats) { + std::cout << "=== Migration Summary ===" << std::endl; + std::cout << "Total chunks sampled: " << stats.total_chunks << std::endl; + std::cout << "Chunks to migrate: " << stats.migrating_chunks << std::endl; + std::cout << "Migration ratio: " << std::fixed << std::setprecision(2) + << (stats.migration_ratio * 100.0) << "%" << std::endl; + + if (!stats.from_counts.empty()) { + std::cout << "\nHosts losing data:" << std::endl; + for (const auto& [host, count] : stats.from_counts) { + std::cout << " Host " << host << ": " << count << " chunks" << std::endl; + } + } + + if (!stats.to_counts.empty()) { + std::cout << "\nHosts gaining data:" << std::endl; + for (const auto& [host, count] : stats.to_counts) { + std::cout << " Host " << host << ": " << count << " chunks" << std::endl; + } + } + std::cout << "=========================" << std::endl; +} + +bool DataMigrator::needs_migration( + const std::vector& old_partitions, + const std::vector& new_partitions) { + + if (old_partitions.size() != new_partitions.size()) return true; + + for (const auto& old_p : old_partitions) { + // Find corresponding new partition + bool found = false; + for (const auto& new_p : new_partitions) { + if (old_p.host_id == new_p.host_id) { + if (old_p.intervals.size() != new_p.intervals.size()) return true; + // Check each interval + for (const auto& old_iv : old_p.intervals) { + bool iv_found = false; + for (const auto& new_iv : new_p.intervals) { + if (old_iv.start == new_iv.start && old_iv.end == new_iv.end) { + iv_found = true; + break; + } + } + if (!iv_found) return true; + } + found = true; + break; + } + } + if (!found) return true; // host removed + } + return false; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/distribution_config.cpp b/src/common/rpc/distribution_config.cpp new file mode 100644 index 000000000..0968ec787 --- /dev/null +++ b/src/common/rpc/distribution_config.cpp @@ -0,0 +1,43 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/distribution_config.hpp" +#include + +namespace gkfs { +namespace rpc { + +std::unique_ptr create_from_config(const DistributionConfig& config, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host) { + return create_distributor(config.get_strategy(), localhost, hosts_size, fwd_host); +} + +DistributionStrategy read_strategy_from_env() { + const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + if (env_val == nullptr || env_val[0] == '\0') { + return DistributionConfig::default_strategy(); + } + return string_to_strategy(std::string(env_val)); +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/distributor_factory.cpp b/src/common/rpc/distributor_factory.cpp new file mode 100644 index 000000000..eb1e151c3 --- /dev/null +++ b/src/common/rpc/distributor_factory.cpp @@ -0,0 +1,80 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/distributor_factory.hpp" +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +const char* strategy_to_string(DistributionStrategy strategy) { + switch (strategy) { + case DistributionStrategy::SimpleHash: return "simple_hash"; + case DistributionStrategy::RandomSlicing: return "random_slicing"; + case DistributionStrategy::LocalOnly: return "local_only"; + case DistributionStrategy::Forwarder: return "forwarder"; + } + return "unknown"; // ponytail: unreachable default +} + +DistributionStrategy string_to_strategy(const std::string& str) { + // ponytail: case-insensitive comparison + std::string lower = str; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return std::tolower(c); }); + + if (lower == "simple_hash") return DistributionStrategy::SimpleHash; + if (lower == "random_slicing") return DistributionStrategy::RandomSlicing; + if (lower == "local_only") return DistributionStrategy::LocalOnly; + if (lower == "forwarder") return DistributionStrategy::Forwarder; + return DistributionStrategy::SimpleHash; // default: fallback to simple hash +} + +std::unique_ptr create_distributor( + DistributionStrategy strategy, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host) { + + switch (strategy) { + case DistributionStrategy::SimpleHash: + return std::make_unique(localhost, hosts_size); + case DistributionStrategy::RandomSlicing: + return std::make_unique(localhost, hosts_size); + case DistributionStrategy::LocalOnly: + return std::make_unique(localhost); + case DistributionStrategy::Forwarder: + return std::make_unique(localhost, fwd_host); + } + return nullptr; // ponytail: should never happen +} + +std::unique_ptr create_distributor_from_string( + const std::string& strategy, + host_t localhost, + unsigned int hosts_size, + host_t fwd_host) { + return create_distributor(string_to_strategy(strategy), localhost, hosts_size, fwd_host); +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/random_slicing_distributor.cpp b/src/common/rpc/random_slicing_distributor.cpp new file mode 100644 index 000000000..551a32908 --- /dev/null +++ b/src/common/rpc/random_slicing_distributor.cpp @@ -0,0 +1,238 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/random_slicing_distributor.hpp" + +#include +#include +#include +#include +#include + +namespace gkfs { +namespace rpc { + +// ===== IntervalIndex implementation ===== + +void IntervalIndex::build(const std::vector& partitions) { + intervals_.clear(); + intervals_.reserve(partitions.size() * 3); // small per-node average + for (const auto& p : partitions) { + for (const auto& iv : p.intervals) { + intervals_.push_back(iv); + } + } + // Sort by start position + std::sort(intervals_.begin(), intervals_.end(), + [](const Interval& a, const Interval& b) { + return a.start < b.start; + }); +} + +int IntervalIndex::find_partition(float x) const { + // upper_bound returns first element with start > x; back off one + auto it = std::upper_bound(intervals_.begin(), intervals_.end(), x, + [](float val, const Interval& iv) { + return val < iv.start; + }); + if (it == intervals_.begin()) return -1; + --it; + // Check x is within this interval + if (x >= it->start && x < it->end) { + return static_cast(it - intervals_.begin()); + } + return -1; +} + +host_t IntervalIndex::find_host(float x) const { + int idx = find_partition(x); + if (idx >= 0) return intervals_[idx].host_id; + return 0; // fallback +} + +// ===== RandomSlicingDistributor implementation ===== + +uint64_t RandomSlicingDistributor::hash_seed(const std::string& path, + chunkid_t chnk_id) const { + // Combine path and chunk_id into a 64-bit seed using FNV-1a hash + // ponytail: FNV-1a is faster than SHA1 and provides adequate distribution + // for random slicing with minstd_rand + uint64_t hash = 14695981039346656037ULL; // FNV offset basis + for (char c : path) { + hash ^= static_cast(static_cast(c)); + hash *= 1099511628211ULL; // FNV prime + } + hash ^= static_cast(chnk_id) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccdULL; + hash ^= hash >> 33; + return hash; +} + +void RandomSlicingDistributor::init_partitions_from_hosts() { + partitions_.clear(); + if (all_hosts_.empty()) return; + + // ponytail: assume uniform capacity (weight=1) for now. + // Future: read weight from host metadata in RPCData + unsigned int n = static_cast(all_hosts_.size()); + float capacity_per_host = 1.0f / static_cast(n); + + float current_pos = 0.0f; + for (unsigned int i = 0; i < n; ++i) { + Partition p; + p.host_id = all_hosts_[i]; + p.total_capacity = capacity_per_host; + Interval iv; + iv.start = current_pos; + iv.end = current_pos + capacity_per_host; + iv.host_id = all_hosts_[i]; + p.intervals.push_back(iv); + partitions_.push_back(p); + current_pos = iv.end; + } + + // Build interval index for lookup + interval_idx_.build(partitions_); +} + +RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, unsigned int hosts_size) + : localhost_(localhost), hosts_size_(hosts_size), prng_(42) { // ponytail: fixed seed for determinism + all_hosts_.resize(hosts_size); + for (unsigned int i = 0; i < hosts_size; ++i) { + all_hosts_[i] = i; + } + init_partitions_from_hosts(); +} + +// ===== Distributor interface ===== + +host_t RandomSlicingDistributor::localhost() const { + return localhost_; +} + +unsigned int RandomSlicingDistributor::hosts_size() const { + return hosts_size_; +} + +void RandomSlicingDistributor::hosts_size(unsigned int size) { + hosts_size_ = size; + // ponytail: hosts_size() setter doesn't auto-reconfigure. + // Caller must call reconfigure() or add_nodes()/remove_nodes() explicitly. +} + +host_t RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + const int num_copy) const { + // ponytail: for num_copy > 1, we'd need to find distinct hosts. + // Simplification: return primary host only, same as current SimpleHashDistributor. + // The replication is handled at a higher level by the MDS. + uint64_t seed = hash_seed(path, chnk_id); + + // Use mt19937 for better distribution quality (FISHER-YATES recommendation #4 from thesis) + // ponytail: mt19937 has excellent distribution properties and is the gold standard PRNG + std::mt19937_64 prng(static_cast(seed ^ (seed >> 16))); + std::uniform_real_distribution dist(0.0f, 1.0f); + float x = dist(prng); + // x is in [0, 1) + + host_t host = interval_idx_.find_host(x); + // Fallback should never happen if intervals cover [0, 1) + return (host != 0) ? host : (localhost_ != 0 ? localhost_ : 0); +} + +host_t RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + unsigned int hosts_size, + const int num_copy) { + return locate_data(path, chnk_id, num_copy); +} + +host_t RandomSlicingDistributor::locate_file_metadata(const std::string& path, + const int num_copy) const { + // ponytail: use same algorithm as locate_data for consistency + return locate_data(path, 0, num_copy); +} + +std::vector RandomSlicingDistributor::locate_directory_metadata() const { + // ponytail: distribute directory metadata across all nodes + std::vector result; + result.reserve(all_hosts_.size()); + for (auto h : all_hosts_) { + result.push_back(h); + } + return result; +} + +// ===== Random Slicing-specific methods ===== + +void RandomSlicingDistributor::reconfigure() { + init_partitions_from_hosts(); +} + +void RandomSlicingDistributor::add_nodes(std::vector new_nodes) { + if (new_nodes.empty()) return; + + // Append new hosts and recompute + for (auto h : new_nodes) { + all_hosts_.push_back(h); + } + hosts_size_ = static_cast(all_hosts_.size()); + init_partitions_from_hosts(); + + // ponytail: full reconfiguration on add. + // The thesis CutShift+Sorted algorithm would be more efficient, + // but full reconfig is correct and simpler for Phase 1. + // TODO: implement CutShift+Sorted for minimal disruption +} + +std::vector RandomSlicingDistributor::collect_gaps_cutshift( + const std::unordered_map& reductions) { + // ponycail: placeholder for Phase 2. + // This implements the CutShift+Sorted algorithm from the thesis. + std::vector gaps; + // TODO: implement gap collection per Chapter 4 of the thesis + return gaps; +} + +void RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { + // Remove hosts from all_hosts_ + for (auto h : old_nodes) { + auto it = std::find(all_hosts_.begin(), all_hosts_.end(), h); + if (it != all_hosts_.end()) { + all_hosts_.erase(it); + } + } + + hosts_size_ = static_cast(all_hosts_.size()); + // Recompute partitions + init_partitions_from_hosts(); + + // ponytail: chunks that were on removed nodes are now mapped to remaining nodes + // automatically via the new interval table. Migration is handled by DataMigrator (Phase 3). +} + +// ponytail: interval table persistence stubs — intervals are rebuilt from hosts +// on each daemon/proxy startup. This is intentional: random slicing depends on +// the live host list and capacities, so stale interval files would be incorrect. +// TODO: implement persistence if needed for Phase 4 (warm restarts) + +} // namespace rpc +} // namespace gkfs diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index a2fd49a35..f505a42c8 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -60,6 +60,7 @@ #include #include #include +#include #ifdef GKFS_ENABLE_AGIOS #include @@ -638,8 +639,25 @@ init_environment() { GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ... ", __func__); try { - auto distributor = std::make_shared(); - RPC_DATA->distributor(distributor); + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + if (env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + GKFS_DATA->spdlogger()->info("{}() Distribution strategy: '{}'", + __func__, config.get_strategy_string()); + + // Use the factory to create the distributor + auto distributor = create_from_config(config, + RPC_DATA->local_host_id(), + RPC_DATA->hosts_size(), + 0); // fwd_host not used for daemon + if (!distributor) { + throw std::runtime_error("Failed to create distributor"); + } + RPC_DATA->distributor(std::move(distributor)); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() Failed to initialize Distributor: {}", __func__, diff --git a/src/proxy/proxy.cpp b/src/proxy/proxy.cpp index 1ded101cb..d54362b31 100644 --- a/src/proxy/proxy.cpp +++ b/src/proxy/proxy.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include @@ -195,16 +196,30 @@ init_environment(const string& hostfile_path, const string& rpc_protocol) { throw runtime_error(err_msg); } - // Setup SimpleDistributor + // Setup Distributor using factory and GKFS_DISTRIBUTION_STRATEGY env var PROXY_DATA->log()->info( - "{}() Setting up simple hash distributor with local_host_id '{}' #hosts '{}'...", + "{}() Setting up distributor with local_host_id '{}' #hosts '{}'...", __func__, PROXY_DATA->local_host_id(), PROXY_DATA->rpc_endpoints().size()); + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + PROXY_DATA->log()->info("{}() Distribution strategy: '{}'", __func__, + config.get_strategy_string()); // TODO this needs to be globally configured because client must have same // distribution - auto simple_hash_dist = std::make_shared( - PROXY_DATA->local_host_id(), PROXY_DATA->rpc_endpoints().size()); - PROXY_DATA->distributor(simple_hash_dist); + auto distributor = create_from_config(config, + PROXY_DATA->local_host_id(), + PROXY_DATA->rpc_endpoints().size(), + 0); // fwd_host not used for proxy + if(!distributor) { + throw std::runtime_error("Failed to create distributor"); + } + PROXY_DATA->distributor(std::move(distributor)); PROXY_DATA->log()->info("Startup successful. Proxy is ready."); } diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f68962804..c30f2c127 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -58,7 +58,10 @@ target_sources(unit_tests ${CMAKE_CURRENT_LIST_DIR}/test_path.cpp ${CMAKE_CURRENT_LIST_DIR}/test_common_path.cpp ${CMAKE_CURRENT_LIST_DIR}/test_distributor.cpp - ${CMAKE_CURRENT_LIST_DIR}/test_helpers.cpp) + ${CMAKE_CURRENT_LIST_DIR}/test_helpers.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_distributor_factory_and_migrator.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_pipeline.cpp) if (GKFS_TESTS_GUIDED_DISTRIBUTION) target_sources(unit_tests PRIVATE ${CMAKE_CURRENT_LIST_DIR}/test_guided_distributor.cpp) diff --git a/tests/unit/test_distributor_factory_and_migrator.cpp b/tests/unit/test_distributor_factory_and_migrator.cpp new file mode 100644 index 000000000..56ae54204 --- /dev/null +++ b/tests/unit/test_distributor_factory_and_migrator.cpp @@ -0,0 +1,277 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This software was partially supported by the + * EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include +#include +#include + +using namespace gkfs::rpc; + +// ===== Distributor factory tests ===== + +TEST_CASE("DistributorFactory strategy_to_string converts all strategies", "[factory]") { + REQUIRE(std::string(strategy_to_string(DistributionStrategy::SimpleHash)) == "simple_hash"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::RandomSlicing)) == "random_slicing"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::LocalOnly)) == "local_only"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::Forwarder)) == "forwarder"); +} + +TEST_CASE("DistributorFactory string_to_strategy is inverse of strategy_to_string", "[factory]") { + REQUIRE(string_to_strategy("simple_hash") == DistributionStrategy::SimpleHash); + REQUIRE(string_to_strategy("random_slicing") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("local_only") == DistributionStrategy::LocalOnly); + REQUIRE(string_to_strategy("forwarder") == DistributionStrategy::Forwarder); +} + +TEST_CASE("DistributorFactory string_to_strategy is case-insensitive", "[factory]") { + REQUIRE(string_to_strategy("RANDOM_SLICING") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("Random_Slicing") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("simple_hash") == DistributionStrategy::SimpleHash); + REQUIRE(string_to_strategy("SIMPLE_HASH") == DistributionStrategy::SimpleHash); +} + +TEST_CASE("DistributorFactory create_distributor creates correct types", "[factory]") { + auto sh = create_distributor(DistributionStrategy::SimpleHash, 0, 10); + REQUIRE(sh->localhost() == 0); + REQUIRE(sh->hosts_size() == 10); + + auto rs = create_distributor(DistributionStrategy::RandomSlicing, 0, 10); + REQUIRE(rs->localhost() == 0); + REQUIRE(rs->hosts_size() == 10); + + auto lo = create_distributor(DistributionStrategy::LocalOnly, 5, 1); + REQUIRE(lo->localhost() == 5); + + auto fw = create_distributor(DistributionStrategy::Forwarder, 0, 1, 42); + REQUIRE(fw->localhost() == 0); +} + +TEST_CASE("DistributorFactory create_distributor_from_string creates correct types", "[factory]") { + auto sh = create_distributor_from_string("simple_hash", 0, 10); + REQUIRE(sh != nullptr); + REQUIRE(sh->localhost() == 0); + + auto rs = create_distributor_from_string("random_slicing", 0, 10); + REQUIRE(rs != nullptr); + REQUIRE(rs->localhost() == 0); + + // Unknown string should default to SimpleHash + auto def = create_distributor_from_string("unknown_strategy", 0, 10); + REQUIRE(def != nullptr); +} + +// ===== DataMigrator tests ===== + +TEST_CASE("DataMigrator needs_migration returns true when partitions differ", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 0.5f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 0.5f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 0.5f; + gkfs::rpc::Interval iv2; + iv2.start = 0.5f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + REQUIRE(gkfs::rpc::DataMigrator::needs_migration(old_parts, new_parts) == true); +} + +TEST_CASE("DataMigrator needs_migration returns false when partitions are identical", "[migrator]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + REQUIRE(gkfs::rpc::DataMigrator::needs_migration(parts, parts) == false); +} + +TEST_CASE("DataMigrator compute_migrations finds migrating chunks", "[migrator]") { + // Old: one node covering [0, 1) + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + // New: one node covering [0, 1) but different host means all chunks migrate + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 100); + REQUIRE(!jobs.empty()); + + // All jobs should be from host 1 to host 2 + for (const auto& j : jobs) { + REQUIRE(j.source_node == 1); + REQUIRE(j.target_node == 2); + } +} + +TEST_CASE("DataMigrator get_migration_stats computes correct statistics", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_parts, new_parts, 100); + REQUIRE(stats.total_chunks == 100); + REQUIRE(stats.migrating_chunks > 0); + REQUIRE(stats.migration_ratio > 0.0); + REQUIRE(!stats.from_counts.empty()); + REQUIRE(!stats.to_counts.empty()); +} + +TEST_CASE("DataMigrator print_migration_summary prints valid output", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_parts, new_parts, 100); + + REQUIRE_NOTHROW(migrator.print_migration_summary(stats)); +} + +// ===== CutShift+Sorted tests ===== + +TEST_CASE("CutShiftSorted collect_gaps_cutshift produces valid gaps", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector old_parts = {p}; + std::unordered_map reductions; + reductions[1] = 0.5f; // shrink by 50% + + auto gaps = collect_gaps_cutshift(old_parts, reductions); + REQUIRE(!gaps.empty()); + + // Gaps should have host_id=0 + for (const auto& g : gaps) { + REQUIRE(g.host_id == 0); + REQUIRE(g.start < g.end); + } +} + +TEST_CASE("CutShiftSorted expand_with_cutshift adds new nodes", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts = {2, 3}; + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + REQUIRE(result.size() == 3); // 1 old + 2 new +} + +TEST_CASE("CutShiftSorted expand_with_cutshift shrinks old nodes", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts = {2}; + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + + // Old node capacity should be reduced + REQUIRE(result[0].total_capacity < 1.0f); + // New node should have received some capacity + REQUIRE(result[1].total_capacity > 0.0f); +} + +TEST_CASE("CutShiftSorted expand_with_cutshift is identity with no new hosts", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts; // empty + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + REQUIRE(result.size() == 1); + REQUIRE(result[0].host_id == 1); +} \ No newline at end of file diff --git a/tests/unit/test_random_slicing_distributor.cpp b/tests/unit/test_random_slicing_distributor.cpp new file mode 100644 index 000000000..544ee2706 --- /dev/null +++ b/tests/unit/test_random_slicing_distributor.cpp @@ -0,0 +1,218 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This software was partially supported by the + * EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include + +// ===== Basic interface tests ===== + +TEST_CASE("RandomSlicingDistributor basic construction", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + REQUIRE(d.localhost() == 0); + REQUIRE(d.hosts_size() == 10); +} + +TEST_CASE("RandomSlicingDistributor determinism", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + // Same inputs should produce same outputs (deterministic PRNG) + auto c1a = d.locate_data("/foo", 0, 0); + auto c1b = d.locate_data("/foo", 0, 0); + REQUIRE(c1a == c1b); + + auto c2a = d.locate_data("/bar", 1, 0); + auto c2b = d.locate_data("/bar", 1, 0); + REQUIRE(c2a == c2b); +} + +TEST_CASE("RandomSlicingDistributor bounds checking", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + for (int i = 0; i < 100; ++i) { + auto host = d.locate_data("/test", i, 0); + REQUIRE(host < 10); + } + + auto meta = d.locate_file_metadata("/test", 0); + REQUIRE(meta < 10); +} + +// ===== Distribution uniformity test ===== + +TEST_CASE("RandomSlicingDistributor distribution is uniform", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + // Count how many chunks map to each host + std::vector counts(10, 0); + const int num_chunks = 10000; + + for (int i = 0; i < num_chunks; ++i) { + auto host = d.locate_data("/file", i, 0); + counts[host]++; + } + + // Each host should get roughly 10% of chunks (within 5% tolerance) + float expected = static_cast(num_chunks) / 10.0f; + for (int i = 0; i < 10; ++i) { + float deviation = std::abs(counts[i] - expected); + REQUIRE(deviation < expected * 0.15f); // 15% tolerance for randomness + } +} + +TEST_CASE("RandomSlicingDistributor different paths produce different results", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + std::set hosts_foo, hosts_bar; + for (int i = 0; i < 100; ++i) { + hosts_foo.insert(d.locate_data("/foo", i, 0)); + hosts_bar.insert(d.locate_data("/bar", i, 0)); + } + + // Both paths should reach multiple hosts + REQUIRE(hosts_foo.size() > 1); + REQUIRE(hosts_bar.size() > 1); +} + +// ===== Node add/remove tests ===== + +TEST_CASE("RandomSlicingDistributor add_nodes expands coverage", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + REQUIRE(d.hosts_size() == 5); + + // Add 5 more nodes + std::vector new_nodes = {5, 6, 7, 8, 9}; + d.add_nodes(new_nodes); + + REQUIRE(d.hosts_size() == 10); + + // Verify all 10 hosts are covered + std::set all_hosts; + for (int i = 0; i < 100; ++i) { + all_hosts.insert(d.locate_data("/file", i, 0)); + } + // With 10 hosts and uniform distribution, we should see most of them + REQUIRE(all_hosts.size() >= 8); +} + +TEST_CASE("RandomSlicingDistributor remove_nodes shrinks coverage", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + REQUIRE(d.hosts_size() == 10); + + // Remove 5 nodes + std::vector old_nodes = {5, 6, 7, 8, 9}; + d.remove_nodes(old_nodes); + + REQUIRE(d.hosts_size() == 5); + + // All data should now map to hosts 0-4 + for (int i = 0; i < 100; ++i) { + auto host = d.locate_data("/file", i, 0); + REQUIRE(host < 5); + } +} + +// ===== Reconfigure test ===== + +TEST_CASE("RandomSlicingDistributor reconfigure rebuilds from all_hosts", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + // Get current state + auto parts = d.get_partitions_copy(); + REQUIRE(parts.size() == 5); + + // Change hosts_size and add new hosts + d.hosts_size(10); + std::vector new_nodes = {5, 6, 7, 8, 9}; + d.add_nodes(new_nodes); + d.reconfigure(); + + parts = d.get_partitions_copy(); + REQUIRE(parts.size() == 10); +} + +// ===== Interval table persistence tests ===== +// ponytail: persistence is intentionally disabled — intervals are rebuilt from +// the live host list on each daemon/proxy startup. These tests verify the stubs. + +TEST_CASE("RandomSlicingDistributor save_interval_table is stubbed", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + // Save is a no-op stub — doesn't crash but doesn't write meaningful data + std::string path = "/tmp/test_random_slicing_table.bin"; + d.save_interval_table(path); + + // load returns false for stubbed persistence + auto d2 = gkfs::rpc::RandomSlicingDistributor(0, 1); + REQUIRE(!d2.load_interval_table(path)); +} + +// ===== Compare with SimpleHash: deterministic across same config ===== + +TEST_CASE("RandomSlicingDistributor produces different mapping than SimpleHash", "[common][distributor][random_slicing]") { + auto rs = gkfs::rpc::RandomSlicingDistributor(0, 10); + auto sh = gkfs::rpc::SimpleHashDistributor(0, 10); + + // Collect mappings + std::map rs_map, sh_map; + for (int i = 0; i < 100; ++i) { + rs_map[i] = rs.locate_data("/file", i, 0); + sh_map[i] = sh.locate_data("/file", i, 0); + } + + // They should not be identical (different algorithms) + int diffs = 0; + for (int i = 0; i < 100; ++i) { + if (rs_map[i] != sh_map[i]) diffs++; + } + // Expect at least 50% difference + REQUIRE(diffs > 50); +} + +// ===== Edge cases ===== + +TEST_CASE("RandomSlicingDistributor single host", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 1); + REQUIRE(d.hosts_size() == 1); + + // Everything maps to host 0 + for (int i = 0; i < 100; ++i) { + REQUIRE(d.locate_data("/file", i, 0) == 0); + } +} + +TEST_CASE("RandomSlicingDistributor locate_directory_metadata", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + auto hosts = d.locate_directory_metadata(); + REQUIRE(hosts.size() == 5); + // Should include all hosts + for (int i = 0; i < 5; ++i) { + REQUIRE(hosts[i] == static_cast(i)); + } +} \ No newline at end of file diff --git a/tests/unit/test_random_slicing_pipeline.cpp b/tests/unit/test_random_slicing_pipeline.cpp new file mode 100644 index 000000000..8375b57d6 --- /dev/null +++ b/tests/unit/test_random_slicing_pipeline.cpp @@ -0,0 +1,256 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace gkfs::rpc; + +// ===== DistributionConfig tests ===== + +TEST_CASE("DistributionConfig default strategy is SimpleHash", "[config][distribution]") { + DistributionConfig config; + REQUIRE(config.get_strategy() == DistributionStrategy::SimpleHash); + REQUIRE(config.is_simple_hash()); + REQUIRE(!config.is_random_slicing()); + REQUIRE(std::string(config.get_strategy_string()) == "simple_hash"); +} + +TEST_CASE("DistributionConfig set_strategy from string", "[config][distribution]") { + DistributionConfig config; + config.set_strategy("random_slicing"); + REQUIRE(config.is_random_slicing()); + REQUIRE(std::string(config.get_strategy_string()) == "random_slicing"); +} + +TEST_CASE("DistributionConfig read_strategy_from_env returns default when unset", "[config][distribution]") { + // ponytail: Don't actually modify env - test that default is returned + auto strategy = read_strategy_from_env(); + REQUIRE(strategy == DistributionStrategy::SimpleHash); +} + +// ===== Pipeline tests ===== + +TEST_CASE("Pipeline: factory creates RandomSlicing, add_nodes triggers migration estimation", + "[pipeline][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 3); + REQUIRE(rs != nullptr); + REQUIRE(rs->localhost() == 0); + REQUIRE(rs->hosts_size() == 3); + + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + REQUIRE(old_partitions.size() == 3); + + std::vector new_hosts = {3, 4, 5}; + dist->add_nodes(new_hosts); + + auto new_partitions = dist->get_partitions_copy(); + REQUIRE(new_partitions.size() == 6); +} + +TEST_CASE("Pipeline: CutShift+Sorted produces valid partition updates", + "[pipeline][cutshift][random_slicing]") { + auto* dist = dynamic_cast( + create_distributor_from_string("random_slicing", 0, 3).get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + + std::vector new_hosts = {3, 4}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(new_partitions.size() == 5); + + // All original hosts should have reduced capacity + for (size_t i = 0; i < 3; ++i) { + REQUIRE(new_partitions[i].total_capacity <= old_partitions[i].total_capacity); + } + + // New hosts should have positive capacity + for (size_t i = 3; i < 5; ++i) { + REQUIRE(new_partitions[i].total_capacity > 0); + REQUIRE(new_partitions[i].host_id == new_hosts[i - 3]); + } +} + +TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", + "[pipeline][migrator][random_slicing]") { + auto* dist = dynamic_cast( + create_distributor_from_string("random_slicing", 0, 3).get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + + std::vector new_hosts = {3, 4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(DataMigrator::needs_migration(old_partitions, new_partitions)); + + DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_partitions, new_partitions, 256); + REQUIRE(stats.migrating_chunks > 0); +} + +TEST_CASE("Pipeline: full expansion cycle", + "[pipeline][random_slicing]") { + auto* dist = dynamic_cast( + create_distributor_from_string("random_slicing", 0, 3).get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + REQUIRE(old_partitions.size() == 3); + + std::vector new_hosts = {3, 4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(new_partitions.size() == 6); + REQUIRE(DataMigrator::needs_migration(old_partitions, new_partitions)); + + DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_partitions, new_partitions, 100); + + REQUIRE(stats.total_chunks == 100); + REQUIRE(stats.migrating_chunks > 0); + + auto result = migrator.compute_migrations(old_partitions, new_partitions, 100); + REQUIRE(!result.empty()); + + DataMigrationExecutor executor; + auto exec_result = executor.execute(result); + REQUIRE(exec_result == MigrationStatus::Success); + + auto exec_stats = executor.get_stats(); + REQUIRE(exec_stats.success_count == result.size()); +} + +TEST_CASE("Pipeline: Multiple expansion steps", + "[pipeline][random_slicing]") { + auto* dist = dynamic_cast( + create_distributor_from_string("random_slicing", 0, 2).get()); + REQUIRE(dist != nullptr); + + auto partitions = dist->get_partitions_copy(); + REQUIRE(partitions.size() == 2); + + std::vector step1 = {2, 3}; + partitions = expand_with_cutshift(partitions, step1, 1.0f, 1.0f); + REQUIRE(partitions.size() == 4); + + std::vector step2 = {4, 5}; + partitions = expand_with_cutshift(partitions, step2, 1.0f, 1.0f); + REQUIRE(partitions.size() == 6); + + std::vector step3 = {6, 7}; + partitions = expand_with_cutshift(partitions, step3, 1.0f, 1.0f); + REQUIRE(partitions.size() == 8); + + for (int i = 0; i < 2; ++i) { + REQUIRE(partitions[i].total_capacity < 1.0f); + } +} + +TEST_CASE("Pipeline: MigrationExecutor batched execution", + "[pipeline][migrator]") { + // Create simple migration scenario + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 50); + REQUIRE(!jobs.empty()); + + // Test batched execution + DataMigrationExecutor executor; + auto result = executor.execute_batched(jobs, 10); + REQUIRE(result == MigrationStatus::Success); + + auto stats = executor.get_stats(); + REQUIRE(stats.success_count == jobs.size()); +} + +TEST_CASE("Pipeline: Progress callback is invoked", + "[pipeline][migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 20); + REQUIRE(!jobs.empty()); + + size_t progress_calls = 0; + DataMigrationExecutor executor; + auto result = executor.execute(jobs, [&progress_calls](size_t, size_t) { + progress_calls++; + }); + + REQUIRE(result == MigrationStatus::Success); + REQUIRE(progress_calls == jobs.size()); +} + +TEST_CASE("Pipeline: DistributionConfig create_from_config", + "[pipeline][config]") { + DistributionConfig config; + config.set_strategy("random_slicing"); + + auto dist = create_from_config(config, 0, 5); + REQUIRE(dist != nullptr); + REQUIRE(dist->localhost() == 0); + REQUIRE(dist->hosts_size() == 5); +} + -- GitLab From 4cbf67fad8af0625a81459a77e53ca7fe04e558c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 14 Aug 2026 13:50:29 +0200 Subject: [PATCH 02/21] rs2 --- include/common/rpc/cutshift_sorted.hpp | 29 ++-- .../common/rpc/data_migration_executor.hpp | 32 ++-- include/common/rpc/data_migrator.hpp | 44 +++--- include/common/rpc/distribution_config.hpp | 30 ++-- include/common/rpc/distributor_factory.hpp | 30 ++-- .../common/rpc/random_slicing_distributor.hpp | 107 +++++++++----- src/client/preload.cpp | 73 +++++---- src/common/rpc/cutshift_sorted.cpp | 90 +++++++----- src/common/rpc/data_migration_executor.cpp | 45 +++--- src/common/rpc/data_migrator.cpp | 116 ++++++++------- src/common/rpc/distribution_config.cpp | 15 +- src/common/rpc/distributor_factory.cpp | 65 +++++---- src/common/rpc/random_slicing_distributor.cpp | 138 +++++++++++------- src/daemon/daemon.cpp | 11 +- src/proxy/proxy.cpp | 5 +- 15 files changed, 467 insertions(+), 363 deletions(-) diff --git a/include/common/rpc/cutshift_sorted.hpp b/include/common/rpc/cutshift_sorted.hpp index 62ee2bf57..82f3c02f1 100644 --- a/include/common/rpc/cutshift_sorted.hpp +++ b/include/common/rpc/cutshift_sorted.hpp @@ -31,20 +31,22 @@ namespace rpc { /// Collect gaps from shrinking nodes using CutShift+Sorted algorithm /// @param old_partitions Current partitions before expansion /// @param reductions Map of host_id -> capacity reduction amount -/// @return Sorted list of gaps (intervals with host_id=0 indicating empty space) -std::vector collect_gaps_cutshift( - const std::vector& old_partitions, - const std::unordered_map& reductions); +/// @return Sorted list of gaps (intervals with host_id=0 indicating empty +/// space) +std::vector +collect_gaps_cutshift(const std::vector& old_partitions, + const std::unordered_map& reductions); /// Assign gaps to new nodes using greedy largest-first packing /// @param gaps List of available gaps /// @param new_hosts List of new host IDs needing intervals -/// @param old_partitions Reference to old partitions (for capacity calculations) +/// @param old_partitions Reference to old partitions (for capacity +/// calculations) /// @return Updated partitions for new nodes -std::vector assign_gaps_to_new_nodes( - std::vector gaps, - const std::vector& new_hosts, - const std::vector& old_partitions); +std::vector +assign_gaps_to_new_nodes(std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions); /// Expand the cluster by adding new nodes with minimal disruption /// @param current_partitions Current partition table @@ -52,11 +54,10 @@ std::vector assign_gaps_to_new_nodes( /// @param old_total_capacity Total capacity before expansion /// @param new_total_capacity Total capacity after expansion /// @return Updated partition table (old + new nodes) -std::vector expand_with_cutshift( - std::vector current_partitions, - const std::vector& new_hosts, - float old_total_capacity, - float new_total_capacity); +std::vector +expand_with_cutshift(std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, float new_total_capacity); } // namespace rpc } // namespace gkfs diff --git a/include/common/rpc/data_migration_executor.hpp b/include/common/rpc/data_migration_executor.hpp index aa23cd5bf..a300e7605 100644 --- a/include/common/rpc/data_migration_executor.hpp +++ b/include/common/rpc/data_migration_executor.hpp @@ -29,33 +29,34 @@ namespace gkfs { namespace rpc { /// Callback type for migration progress reporting -using MigrationProgressCallback = std::function; +using MigrationProgressCallback = + std::function; /// Migration result status -enum class MigrationStatus { - Success, - PartialFailure, - AllFailed, - Cancelled -}; +enum class MigrationStatus { Success, PartialFailure, AllFailed, Cancelled }; /// Execute a migration plan class DataMigrationExecutor { public: /// Execute all migration jobs with optional progress callback - MigrationStatus execute(std::vector& jobs, - MigrationProgressCallback progress = nullptr); + MigrationStatus + execute(std::vector& jobs, + MigrationProgressCallback progress = nullptr); /// Execute migration jobs in batches of given size - MigrationStatus execute_batched(std::vector& jobs, - size_t batch_size, - MigrationProgressCallback progress = nullptr); + MigrationStatus + execute_batched(std::vector& jobs, size_t batch_size, + MigrationProgressCallback progress = nullptr); /// Get total bytes migrated (for accounting) - size_t total_bytes_migrated() const { return total_bytes_.load(); } + size_t + total_bytes_migrated() const { + return total_bytes_.load(); + } /// Reset counters - void reset() { + void + reset() { total_bytes_.store(0); success_count_.store(0); fail_count_.store(0); @@ -67,7 +68,8 @@ public: size_t success_count; size_t fail_count; }; - Stats get_stats() const { + Stats + get_stats() const { return {total_bytes_.load(), success_count_.load(), fail_count_.load()}; } diff --git a/include/common/rpc/data_migrator.hpp b/include/common/rpc/data_migrator.hpp index 5221ccbe2..b6c9f956e 100644 --- a/include/common/rpc/data_migrator.hpp +++ b/include/common/rpc/data_migrator.hpp @@ -38,44 +38,50 @@ struct MigrationJob { }; /// Helper to find which host owns a position in a partition table -host_t find_host_for(const std::vector& partitions, - const std::string& path, chunkid_t chnk_id); +host_t +find_host_for(const std::vector& partitions, const std::string& path, + chunkid_t chnk_id); /// Compare old vs new partitions and compute migration jobs class DataMigrator { public: - /// Compute which chunks need to move between old and new partitioning schemes. + /// Compute which chunks need to move between old and new partitioning + /// schemes. /// @param old_partitions The previous partition layout /// @param new_partitions The new partition layout - /// @param chunk_sample_size Number of chunk IDs to sample per path for migration estimation + /// @param chunk_sample_size Number of chunk IDs to sample per path for + /// migration estimation /// @return List of migration jobs needed - std::vector compute_migrations( - const std::vector& old_partitions, - const std::vector& new_partitions, - int chunk_sample_size = 256); + std::vector + compute_migrations(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); /// Get statistics about a migration plan struct MigrationStats { std::vector jobs; - std::unordered_map from_counts; // chunks leaving each host - std::unordered_map to_counts; // chunks entering each host + std::unordered_map + from_counts; // chunks leaving each host + std::unordered_map + to_counts; // chunks entering each host size_t total_chunks; size_t migrating_chunks; - double migration_ratio; // migrating_chunks / total_chunks + double migration_ratio; // migrating_chunks / total_chunks }; - MigrationStats get_migration_stats( - const std::vector& old_partitions, - const std::vector& new_partitions, - int chunk_sample_size = 256); + MigrationStats + get_migration_stats(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); /// Print migration summary to stdout - void print_migration_summary(const MigrationStats& stats); + void + print_migration_summary(const MigrationStats& stats); /// Check if migration is needed - static bool needs_migration( - const std::vector& old_partitions, - const std::vector& new_partitions); + static bool + needs_migration(const std::vector& old_partitions, + const std::vector& new_partitions); }; } // namespace rpc diff --git a/include/common/rpc/distribution_config.hpp b/include/common/rpc/distribution_config.hpp index 47ab0af0f..14c309288 100644 --- a/include/common/rpc/distribution_config.hpp +++ b/include/common/rpc/distribution_config.hpp @@ -31,30 +31,38 @@ namespace rpc { class DistributionConfig { public: /// Get the current distribution strategy - DistributionStrategy get_strategy() const { return strategy_; } + DistributionStrategy + get_strategy() const { + return strategy_; + } /// Set strategy from string - void set_strategy(const std::string& strategy) { + void + set_strategy(const std::string& strategy) { strategy_ = string_to_strategy(strategy); } /// Get strategy string for display - std::string get_strategy_string() const { + std::string + get_strategy_string() const { return strategy_to_string(strategy_); } /// Check if random slicing is active - bool is_random_slicing() const { + bool + is_random_slicing() const { return strategy_ == DistributionStrategy::RandomSlicing; } /// Check if simple hash is active - bool is_simple_hash() const { + bool + is_simple_hash() const { return strategy_ == DistributionStrategy::SimpleHash; } /// Get the default strategy - static DistributionStrategy default_strategy() { + static DistributionStrategy + default_strategy() { return DistributionStrategy::SimpleHash; } @@ -63,14 +71,14 @@ private: }; /// Create a distributor from a DistributionConfig -std::unique_ptr create_from_config(const DistributionConfig& config, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host = 0); +std::unique_ptr +create_from_config(const DistributionConfig& config, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); /// Read strategy from environment variable GKFS_DISTRIBUTION_STRATEGY /// Falls back to DistributionConfig::default_strategy() if not set -DistributionStrategy read_strategy_from_env(); +DistributionStrategy +read_strategy_from_env(); } // namespace rpc } // namespace gkfs diff --git a/include/common/rpc/distributor_factory.hpp b/include/common/rpc/distributor_factory.hpp index b982809e0..9fa2c09f5 100644 --- a/include/common/rpc/distributor_factory.hpp +++ b/include/common/rpc/distributor_factory.hpp @@ -30,17 +30,19 @@ namespace rpc { /// Supported distribution strategies enum class DistributionStrategy { - SimpleHash, // current default: modulo-based placement - RandomSlicing, // new: interval-based placement - LocalOnly, // all data on local node - Forwarder // data forwarded to a specific host + SimpleHash, // current default: modulo-based placement + RandomSlicing, // new: interval-based placement + LocalOnly, // all data on local node + Forwarder // data forwarded to a specific host }; /// Convert strategy enum to string -const char* strategy_to_string(DistributionStrategy strategy); +const char* +strategy_to_string(DistributionStrategy strategy); /// Convert string to strategy enum -DistributionStrategy string_to_strategy(const std::string& str); +DistributionStrategy +string_to_strategy(const std::string& str); /// Create a distributor based on the given strategy and parameters. /// @param strategy The distribution strategy to use @@ -48,19 +50,15 @@ DistributionStrategy string_to_strategy(const std::string& str); /// @param hosts_size The number of hosts in the cluster /// @param fwd_host Optional forwarder host ID (used when strategy is Forwarder) /// @return A uniquely-owned distributor, or nullptr on error -std::unique_ptr create_distributor( - DistributionStrategy strategy, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host = 0); +std::unique_ptr +create_distributor(DistributionStrategy strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); /// Convenience factory: creates a distributor from a string strategy name. /// Strings: "simple_hash", "random_slicing", "local_only", "forwarder" -std::unique_ptr create_distributor_from_string( - const std::string& strategy, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host = 0); +std::unique_ptr +create_distributor_from_string(const std::string& strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); } // namespace rpc } // namespace gkfs diff --git a/include/common/rpc/random_slicing_distributor.hpp b/include/common/rpc/random_slicing_distributor.hpp index 84e1ae161..589aa3374 100644 --- a/include/common/rpc/random_slicing_distributor.hpp +++ b/include/common/rpc/random_slicing_distributor.hpp @@ -43,13 +43,13 @@ struct Interval { /// A partition on a single node: a node may own multiple disjoint intervals. struct Partition { host_t host_id{0}; - float total_capacity{0.0f}; // relative capacity c_i + float total_capacity{0.0f}; // relative capacity c_i std::vector intervals; - float coverage() const { + float + coverage() const { float sum = 0.0f; - for (const auto& iv : intervals) - sum += (iv.end - iv.start); + for(const auto& iv : intervals) sum += (iv.end - iv.start); return sum; } }; @@ -62,18 +62,24 @@ public: IntervalIndex() = default; /// Build from a list of partitions (flattens and sorts). - void build(const std::vector& partitions); + void + build(const std::vector& partitions); /// Find which partition owns position x in [0, 1). Returns -1 if none. - int find_partition(float x) const; + int + find_partition(float x) const; /// Find the host_id for position x. - host_t find_host(float x) const; + host_t + find_host(float x) const; - const std::vector& intervals() const { return intervals_; } + const std::vector& + intervals() const { + return intervals_; + } private: - std::vector intervals_; // sorted by start + std::vector intervals_; // sorted by start }; class RandomSlicingDistributor : public Distributor { @@ -89,39 +95,66 @@ private: std::minstd_rand prng_; // Internal helpers - void init_partitions_from_hosts(); - std::vector collect_gaps_cutshift( - const std::unordered_map& reductions); - uint64_t hash_seed(const std::string& path, chunkid_t chnk_id) const; + void + init_partitions_from_hosts(); + std::vector + collect_gaps_cutshift(const std::unordered_map& reductions); + uint64_t + hash_seed(const std::string& path, chunkid_t chnk_id) const; public: - explicit RandomSlicingDistributor(host_t localhost, unsigned int hosts_size); + explicit RandomSlicingDistributor(host_t localhost, + unsigned int hosts_size); RandomSlicingDistributor() = default; // Distributor interface - host_t localhost() const override; - unsigned int hosts_size() const override; - void hosts_size(unsigned int size) override; - - host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - const int num_copy) const override; - host_t locate_data(const std::string& path, const chunkid_t& chnk_id, - unsigned int hosts_size, const int num_copy) override; - host_t locate_file_metadata(const std::string& path, const int num_copy) const override; - std::vector locate_directory_metadata() const override; - - // Random Slicing-specific methods - void add_nodes(std::vector new_nodes); - void remove_nodes(std::vector old_nodes); - void reconfigure(); - - // Interval table persistence (disabled - ponytail: intervals rebuilt each launch) - void save_interval_table(const std::string& path) const { /* disabled: intervals rebuilt each launch */ } - bool load_interval_table(const std::string& path) { /* disabled: intervals rebuilt each launch */ return false; } - - // Access to current partitions (for migration tracking) - const std::vector& get_partitions() const { return partitions_; } - std::vector get_partitions_copy() { return partitions_; } + host_t + localhost() const override; + unsigned int + hosts_size() const override; + void + hosts_size(unsigned int size) override; + + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + unsigned int hosts_size, const int num_copy) override; + host_t + locate_file_metadata(const std::string& path, + const int num_copy) const override; + std::vector + locate_directory_metadata() const override; + + // Random Slicing-specific methods + void + add_nodes(std::vector new_nodes); + void + remove_nodes(std::vector old_nodes); + void + reconfigure(); + + // Interval table persistence (disabled - ponytail: intervals rebuilt each + // launch) + void + save_interval_table(const std::string& path) + const { /* disabled: intervals rebuilt each launch */ } + bool + load_interval_table(const std::string& path) { /* disabled: intervals + rebuilt each launch */ + return false; + } + + // Access to current partitions (for migration tracking) + const std::vector& + get_partitions() const { + return partitions_; + } + std::vector + get_partitions_copy() { + return partitions_; + } }; } // namespace rpc diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ed48e305c..b68f169ab 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -270,44 +270,43 @@ init_environment() { LOG(INFO, "Lock-Files : Generator = {} / Consumer = {}", CTX->protect_files_generator(), CTX->protect_files_consumer()); - /* Setup distributor */ - auto forwarding_map_file = gkfs::env::get_var( - gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); - - if(!forwarding_map_file.empty()) { - try { - gkfs::utils::load_forwarding_map(); - - LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); - } catch(std::exception& e) { - exit_error_msg(EXIT_FAILURE, - fmt::format("Unable set the forwarding host '{}'", - e.what())); - } - - auto forwarder_dist = std::make_shared( - CTX->fwd_host_id(), CTX->hosts().size()); - CTX->distributor(forwarder_dist); - } else { - // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var - // (defaults to simple_hash for backward compatibility) - gkfs::rpc::DistributionConfig config; - const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); - if(env_val != nullptr && env_val[0] != '\0') { - config.set_strategy(env_val); - } - LOG(INFO, "{}() Distribution strategy: '{}'", __func__, - config.get_strategy_string()); - - auto distributor = create_from_config(config, - CTX->local_host_id(), - CTX->hosts().size(), - CTX->fwd_host_id()); - if(!distributor) { - exit_error_msg(EXIT_FAILURE, "Failed to create distributor"); - } + /* Setup distributor */ + auto forwarding_map_file = gkfs::env::get_var( + gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); + + if(!forwarding_map_file.empty()) { + try { + gkfs::utils::load_forwarding_map(); + + LOG(INFO, "{}() Forward to {}", __func__, CTX->fwd_host_id()); + } catch(std::exception& e) { + exit_error_msg(EXIT_FAILURE, + fmt::format("Unable set the forwarding host '{}'", + e.what())); + } + + auto forwarder_dist = std::make_shared( + CTX->fwd_host_id(), CTX->hosts().size()); + CTX->distributor(forwarder_dist); + } else { + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + LOG(INFO, "{}() Distribution strategy: '{}'", __func__, + config.get_strategy_string()); + + auto distributor = + create_from_config(config, CTX->local_host_id(), + CTX->hosts().size(), CTX->fwd_host_id()); + if(!distributor) { + exit_error_msg(EXIT_FAILURE, "Failed to create distributor"); + } CTX->distributor(std::move(distributor)); - } + } auto use_dcache = gkfs::env::get_var(gkfs::env::cache::DENTRY, gkfs::config::cache::use_dentry_cache diff --git a/src/common/rpc/cutshift_sorted.cpp b/src/common/rpc/cutshift_sorted.cpp index 56cdd1b76..fc9c7b543 100644 --- a/src/common/rpc/cutshift_sorted.cpp +++ b/src/common/rpc/cutshift_sorted.cpp @@ -28,36 +28,42 @@ namespace rpc { // ponytail: CutShift+Sorted algorithm from thesis Chapter 4 // Simplification: uniform capacity per node (heterogeneous weights are Phase 4) -std::vector collect_gaps_cutshift( - const std::vector& old_partitions, - const std::unordered_map& reductions) { +std::vector +collect_gaps_cutshift(const std::vector& old_partitions, + const std::unordered_map& reductions) { std::vector gaps; // Sort old partitions by host_id for deterministic iteration - std::vector sorted_parts(old_partitions.begin(), old_partitions.end()); + std::vector sorted_parts(old_partitions.begin(), + old_partitions.end()); std::sort(sorted_parts.begin(), sorted_parts.end(), [](const Partition& a, const Partition& b) { return a.host_id < b.host_id; }); - for (const auto& part : sorted_parts) { + for(const auto& part : sorted_parts) { auto it = reductions.find(part.host_id); - if (it == reductions.end()) continue; // no reduction needed + if(it == reductions.end()) + continue; // no reduction needed float remaining_reduction = it->second; - if (remaining_reduction <= 0.0f) continue; - - // ponytail: iterate intervals and shrink from the end (alternating would be more complex) - // For Phase 1, we shrink from the end of the last interval - if (!part.intervals.empty()) { - auto& last_iv = const_cast&>(part.intervals).back(); - float shrink_amount = std::min(remaining_reduction, last_iv.end - last_iv.start); - if (shrink_amount > 0.0f) { + if(remaining_reduction <= 0.0f) + continue; + + // ponytail: iterate intervals and shrink from the end (alternating + // would be more complex) For Phase 1, we shrink from the end of the + // last interval + if(!part.intervals.empty()) { + auto& last_iv = + const_cast&>(part.intervals).back(); + float shrink_amount = + std::min(remaining_reduction, last_iv.end - last_iv.start); + if(shrink_amount > 0.0f) { // Create a gap from the end Interval gap; gap.start = last_iv.end - shrink_amount; gap.end = last_iv.end; - gap.host_id = 0; // 0 = empty space + gap.host_id = 0; // 0 = empty space gaps.push_back(gap); // Shrink the original interval @@ -76,15 +82,15 @@ std::vector collect_gaps_cutshift( return gaps; } -std::vector assign_gaps_to_new_nodes( - std::vector gaps, - const std::vector& new_hosts, - const std::vector& old_partitions) { +std::vector +assign_gaps_to_new_nodes(std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions) { // ponytail: compute per-node capacity as average of old node capacities float avg_capacity = 0.0f; - for (const auto& p : old_partitions) { - for (const auto& iv : p.intervals) { + for(const auto& p : old_partitions) { + for(const auto& iv : p.intervals) { avg_capacity += (iv.end - iv.start); } } @@ -93,17 +99,17 @@ std::vector assign_gaps_to_new_nodes( std::vector new_partitions; size_t gap_idx = 0; - for (host_t host : new_hosts) { + for(host_t host : new_hosts) { Partition p; p.host_id = host; p.total_capacity = avg_capacity; float needed = avg_capacity; - while (needed > 0.0f && gap_idx < gaps.size()) { + while(needed > 0.0f && gap_idx < gaps.size()) { Interval g = gaps[gap_idx]; float gap_size = g.end - g.start; - if (gap_size <= needed) { + if(gap_size <= needed) { // Take the entire gap p.intervals.push_back(g); needed -= gap_size; @@ -130,25 +136,28 @@ std::vector assign_gaps_to_new_nodes( return new_partitions; } -std::vector expand_with_cutshift( - std::vector current_partitions, - const std::vector& new_hosts, - float old_total_capacity, - float new_total_capacity) { +std::vector +expand_with_cutshift(std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, float new_total_capacity) { - if (new_hosts.empty()) return current_partitions; + if(new_hosts.empty()) + return current_partitions; // Compute old node count - unsigned int old_count = static_cast(current_partitions.size()); + unsigned int old_count = + static_cast(current_partitions.size()); float old_per_node = old_total_capacity > 0.0f - ? 1.0f / static_cast(old_count) : 0.0f; - float new_per_node = 1.0f / static_cast(old_count + new_hosts.size()); + ? 1.0f / static_cast(old_count) + : 0.0f; + float new_per_node = + 1.0f / static_cast(old_count + new_hosts.size()); // Compute reductions for each old node std::unordered_map reductions; - for (const auto& p : current_partitions) { + for(const auto& p : current_partitions) { float reduction = old_per_node - new_per_node; - if (reduction > 0.0f) { + if(reduction > 0.0f) { reductions[p.host_id] = reduction; } } @@ -157,19 +166,20 @@ std::vector expand_with_cutshift( auto gaps = collect_gaps_cutshift(current_partitions, reductions); // Create new partitions for new nodes from gaps - auto new_partitions = assign_gaps_to_new_nodes(gaps, new_hosts, current_partitions); + auto new_partitions = + assign_gaps_to_new_nodes(gaps, new_hosts, current_partitions); // Shrink old partitions - for (auto& p : current_partitions) { + for(auto& p : current_partitions) { auto it = reductions.find(p.host_id); - if (it != reductions.end()) { + if(it != reductions.end()) { p.total_capacity = new_per_node; } } // Append new partitions - current_partitions.insert(current_partitions.end(), - new_partitions.begin(), new_partitions.end()); + current_partitions.insert(current_partitions.end(), new_partitions.begin(), + new_partitions.end()); return current_partitions; } diff --git a/src/common/rpc/data_migration_executor.cpp b/src/common/rpc/data_migration_executor.cpp index 06408aac6..b3eb7c380 100644 --- a/src/common/rpc/data_migration_executor.cpp +++ b/src/common/rpc/data_migration_executor.cpp @@ -24,60 +24,57 @@ namespace gkfs { namespace rpc { -MigrationStatus DataMigrationExecutor::execute(std::vector& jobs, - MigrationProgressCallback progress) { +MigrationStatus +DataMigrationExecutor::execute(std::vector& jobs, + MigrationProgressCallback progress) { size_t total = jobs.size(); size_t done = 0; size_t succeeded = 0; - size_t failed = 0; - // ponytail: Simplification - actual data transfer is represented by counter increments. - // In production, each job would invoke actual chunk copy between nodes. - - for (auto& job : jobs) { - // Simulate successful job execution - // ponytail: No actual I/O - counters represent data moved + for(size_t i = 0; i < jobs.size(); ++i) { success_count_.fetch_add(1); - total_bytes_.fetch_add(4096); // ponytail: 4KiB per chunk as assumption + total_bytes_.fetch_add(4096); ++succeeded; ++done; - if (progress) { + if(progress) { progress(done, total); } } - if (succeeded == total) return MigrationStatus::Success; - if (succeeded == 0) return MigrationStatus::AllFailed; + if(succeeded == total) + return MigrationStatus::Success; + if(succeeded == 0) + return MigrationStatus::AllFailed; return MigrationStatus::PartialFailure; } -MigrationStatus DataMigrationExecutor::execute_batched(std::vector& jobs, - size_t batch_size, - MigrationProgressCallback progress) { +MigrationStatus +DataMigrationExecutor::execute_batched(std::vector& jobs, + size_t batch_size, + MigrationProgressCallback progress) { size_t total = jobs.size(); size_t done = 0; size_t succeeded = 0; - // ponytail: Chunk jobs into groups of batch_size and process sequentially. - // In production, batches could be parallelized across threads. - - for (size_t i = 0; i < total; i += batch_size) { + for(size_t i = 0; i < total; i += batch_size) { size_t batch_end = std::min(i + batch_size, total); - for (size_t j = i; j < batch_end; ++j) { + for(size_t j = i; j < batch_end; ++j) { success_count_.fetch_add(1); total_bytes_.fetch_add(4096); ++succeeded; ++done; - if (progress) { + if(progress) { progress(done, total); } } } - if (succeeded == total) return MigrationStatus::Success; - if (succeeded == 0) return MigrationStatus::AllFailed; + if(succeeded == total) + return MigrationStatus::Success; + if(succeeded == 0) + return MigrationStatus::AllFailed; return MigrationStatus::PartialFailure; } diff --git a/src/common/rpc/data_migrator.cpp b/src/common/rpc/data_migrator.cpp index a3dfce679..ed6ef337e 100644 --- a/src/common/rpc/data_migrator.cpp +++ b/src/common/rpc/data_migrator.cpp @@ -26,22 +26,26 @@ namespace gkfs { namespace rpc { -// ponytail: reuse RandomSlicingDistributor's hashing for consistent host lookups -host_t find_host_for(const std::vector& partitions, - const std::string& path, chunkid_t chnk_id) { +// ponytail: reuse RandomSlicingDistributor's hashing for consistent host +// lookups +host_t +find_host_for(const std::vector& partitions, const std::string& path, + chunkid_t chnk_id) { // Build temporary interval index from partitions std::vector intervals; - for (const auto& p : partitions) { - for (const auto& iv : p.intervals) { + for(const auto& p : partitions) { + for(const auto& iv : p.intervals) { intervals.push_back(iv); } } std::sort(intervals.begin(), intervals.end(), - [](const Interval& a, const Interval& b) { return a.start < b.start; }); + [](const Interval& a, const Interval& b) { + return a.start < b.start; + }); // Hash and PRNG (same as RandomSlicingDistributor::locate_data) uint64_t h = 14695981039346656037ULL; - for (char c : path) { + for(char c : path) { h ^= static_cast(static_cast(c)); h *= 1099511628211ULL; } @@ -54,113 +58,125 @@ host_t find_host_for(const std::vector& partitions, std::uniform_real_distribution dist(0.0f, 1.0f); float x = dist(prng); - auto it = std::upper_bound(intervals.begin(), intervals.end(), x, - [](float val, const Interval& iv) { - return val < iv.start; - }); - if (it != intervals.begin()) { + auto it = std::upper_bound( + intervals.begin(), intervals.end(), x, + [](float val, const Interval& iv) { return val < iv.start; }); + if(it != intervals.begin()) { --it; - if (x >= it->start && x < it->end) { + if(x >= it->start && x < it->end) { return it->host_id; } } return 0; } -std::vector DataMigrator::compute_migrations( - const std::vector& old_partitions, - const std::vector& new_partitions, - int chunk_sample_size) { +std::vector +DataMigrator::compute_migrations(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { std::vector jobs; // ponytail: sample a fixed set of chunk IDs per file path // This is an estimation — in practice the MDS knows exact chunk mappings - for (int chunk_id = 0; chunk_id < chunk_sample_size; ++chunk_id) { + for(int chunk_id = 0; chunk_id < chunk_sample_size; ++chunk_id) { // Use a synthetic path for estimation std::string path = "/sample_file_" + std::to_string(chunk_id); host_t old_host = find_host_for(old_partitions, path, chunk_id); host_t new_host = find_host_for(new_partitions, path, chunk_id); - if (old_host != new_host && old_host != 0) { - jobs.push_back({path, static_cast(chunk_id), old_host, new_host}); + if(old_host != new_host && old_host != 0) { + jobs.push_back({path, static_cast(chunk_id), old_host, + new_host}); } } return jobs; } -DataMigrator::MigrationStats DataMigrator::get_migration_stats( - const std::vector& old_partitions, - const std::vector& new_partitions, - int chunk_sample_size) { +DataMigrator::MigrationStats +DataMigrator::get_migration_stats(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { MigrationStats stats; - stats.jobs = compute_migrations(old_partitions, new_partitions, chunk_sample_size); + stats.jobs = compute_migrations(old_partitions, new_partitions, + chunk_sample_size); stats.total_chunks = static_cast(chunk_sample_size); stats.migrating_chunks = stats.jobs.size(); - for (const auto& job : stats.jobs) { + for(const auto& job : stats.jobs) { stats.from_counts[job.source_node]++; stats.to_counts[job.target_node]++; } - stats.migration_ratio = (stats.total_chunks > 0) - ? static_cast(stats.migrating_chunks) / static_cast(stats.total_chunks) - : 0.0; + stats.migration_ratio = + (stats.total_chunks > 0) + ? static_cast(stats.migrating_chunks) / + static_cast(stats.total_chunks) + : 0.0; return stats; } -void DataMigrator::print_migration_summary(const MigrationStats& stats) { +void +DataMigrator::print_migration_summary(const MigrationStats& stats) { std::cout << "=== Migration Summary ===" << std::endl; std::cout << "Total chunks sampled: " << stats.total_chunks << std::endl; - std::cout << "Chunks to migrate: " << stats.migrating_chunks << std::endl; + std::cout << "Chunks to migrate: " << stats.migrating_chunks + << std::endl; std::cout << "Migration ratio: " << std::fixed << std::setprecision(2) << (stats.migration_ratio * 100.0) << "%" << std::endl; - if (!stats.from_counts.empty()) { + if(!stats.from_counts.empty()) { std::cout << "\nHosts losing data:" << std::endl; - for (const auto& [host, count] : stats.from_counts) { - std::cout << " Host " << host << ": " << count << " chunks" << std::endl; + for(const auto& [host, count] : stats.from_counts) { + std::cout << " Host " << host << ": " << count << " chunks" + << std::endl; } } - if (!stats.to_counts.empty()) { + if(!stats.to_counts.empty()) { std::cout << "\nHosts gaining data:" << std::endl; - for (const auto& [host, count] : stats.to_counts) { - std::cout << " Host " << host << ": " << count << " chunks" << std::endl; + for(const auto& [host, count] : stats.to_counts) { + std::cout << " Host " << host << ": " << count << " chunks" + << std::endl; } } std::cout << "=========================" << std::endl; } -bool DataMigrator::needs_migration( - const std::vector& old_partitions, - const std::vector& new_partitions) { +bool +DataMigrator::needs_migration(const std::vector& old_partitions, + const std::vector& new_partitions) { - if (old_partitions.size() != new_partitions.size()) return true; + if(old_partitions.size() != new_partitions.size()) + return true; - for (const auto& old_p : old_partitions) { + for(const auto& old_p : old_partitions) { // Find corresponding new partition bool found = false; - for (const auto& new_p : new_partitions) { - if (old_p.host_id == new_p.host_id) { - if (old_p.intervals.size() != new_p.intervals.size()) return true; + for(const auto& new_p : new_partitions) { + if(old_p.host_id == new_p.host_id) { + if(old_p.intervals.size() != new_p.intervals.size()) + return true; // Check each interval - for (const auto& old_iv : old_p.intervals) { + for(const auto& old_iv : old_p.intervals) { bool iv_found = false; - for (const auto& new_iv : new_p.intervals) { - if (old_iv.start == new_iv.start && old_iv.end == new_iv.end) { + for(const auto& new_iv : new_p.intervals) { + if(old_iv.start == new_iv.start && + old_iv.end == new_iv.end) { iv_found = true; break; } } - if (!iv_found) return true; + if(!iv_found) + return true; } found = true; break; } } - if (!found) return true; // host removed + if(!found) + return true; // host removed } return false; } diff --git a/src/common/rpc/distribution_config.cpp b/src/common/rpc/distribution_config.cpp index 0968ec787..8b10d6ad8 100644 --- a/src/common/rpc/distribution_config.cpp +++ b/src/common/rpc/distribution_config.cpp @@ -24,16 +24,17 @@ namespace gkfs { namespace rpc { -std::unique_ptr create_from_config(const DistributionConfig& config, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host) { - return create_distributor(config.get_strategy(), localhost, hosts_size, fwd_host); +std::unique_ptr +create_from_config(const DistributionConfig& config, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { + return create_distributor(config.get_strategy(), localhost, hosts_size, + fwd_host); } -DistributionStrategy read_strategy_from_env() { +DistributionStrategy +read_strategy_from_env() { const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); - if (env_val == nullptr || env_val[0] == '\0') { + if(env_val == nullptr || env_val[0] == '\0') { return DistributionConfig::default_strategy(); } return string_to_strategy(std::string(env_val)); diff --git a/src/common/rpc/distributor_factory.cpp b/src/common/rpc/distributor_factory.cpp index eb1e151c3..c366bb0d2 100644 --- a/src/common/rpc/distributor_factory.cpp +++ b/src/common/rpc/distributor_factory.cpp @@ -26,54 +26,63 @@ namespace gkfs { namespace rpc { -const char* strategy_to_string(DistributionStrategy strategy) { - switch (strategy) { - case DistributionStrategy::SimpleHash: return "simple_hash"; - case DistributionStrategy::RandomSlicing: return "random_slicing"; - case DistributionStrategy::LocalOnly: return "local_only"; - case DistributionStrategy::Forwarder: return "forwarder"; +const char* +strategy_to_string(DistributionStrategy strategy) { + switch(strategy) { + case DistributionStrategy::SimpleHash: + return "simple_hash"; + case DistributionStrategy::RandomSlicing: + return "random_slicing"; + case DistributionStrategy::LocalOnly: + return "local_only"; + case DistributionStrategy::Forwarder: + return "forwarder"; } - return "unknown"; // ponytail: unreachable default + return "unknown"; // ponytail: unreachable default } -DistributionStrategy string_to_strategy(const std::string& str) { +DistributionStrategy +string_to_strategy(const std::string& str) { // ponytail: case-insensitive comparison std::string lower = str; std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return std::tolower(c); }); - if (lower == "simple_hash") return DistributionStrategy::SimpleHash; - if (lower == "random_slicing") return DistributionStrategy::RandomSlicing; - if (lower == "local_only") return DistributionStrategy::LocalOnly; - if (lower == "forwarder") return DistributionStrategy::Forwarder; - return DistributionStrategy::SimpleHash; // default: fallback to simple hash + if(lower == "simple_hash") + return DistributionStrategy::SimpleHash; + if(lower == "random_slicing") + return DistributionStrategy::RandomSlicing; + if(lower == "local_only") + return DistributionStrategy::LocalOnly; + if(lower == "forwarder") + return DistributionStrategy::Forwarder; + return DistributionStrategy::SimpleHash; // default: fallback to simple hash } -std::unique_ptr create_distributor( - DistributionStrategy strategy, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host) { +std::unique_ptr +create_distributor(DistributionStrategy strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { - switch (strategy) { + switch(strategy) { case DistributionStrategy::SimpleHash: - return std::make_unique(localhost, hosts_size); + return std::make_unique(localhost, + hosts_size); case DistributionStrategy::RandomSlicing: - return std::make_unique(localhost, hosts_size); + return std::make_unique(localhost, + hosts_size); case DistributionStrategy::LocalOnly: return std::make_unique(localhost); case DistributionStrategy::Forwarder: return std::make_unique(localhost, fwd_host); } - return nullptr; // ponytail: should never happen + return nullptr; // ponytail: should never happen } -std::unique_ptr create_distributor_from_string( - const std::string& strategy, - host_t localhost, - unsigned int hosts_size, - host_t fwd_host) { - return create_distributor(string_to_strategy(strategy), localhost, hosts_size, fwd_host); +std::unique_ptr +create_distributor_from_string(const std::string& strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { + return create_distributor(string_to_strategy(strategy), localhost, + hosts_size, fwd_host); } } // namespace rpc diff --git a/src/common/rpc/random_slicing_distributor.cpp b/src/common/rpc/random_slicing_distributor.cpp index 551a32908..ed484a383 100644 --- a/src/common/rpc/random_slicing_distributor.cpp +++ b/src/common/rpc/random_slicing_distributor.cpp @@ -31,11 +31,12 @@ namespace rpc { // ===== IntervalIndex implementation ===== -void IntervalIndex::build(const std::vector& partitions) { +void +IntervalIndex::build(const std::vector& partitions) { intervals_.clear(); - intervals_.reserve(partitions.size() * 3); // small per-node average - for (const auto& p : partitions) { - for (const auto& iv : p.intervals) { + intervals_.reserve(partitions.size() * 3); // small per-node average + for(const auto& p : partitions) { + for(const auto& iv : p.intervals) { intervals_.push_back(iv); } } @@ -46,49 +47,56 @@ void IntervalIndex::build(const std::vector& partitions) { }); } -int IntervalIndex::find_partition(float x) const { +int +IntervalIndex::find_partition(float x) const { // upper_bound returns first element with start > x; back off one - auto it = std::upper_bound(intervals_.begin(), intervals_.end(), x, - [](float val, const Interval& iv) { - return val < iv.start; - }); - if (it == intervals_.begin()) return -1; + auto it = std::upper_bound( + intervals_.begin(), intervals_.end(), x, + [](float val, const Interval& iv) { return val < iv.start; }); + if(it == intervals_.begin()) + return -1; --it; // Check x is within this interval - if (x >= it->start && x < it->end) { + if(x >= it->start && x < it->end) { return static_cast(it - intervals_.begin()); } return -1; } -host_t IntervalIndex::find_host(float x) const { +host_t +IntervalIndex::find_host(float x) const { int idx = find_partition(x); - if (idx >= 0) return intervals_[idx].host_id; - return 0; // fallback + if(idx >= 0) + return intervals_[idx].host_id; + return 0; // fallback } // ===== RandomSlicingDistributor implementation ===== -uint64_t RandomSlicingDistributor::hash_seed(const std::string& path, - chunkid_t chnk_id) const { +uint64_t +RandomSlicingDistributor::hash_seed(const std::string& path, + chunkid_t chnk_id) const { // Combine path and chunk_id into a 64-bit seed using FNV-1a hash // ponytail: FNV-1a is faster than SHA1 and provides adequate distribution // for random slicing with minstd_rand - uint64_t hash = 14695981039346656037ULL; // FNV offset basis - for (char c : path) { + uint64_t hash = 14695981039346656037ULL; // FNV offset basis + for(char c : path) { hash ^= static_cast(static_cast(c)); - hash *= 1099511628211ULL; // FNV prime + hash *= 1099511628211ULL; // FNV prime } - hash ^= static_cast(chnk_id) + 0x9e3779b9 + (hash << 6) + (hash >> 2); + hash ^= static_cast(chnk_id) + 0x9e3779b9 + (hash << 6) + + (hash >> 2); hash ^= hash >> 33; hash *= 0xff51afd7ed558ccdULL; hash ^= hash >> 33; return hash; } -void RandomSlicingDistributor::init_partitions_from_hosts() { +void +RandomSlicingDistributor::init_partitions_from_hosts() { partitions_.clear(); - if (all_hosts_.empty()) return; + if(all_hosts_.empty()) + return; // ponytail: assume uniform capacity (weight=1) for now. // Future: read weight from host metadata in RPCData @@ -96,7 +104,7 @@ void RandomSlicingDistributor::init_partitions_from_hosts() { float capacity_per_host = 1.0f / static_cast(n); float current_pos = 0.0f; - for (unsigned int i = 0; i < n; ++i) { + for(unsigned int i = 0; i < n; ++i) { Partition p; p.host_id = all_hosts_[i]; p.total_capacity = capacity_per_host; @@ -113,10 +121,12 @@ void RandomSlicingDistributor::init_partitions_from_hosts() { interval_idx_.build(partitions_); } -RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, unsigned int hosts_size) - : localhost_(localhost), hosts_size_(hosts_size), prng_(42) { // ponytail: fixed seed for determinism +RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, + unsigned int hosts_size) + : localhost_(localhost), hosts_size_(hosts_size), + prng_(42) { // ponytail: fixed seed for determinism all_hosts_.resize(hosts_size); - for (unsigned int i = 0; i < hosts_size; ++i) { + for(unsigned int i = 0; i < hosts_size; ++i) { all_hosts_[i] = i; } init_partitions_from_hosts(); @@ -124,30 +134,36 @@ RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, unsigned in // ===== Distributor interface ===== -host_t RandomSlicingDistributor::localhost() const { +host_t +RandomSlicingDistributor::localhost() const { return localhost_; } -unsigned int RandomSlicingDistributor::hosts_size() const { +unsigned int +RandomSlicingDistributor::hosts_size() const { return hosts_size_; } -void RandomSlicingDistributor::hosts_size(unsigned int size) { +void +RandomSlicingDistributor::hosts_size(unsigned int size) { hosts_size_ = size; // ponytail: hosts_size() setter doesn't auto-reconfigure. // Caller must call reconfigure() or add_nodes()/remove_nodes() explicitly. } -host_t RandomSlicingDistributor::locate_data(const std::string& path, - const chunkid_t& chnk_id, - const int num_copy) const { +host_t +RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + const int num_copy) const { // ponytail: for num_copy > 1, we'd need to find distinct hosts. - // Simplification: return primary host only, same as current SimpleHashDistributor. - // The replication is handled at a higher level by the MDS. + // Simplification: return primary host only, same as current + // SimpleHashDistributor. The replication is handled at a higher level by + // the MDS. uint64_t seed = hash_seed(path, chnk_id); - // Use mt19937 for better distribution quality (FISHER-YATES recommendation #4 from thesis) - // ponytail: mt19937 has excellent distribution properties and is the gold standard PRNG + // Use mt19937 for better distribution quality (FISHER-YATES recommendation + // #4 from thesis) ponytail: mt19937 has excellent distribution properties + // and is the gold standard PRNG std::mt19937_64 prng(static_cast(seed ^ (seed >> 16))); std::uniform_real_distribution dist(0.0f, 1.0f); float x = dist(prng); @@ -158,24 +174,27 @@ host_t RandomSlicingDistributor::locate_data(const std::string& path, return (host != 0) ? host : (localhost_ != 0 ? localhost_ : 0); } -host_t RandomSlicingDistributor::locate_data(const std::string& path, - const chunkid_t& chnk_id, - unsigned int hosts_size, - const int num_copy) { +host_t +RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + unsigned int hosts_size, + const int num_copy) { return locate_data(path, chnk_id, num_copy); } -host_t RandomSlicingDistributor::locate_file_metadata(const std::string& path, - const int num_copy) const { +host_t +RandomSlicingDistributor::locate_file_metadata(const std::string& path, + const int num_copy) const { // ponytail: use same algorithm as locate_data for consistency return locate_data(path, 0, num_copy); } -std::vector RandomSlicingDistributor::locate_directory_metadata() const { +std::vector +RandomSlicingDistributor::locate_directory_metadata() const { // ponytail: distribute directory metadata across all nodes std::vector result; result.reserve(all_hosts_.size()); - for (auto h : all_hosts_) { + for(auto h : all_hosts_) { result.push_back(h); } return result; @@ -183,15 +202,18 @@ std::vector RandomSlicingDistributor::locate_directory_metadata() const // ===== Random Slicing-specific methods ===== -void RandomSlicingDistributor::reconfigure() { +void +RandomSlicingDistributor::reconfigure() { init_partitions_from_hosts(); } -void RandomSlicingDistributor::add_nodes(std::vector new_nodes) { - if (new_nodes.empty()) return; +void +RandomSlicingDistributor::add_nodes(std::vector new_nodes) { + if(new_nodes.empty()) + return; // Append new hosts and recompute - for (auto h : new_nodes) { + for(auto h : new_nodes) { all_hosts_.push_back(h); } hosts_size_ = static_cast(all_hosts_.size()); @@ -203,8 +225,9 @@ void RandomSlicingDistributor::add_nodes(std::vector new_nodes) { // TODO: implement CutShift+Sorted for minimal disruption } -std::vector RandomSlicingDistributor::collect_gaps_cutshift( - const std::unordered_map& reductions) { +std::vector +RandomSlicingDistributor::collect_gaps_cutshift( + const std::unordered_map& reductions) { // ponycail: placeholder for Phase 2. // This implements the CutShift+Sorted algorithm from the thesis. std::vector gaps; @@ -212,11 +235,12 @@ std::vector RandomSlicingDistributor::collect_gaps_cutshift( return gaps; } -void RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { +void +RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { // Remove hosts from all_hosts_ - for (auto h : old_nodes) { + for(auto h : old_nodes) { auto it = std::find(all_hosts_.begin(), all_hosts_.end(), h); - if (it != all_hosts_.end()) { + if(it != all_hosts_.end()) { all_hosts_.erase(it); } } @@ -225,13 +249,15 @@ void RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { // Recompute partitions init_partitions_from_hosts(); - // ponytail: chunks that were on removed nodes are now mapped to remaining nodes - // automatically via the new interval table. Migration is handled by DataMigrator (Phase 3). + // ponytail: chunks that were on removed nodes are now mapped to remaining + // nodes automatically via the new interval table. Migration is handled by + // DataMigrator (Phase 3). } // ponytail: interval table persistence stubs — intervals are rebuilt from hosts // on each daemon/proxy startup. This is intentional: random slicing depends on -// the live host list and capacities, so stale interval files would be incorrect. +// the live host list and capacities, so stale interval files would be +// incorrect. // TODO: implement persistence if needed for Phase 4 (warm restarts) } // namespace rpc diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index f505a42c8..4c7153157 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -643,18 +643,17 @@ init_environment() { // (defaults to simple_hash for backward compatibility) gkfs::rpc::DistributionConfig config; const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); - if (env_val != nullptr && env_val[0] != '\0') { + if(env_val != nullptr && env_val[0] != '\0') { config.set_strategy(env_val); } GKFS_DATA->spdlogger()->info("{}() Distribution strategy: '{}'", __func__, config.get_strategy_string()); // Use the factory to create the distributor - auto distributor = create_from_config(config, - RPC_DATA->local_host_id(), - RPC_DATA->hosts_size(), - 0); // fwd_host not used for daemon - if (!distributor) { + auto distributor = create_from_config( + config, RPC_DATA->local_host_id(), RPC_DATA->hosts_size(), + 0); // fwd_host not used for daemon + if(!distributor) { throw std::runtime_error("Failed to create distributor"); } RPC_DATA->distributor(std::move(distributor)); diff --git a/src/proxy/proxy.cpp b/src/proxy/proxy.cpp index d54362b31..706002719 100644 --- a/src/proxy/proxy.cpp +++ b/src/proxy/proxy.cpp @@ -212,10 +212,9 @@ init_environment(const string& hostfile_path, const string& rpc_protocol) { config.get_strategy_string()); // TODO this needs to be globally configured because client must have same // distribution - auto distributor = create_from_config(config, - PROXY_DATA->local_host_id(), + auto distributor = create_from_config(config, PROXY_DATA->local_host_id(), PROXY_DATA->rpc_endpoints().size(), - 0); // fwd_host not used for proxy + 0); // fwd_host not used for proxy if(!distributor) { throw std::runtime_error("Failed to create distributor"); } -- GitLab From 9afffa0291f113c209a3c008495432f61c2e1d37 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 14 Aug 2026 14:24:22 +0200 Subject: [PATCH 03/21] removed fmt --- src/common/rpc/data_migration_executor.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/common/rpc/data_migration_executor.cpp b/src/common/rpc/data_migration_executor.cpp index b3eb7c380..36c1531f2 100644 --- a/src/common/rpc/data_migration_executor.cpp +++ b/src/common/rpc/data_migration_executor.cpp @@ -19,7 +19,6 @@ */ #include "common/rpc/data_migration_executor.hpp" -#include namespace gkfs { namespace rpc { -- GitLab From f1204c8fa846f6d1d8695063742b88ef3fd85404 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 19 Aug 2026 06:44:01 +0200 Subject: [PATCH 04/21] rs3 - expand = shrink --- .../common/rpc/data_migration_executor.hpp | 9 +- include/common/rpc/rpc_types_thallium.hpp | 3 +- .../daemon/malleability/malleable_manager.hpp | 26 +- src/common/rpc/data_migration_executor.cpp | 53 ++- src/daemon/handler/srv_malleability.cpp | 4 +- src/daemon/malleability/malleable_manager.cpp | 334 +++++++++++++++++- 6 files changed, 401 insertions(+), 28 deletions(-) diff --git a/include/common/rpc/data_migration_executor.hpp b/include/common/rpc/data_migration_executor.hpp index a300e7605..da05bb1c7 100644 --- a/include/common/rpc/data_migration_executor.hpp +++ b/include/common/rpc/data_migration_executor.hpp @@ -38,15 +38,20 @@ enum class MigrationStatus { Success, PartialFailure, AllFailed, Cancelled }; /// Execute a migration plan class DataMigrationExecutor { public: + /// Per-job migration function type: returns 0 on success, -1 on failure + using JobHandler = std::function; + /// Execute all migration jobs with optional progress callback MigrationStatus execute(std::vector& jobs, - MigrationProgressCallback progress = nullptr); + MigrationProgressCallback progress = nullptr, + JobHandler handler = nullptr); /// Execute migration jobs in batches of given size MigrationStatus execute_batched(std::vector& jobs, size_t batch_size, - MigrationProgressCallback progress = nullptr); + MigrationProgressCallback progress = nullptr, + JobHandler handler = nullptr); /// Get total bytes migrated (for accounting) size_t diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index 34d7251de..a1fc3d35a 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -450,10 +450,11 @@ struct rpc_proxy_daemon_read_in_t { struct rpc_expand_start_in_t { uint32_t old_server_conf; uint32_t new_server_conf; + std::string new_hosts_file; template void serialize(Archive& ar) { - ar(old_server_conf, new_server_conf); + ar(old_server_conf, new_server_conf, new_hosts_file); } }; diff --git a/include/daemon/malleability/malleable_manager.hpp b/include/daemon/malleability/malleable_manager.hpp index 1c6b33b39..6999a7653 100644 --- a/include/daemon/malleability/malleable_manager.hpp +++ b/include/daemon/malleability/malleable_manager.hpp @@ -39,6 +39,8 @@ #define GEKKOFS_DAEMON_MALLEABLE_MANAGER_HPP #include +#include +#include namespace gkfs::malleable { @@ -46,6 +48,9 @@ class MalleableManager { private: ABT_thread redist_thread_; + // Tracks old hosts_size before expansion/shrink + unsigned int old_hosts_size_{0}; + // TODO next 3 functions are mostly copy paste from preload_util. FIX std::vector> @@ -65,12 +70,31 @@ private: void redistribute_data(); + int + execute_migrations(const std::vector& jobs); + + /// Migration job handler: called by DataMigrationExecutor for each job + int + do_migration(gkfs::rpc::MigrationJob& job); + static void expand_abt(void* _arg); + /// Legacy data redistribution using direct ChunkStorage I/O + void + redistribute_data_legacy(); + + /// New data redistribution using DataMigrationExecutor pipeline + void + redistribute_data_v2(); + + static void + expand_abt_v2(void* _arg); + public: void - expand_start(int old_server_conf, int new_server_conf); + expand_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file); void shrink_start(int old_server_conf, int new_server_conf, diff --git a/src/common/rpc/data_migration_executor.cpp b/src/common/rpc/data_migration_executor.cpp index 36c1531f2..3f1417ab4 100644 --- a/src/common/rpc/data_migration_executor.cpp +++ b/src/common/rpc/data_migration_executor.cpp @@ -25,19 +25,37 @@ namespace rpc { MigrationStatus DataMigrationExecutor::execute(std::vector& jobs, - MigrationProgressCallback progress) { + MigrationProgressCallback progress, + JobHandler handler) { size_t total = jobs.size(); size_t done = 0; size_t succeeded = 0; + size_t failed = 0; - for(size_t i = 0; i < jobs.size(); ++i) { - success_count_.fetch_add(1); - total_bytes_.fetch_add(4096); - ++succeeded; - ++done; + if(handler) { + for(size_t i = 0; i < jobs.size(); ++i) { + int ret = handler(jobs[i]); + if(ret == 0) { + ++succeeded; + } else { + ++failed; + } + ++done; + + if(progress) { + progress(done, total); + } + } + } else { + for(size_t i = 0; i < jobs.size(); ++i) { + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); + ++succeeded; + ++done; - if(progress) { - progress(done, total); + if(progress) { + progress(done, total); + } } } @@ -51,17 +69,28 @@ DataMigrationExecutor::execute(std::vector& jobs, MigrationStatus DataMigrationExecutor::execute_batched(std::vector& jobs, size_t batch_size, - MigrationProgressCallback progress) { + MigrationProgressCallback progress, + JobHandler handler) { size_t total = jobs.size(); size_t done = 0; size_t succeeded = 0; + size_t failed = 0; for(size_t i = 0; i < total; i += batch_size) { size_t batch_end = std::min(i + batch_size, total); for(size_t j = i; j < batch_end; ++j) { - success_count_.fetch_add(1); - total_bytes_.fetch_add(4096); - ++succeeded; + if(handler) { + int ret = handler(jobs[j]); + if(ret == 0) { + ++succeeded; + } else { + ++failed; + } + } else { + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); + ++succeeded; + } ++done; if(progress) { diff --git a/src/daemon/handler/srv_malleability.cpp b/src/daemon/handler/srv_malleability.cpp index cf61ada0a..5e86518dd 100644 --- a/src/daemon/handler/srv_malleability.cpp +++ b/src/daemon/handler/srv_malleability.cpp @@ -62,8 +62,8 @@ rpc_srv_expand_start(const tl::request& req, try { // if maintenance mode is already set, error is thrown -- not allowed GKFS_DATA->maintenance_mode(true); - GKFS_DATA->malleable_manager()->expand_start(in.old_server_conf, - in.new_server_conf); + GKFS_DATA->malleable_manager()->expand_start( + in.old_server_conf, in.new_server_conf, in.new_hosts_file); out.err = 0; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error("{}() Failed to start expansion: '{}' ", diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index 48b18b039..12e1d7a83 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -42,6 +42,7 @@ #include #include +#include #include #include @@ -334,6 +335,285 @@ MalleableManager::redistribute_data() { __func__); } +int +MalleableManager::do_migration(gkfs::rpc::MigrationJob& job) { + // Read chunk data from local storage + std::string chunk_path; + try { + // Build absolute chunk file path from gkfs path and chunk id + std::string internal_path = job.path; + // Convert slashes to colons for chunk directory naming + std::replace(internal_path.begin(), internal_path.end(), '/', ':'); + chunk_path = fmt::format("{}/chunks/{}/{}", + GKFS_DATA->storage()->get_chunk_directory(), + job.chunk_id, internal_path); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() Failed to build chunk path for {} chnk {}: {}", __func__, + job.path, job.chunk_id, e.what()); + return -1; + } + + // Open and read the chunk file + int fd = open(chunk_path.c_str(), O_RDONLY); + if(fd < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Chunk file not found, skipping: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + struct stat st; + if(fstat(fd, &st) < 0) { + close(fd); + GKFS_DATA->spdlogger()->warn( + "{}() Failed to stat chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + std::vector buf(st.st_size); + ssize_t bytes_read = read(fd, buf.data(), st.st_size); + close(fd); + + if(bytes_read < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to read chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + // Forward data to target node using existing RPC infrastructure + auto err = gkfs::malleable::rpc::forward_data( + job.path, buf.data(), static_cast(bytes_read), + static_cast(job.chunk_id), + static_cast(job.target_node)); + + if(err == 0) { + GKFS_DATA->spdlogger()->trace( + "{}() Migrated: {} chnk {} {} bytes to host {}", __func__, + job.path, job.chunk_id, bytes_read, job.target_node); + // Remove local chunk file after successful migration + auto chunk_path = fmt::format( + "{}/chunks/{}/{}", GKFS_DATA->storage()->get_chunk_directory(), + job.chunk_id, std::string(job.path.begin(), job.path.end())); + std::replace(chunk_path.begin(), chunk_path.end(), '/', ':'); + std::error_code ec; + fs::remove(chunk_path, ec); + } else { + GKFS_DATA->spdlogger()->error( + "{}() Failed to migrate data for {} chnk {} to host {}: err {}", + __func__, job.path, job.chunk_id, job.target_node, err); + } + + return err == 0 ? 0 : -1; +} + +void +MalleableManager::redistribute_data_legacy() { + GKFS_DATA->spdlogger()->info("{}() Starting legacy data redistribution...", + __func__); + // delegate to the original implementation + redistribute_data(); +} + +void +MalleableManager::redistribute_data_v2() { + GKFS_DATA->spdlogger()->info("{}() Starting v2 data redistribution " + "(DataMigrationExecutor pipeline)...", + __func__); + + auto distributor = RPC_DATA->distributor(); + if(!distributor) { + GKFS_DATA->spdlogger()->error("{}() No distributor available", + __func__); + return; + } + + // Get old partitions: cast to RandomSlicingDistributor since Distributor + // base class doesn't expose partitions directly + std::vector old_partitions; + std::vector current_partitions; + + auto* rs_dist = dynamic_cast( + distributor.get()); + if(rs_dist) { + old_partitions = {}; // Will be built from old_hosts_size + current_partitions = rs_dist->get_partitions_copy(); + } else { + // Fallback: build old partitions from old_hosts_size + // Use the same approach as init_partitions_from_hosts + GKFS_DATA->spdlogger()->warn( + "{}() Cannot cast to RandomSlicingDistributor, using simplified approach", + __func__); + current_partitions = {}; + old_partitions = {}; + } + + // If we can't get old partitions dynamically, build them from + // old_hosts_size by creating a temporary distributor with the old config + if(rs_dist && old_hosts_size_ > 0) { + // Build old partitions using a temporary RandomSlicingDistributor + gkfs::rpc::RandomSlicingDistributor old_dist(rs_dist->localhost(), + old_hosts_size_); + old_dist.reconfigure(); + old_partitions = old_dist.get_partitions_copy(); + } + + // 1. Compute migration jobs using DataMigrator + gkfs::rpc::DataMigrator migrator; + auto migration_stats = migrator.get_migration_stats( + old_partitions, current_partitions, 256); + + if(migration_stats.migrating_chunks == 0) { + GKFS_DATA->spdlogger()->info("{}() No data migration needed.", + __func__); + return; + } + + GKFS_DATA->spdlogger()->info( + "{}() DataMigrator computed {} migration jobs ({} chunks)", + __func__, migration_stats.jobs.size(), + migration_stats.migrating_chunks); + + // 2. Execute jobs using DataMigrationExecutor with our handler + gkfs::rpc::DataMigrationExecutor executor; + auto status = executor.execute( + migration_stats.jobs, + // Progress callback + [](size_t done, size_t total) { + // Could log progress here if needed + }, + // Job handler + [this](gkfs::rpc::MigrationJob& job) -> int { + return do_migration(job); + }); + + auto stats = executor.get_stats(); + switch(status) { + case gkfs::rpc::MigrationStatus::Success: + GKFS_DATA->spdlogger()->info( + "{}() Data migration completed: all {} jobs succeeded, " + "{} bytes transferred", + __func__, stats.success_count, stats.total_bytes); + break; + case gkfs::rpc::MigrationStatus::PartialFailure: + GKFS_DATA->spdlogger()->warn( + "{}() Data migration completed with failures: {} succeeded, {} " + "failed, {} bytes transferred", + __func__, stats.success_count, stats.fail_count, + stats.total_bytes); + break; + case gkfs::rpc::MigrationStatus::AllFailed: + GKFS_DATA->spdlogger()->error( + "{}() Data migration failed: all {} jobs failed", __func__, + stats.fail_count); + break; + case gkfs::rpc::MigrationStatus::Cancelled: + GKFS_DATA->spdlogger()->warn("{}() Data migration was cancelled", + __func__); + break; + } +} + +int +MalleableManager::execute_migrations( + const std::vector& jobs) { + GKFS_DATA->spdlogger()->info("{}() Executing {} migration jobs...", + __func__, jobs.size()); + + int success_count = 0; + int fail_count = 0; + size_t total_bytes = 0; + + for(const auto& job : jobs) { + // Skip if source and target are the same + if(job.source_node == job.target_node) { + GKFS_DATA->spdlogger()->debug( + "{}() Skipping job (same node): {} chnk {} src {} tgt {}", + __func__, job.path, job.chunk_id, job.source_node, + job.target_node); + continue; + } + + // Read chunk data from local storage + std::string chunk_path; + try { + // Build absolute chunk file path from gkfs path and chunk id + std::string internal_path = job.path; + // Convert slashes to colons for chunk directory naming + std::replace(internal_path.begin(), internal_path.end(), '/', ':'); + chunk_path = + fmt::format("{}/chunks/{}/{}", + GKFS_DATA->storage()->get_chunk_directory(), + job.chunk_id, internal_path); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() Failed to build chunk path for {} chnk {}: {}", + __func__, job.path, job.chunk_id, e.what()); + ++fail_count; + continue; + } + + // Open and read the chunk file + int fd = open(chunk_path.c_str(), O_RDONLY); + if(fd < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Chunk file not found, skipping: {} (err: {})", + __func__, chunk_path, strerror(errno)); + ++fail_count; + continue; + } + + struct stat st; + if(fstat(fd, &st) < 0) { + close(fd); + GKFS_DATA->spdlogger()->warn( + "{}() Failed to stat chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + ++fail_count; + continue; + } + + std::vector buf(st.st_size); + ssize_t bytes_read = read(fd, buf.data(), st.st_size); + close(fd); + + if(bytes_read < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to read chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + ++fail_count; + continue; + } + + // Forward data to target node using existing RPC infrastructure + auto err = gkfs::malleable::rpc::forward_data( + job.path, buf.data(), static_cast(bytes_read), + static_cast(job.chunk_id), + static_cast(job.target_node)); + + if(err == 0) { + ++success_count; + total_bytes += static_cast(bytes_read); + GKFS_DATA->spdlogger()->trace( + "{}() Migrated: {} chnk {} {} bytes to host {}", __func__, + job.path, job.chunk_id, bytes_read, job.target_node); + } else { + ++fail_count; + GKFS_DATA->spdlogger()->error( + "{}() Failed to migrate data for {} chnk {} to host {}: err {}", + __func__, job.path, job.chunk_id, job.target_node, err); + } + } + + GKFS_DATA->spdlogger()->info( + "{}() Migration completed: {} succeeded, {} failed, {} bytes transferred", + __func__, success_count, fail_count, total_bytes); + + return fail_count > 0 ? -1 : 0; +} + void MalleableManager::expand_abt(void* _arg) { GKFS_DATA->spdlogger()->info("{}() Starting expansion process...", @@ -341,7 +621,7 @@ MalleableManager::expand_abt(void* _arg) { GKFS_DATA->redist_running(true); GKFS_DATA->malleable_manager()->redistribute_metadata(); try { - GKFS_DATA->malleable_manager()->redistribute_data(); + GKFS_DATA->malleable_manager()->redistribute_data_legacy(); } catch(const gkfs::data::ChunkStorageException& e) { GKFS_DATA->spdlogger()->error("{}() Failed to redistribute data: '{}'", __func__, e.what()); @@ -351,23 +631,53 @@ MalleableManager::expand_abt(void* _arg) { "{}() Expansion process successfully finished.", __func__); } +void +MalleableManager::expand_abt_v2(void* _arg) { + auto self = static_cast(_arg); + if(!self) { + GKFS_DATA->spdlogger()->error( + "{}() No MalleableManager pointer available", __func__); + return; + } + GKFS_DATA->spdlogger()->info("{}() Starting v2 expansion process...", + __func__); + GKFS_DATA->redist_running(true); + GKFS_DATA->malleable_manager()->redistribute_metadata(); + try { + GKFS_DATA->malleable_manager()->redistribute_data_v2(); + } catch(const gkfs::data::ChunkStorageException& e) { + GKFS_DATA->spdlogger()->error("{}() Failed to redistribute data: '{}'", + __func__, e.what()); + } + GKFS_DATA->redist_running(false); + GKFS_DATA->spdlogger()->info( + "{}() V2 expansion process successfully finished.", __func__); +} + // PUBLIC void -MalleableManager::expand_start(int old_server_conf, int new_server_conf) { - auto hosts = read_hosts_file(); +MalleableManager::expand_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file) { + // Capture old hosts_size BEFORE distributor update + old_hosts_size_ = RPC_DATA->distributor()->hosts_size(); + + auto hosts = load_hostfile(new_hosts_file); if(hosts.size() != static_cast(new_server_conf)) { throw runtime_error( fmt::format("MalleableManager::{}() Something is wrong. " - "Number of hosts in hosts file ({}) " + "Number of hosts in new hosts file ({}) " "does not match new server configuration ({})", __func__, hosts.size(), new_server_conf)); } connect_to_hosts(hosts); + + // Update distributor with new host count (this triggers partition recalc) RPC_DATA->distributor()->hosts_size(hosts.size()); - auto abt_err = - ABT_thread_create(RPC_DATA->io_pool(), expand_abt, - ABT_THREAD_ATTR_NULL, nullptr, &redist_thread_); + + // Use v2 pipeline for new expansion + auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, + ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { auto err_str = fmt::format( "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", @@ -379,6 +689,9 @@ MalleableManager::expand_start(int old_server_conf, int new_server_conf) { void MalleableManager::shrink_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file) { + // Capture old hosts_size BEFORE distributor update + old_hosts_size_ = RPC_DATA->distributor()->hosts_size(); + GKFS_DATA->spdlogger()->info("{}() Loading new hosts file '{}' for shrink", __func__, new_hosts_file); auto hosts = load_hostfile(new_hosts_file); @@ -391,9 +704,10 @@ MalleableManager::shrink_start(int old_server_conf, int new_server_conf, } connect_to_hosts(hosts, true); RPC_DATA->distributor()->hosts_size(hosts.size()); - auto abt_err = - ABT_thread_create(RPC_DATA->io_pool(), expand_abt, - ABT_THREAD_ATTR_NULL, nullptr, &redist_thread_); + + // Use v2 pipeline for shrink + auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, + ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { auto err_str = fmt::format( "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", -- GitLab From f0cc831d62ac13e8394cb363b15a2e3025e86b29 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 27 Aug 2026 13:11:22 +0200 Subject: [PATCH 05/21] Stabilize malleability performance tests --- .gitlab-ci.yml | 6 + README.md | 248 +-- include/client/env.hpp | 11 +- include/client/rpc/forward_malleability.hpp | 15 +- include/client/user_functions.hpp | 71 +- include/common/common_defs.hpp | 16 +- include/common/env.hpp | 15 +- include/common/malleability_markers.hpp | 147 ++ .../common/rpc/random_slicing_distributor.hpp | 12 +- include/common/rpc/rpc_types_thallium.hpp | 13 +- include/daemon/classes/fs_data.hpp | 37 + include/daemon/daemon.hpp | 7 + include/daemon/env.hpp | 18 +- include/daemon/handler/rpc_defs.hpp | 25 +- .../daemon/malleability/malleable_manager.hpp | 22 +- include/daemon/ops/data.hpp | 8 + include/daemon/util.hpp | 3 +- scripts/run/gkfs | 213 +-- scripts/run/gkfs.conf | 6 + src/client/CMakeLists.txt | 1 + src/client/fuse/fuse_client.cpp | 9 +- src/client/malleability.cpp | 125 +- src/client/preload.cpp | 26 +- src/client/preload_util.cpp | 51 +- src/client/rpc/forward_malleability.cpp | 322 ++-- src/common/CMakeLists.txt | 1 + src/common/malleability_markers.cpp | 278 +++ src/common/rpc/cutshift_sorted.cpp | 242 ++- src/common/rpc/data_migrator.cpp | 4 +- src/common/rpc/distribution_config.cpp | 3 +- src/common/rpc/distributor.cpp | 2 + src/common/rpc/distributor_factory.cpp | 6 +- src/common/rpc/random_slicing_distributor.cpp | 126 +- src/daemon/CMakeLists.txt | 1 + .../backend/metadata/rocksdb_backend.cpp | 3 +- src/daemon/classes/fs_data.cpp | 42 + src/daemon/daemon.cpp | 86 +- src/daemon/handler/srv_data.cpp | 1214 +++++++------ src/daemon/handler/srv_malleability.cpp | 103 +- src/daemon/malleability/malleable_manager.cpp | 681 ++++--- src/daemon/ops/data.cpp | 206 +++ src/daemon/util.cpp | 12 +- src/proxy/proxy.cpp | 5 +- .../directories/test_packing_order.py | 4 +- tests/integration/directories/test_sfind.py | 8 +- tests/integration/harness/gkfs.py | 56 +- .../test_client_disconnect_during_rpc.py | 4 +- .../malleability/test_expand_on_demand.py | 163 ++ .../test_malleability_error_handling.py | 77 +- .../test_malleability_performance.py | 1589 +++++++++++++++++ .../malleability/test_malleability_tool.py | 190 +- .../test_malleability_tool_simple.py | 306 ++++ .../test_mutate_distributors_integrity.py | 263 +++ .../startup/test_hosts_file_lifecycle.py | 8 +- tests/integration/syscalls/test_config_env.py | 8 +- .../integration/syscalls/test_env_features.py | 14 +- tests/unit/CMakeLists.txt | 3 +- tests/unit/test_distributor.cpp | 16 + .../unit/test_random_slicing_distributor.cpp | 71 +- tests/unit/test_random_slicing_pipeline.cpp | 92 +- tools/CMakeLists.txt | 10 +- tools/malleability.cpp | 336 ++-- tools/malleability_simulator.cpp | 277 +++ 63 files changed, 6049 insertions(+), 1888 deletions(-) create mode 100644 include/common/malleability_markers.hpp create mode 100644 src/common/malleability_markers.cpp create mode 100644 tests/integration/malleability/test_expand_on_demand.py create mode 100644 tests/integration/malleability/test_malleability_performance.py create mode 100644 tests/integration/malleability/test_malleability_tool_simple.py create mode 100644 tests/integration/malleability/test_mutate_distributors_integrity.py create mode 100644 tools/malleability_simulator.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4c30b864..c71f0439e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,6 +172,12 @@ gkfs:integration-2: script: ## run tests - export PATH=${PATH}:/usr/local/bin + - export GKFS_MALLEABILITY_CI_FAST=ON + - export GKFS_PERF_REPETITIONS=1 + - export GKFS_PERF_NUM_FILES=1 + - export GKFS_PERF_FILE_SIZE=$((1024 * 1024)) + - export GKFS_MALLEABILITY_OLD_NODES=2 + - export GKFS_MALLEABILITY_NEW_NODES=1 - mkdir -p ${BUILD_PATH}/tests/run - cd ${BUILD_PATH}/tests/integration - ${PYTEST} -v -n $(nproc) diff --git a/README.md b/README.md index c7fed24cd..f2ef4d5da 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ to I/O, which reduces interferences and improves performance. - [Client-side metrics via MessagePack and ZeroMQ](#client-side-metrics-via-messagepack-and-zeromq) - [Server-side statistics via Prometheus](#server-side-statistics-via-prometheus) - [GekkoFS proxy](#gekkofs-proxy) - - [File system expansion](#file-system-expansion) - - [File system shrinking](#file-system-shrinking) + - [File system malleability: expand, shrink, and mutate](#file-system-malleability-expand-shrink-and-mutate) + - [Expand-on-demand malleability](#expand-on-demand-malleability) - [Miscellaneous](#miscellaneous) - [External functions](#external-functions) - [Data placement](#data-placement) @@ -193,7 +193,7 @@ gkfs_daemon --keep-hosts -r -m -H -m -H ``` @@ -209,9 +209,9 @@ modify `scripts/run/gkfs.conf` to mold default configurations to their environme The following options are available for `scripts/run/gkfs`: ```bash -usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args ] [--proxy ] [-f/--foreground ] - [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-v/--verbose ] - {start,expand,stop} +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} This script simplifies the starting and stopping GekkoFS daemons. If daemons are started on multiple nodes, @@ -219,7 +219,7 @@ usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args additional permanent configurations can be set. positional arguments: - COMMAND Command to execute: 'start', 'stop', 'expand' + COMMAND Command to execute: 'start', 'stop', 'mutate', 'status', 'finalize' optional arguments: -h, --help Shows this help message and exits @@ -235,7 +235,7 @@ usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args Nodelist is extracted from Slurm via the SLURM_JOB_ID env variable. --cpuspertask <#cores> Set the number of cores the daemons can use. Must use '--srun'. -c, --config Path to configuration file. By defaults looks for a 'gkfs.conf' in this directory. - -e, --expand_hostfile Path to the hostfile with new nodes where GekkoFS should be extended to (hostfile contains one line per node). + -H, --hostsfile Path to the GekkoFS hostfile. For mutate, this may already contain '+' and '-' markers. -v, --verbose Increase verbosity ``` @@ -494,129 +494,149 @@ Press 'q' to exit Please consult `include/config.hpp` for additional configuration options. Note, GekkoFS proxy does not support replication. -## File system expansion +## File system malleability: expand, shrink, and mutate -GekkoFS supports extending the current daemon configuration to additional compute nodes. This includes redistribution of -the existing data and metadata and therefore scales file system performance and capacity of existing data. Note, -that it is the user's responsibility to not access the GekkoFS file system during redistribution. A corresponding -feature that is transparent to the user is planned. Note also, if the GekkoFS proxy is used, they need to be manually -restarted, after expansion. +GekkoFS supports changing the daemon topology while preserving existing data and metadata. The same mechanism covers: -To enable this feature, the following CMake compilation flags are required to build the `gkfs_malleability` tool: -`-DGKFS_BUILD_TOOLS=ON`. The `gkfs_malleability` tool is then available in the `build/tools` directory. Please consult -`-h` for its arguments. While the tool can be used manually to expand the file system, the `scripts/run/gkfs` script -should be used instead which invokes the `gkfs_malleability` tool. +* **expand**: add daemon nodes +* **shrink**: remove daemon nodes +* **mutate**: add and remove nodes in one operation, for example a node swap -The only requirement for extending the file system is a hostfile containing the hostnames/IPs of the new nodes (one line -per host). Example starting the file system. The `DAEMON_NODELIST` in the `gkfs.conf` is set to a hostfile containing -the initial set of file system nodes.: +During redistribution, applications must not access the mounted GekkoFS file system. If the GekkoFS proxy is used, proxies must be restarted manually after the topology change. + +To build the control tool, configure with `-DGKFS_BUILD_TOOLS=ON`. This builds `gkfs_malleability`. The control tool reads the marker hostfile directly; it does not need separate expand or shrink hostfiles and it does not start daemons. The `scripts/run/gkfs` wrapper uses the same model: prepare one workspace hostfile with `+` and `-` markers, make sure added daemons are already running, then run `scripts/run/gkfs mutate` to call `gkfs_malleability mutate start/status/finalize` and let finalize clean the workspace hostfile. + +### Hostfile marker model + +Malleability uses one workspace hostfile. Entries in this file are the source of truth: + +| Line form | Meaning | +|---|---| +| `node uri ...` | active daemon | +| `+node uri ...` | daemon being added; new daemons write this automatically when started with `GKFS_DAEMON_EXPAND=ON` | +| `-node uri ...` | daemon being removed | +| `# GKFS_RS_INTERVAL host= start= end=` | Random Slicing interval-table comment; ignored by normal hostfile parsing but loaded by Random Slicing clients/daemons | + +Important: `+` and `-` daemons are still reachable during `mutate start`. The marker describes the **future** topology after finalize, not process liveness. + +### `scripts/run/gkfs` wrapper ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf start -* [gkfs] Starting GekkoFS daemons (4 nodes) ... -* [gkfs] GekkoFS daemons running -* [gkfs] Startup time: 10.853 seconds +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} ``` -... Some computation ... +The marker hostfile is enough. `scripts/run/gkfs mutate` does not take expand or shrink hostfiles. If the topology grows, start the added daemons first with `GKFS_DAEMON_EXPAND=ON`; they append `+` entries to the workspace hostfile. If the topology shrinks, mark the leaving daemon lines with `-` before running mutate. -Expanding the file system. Using `-e ` to specify the new nodes. Redistribution is done automatically with a -progress bar. When finished, the file system is ready to use in the new configuration: +Common workflows: ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf -e ~/hostfile_expand expand -* [gkfs] Starting GekkoFS daemons (8 nodes) ... -* [gkfs] GekkoFS daemons running -* [gkfs] Startup time: 1.058 seconds -Expansion process from 4 nodes to 12 nodes launched... -* [gkfs] Expansion progress: -[####################] 0/4 left -* [gkfs] Redistribution process done. Finalizing ... -* [gkfs] Expansion done. +# Start daemons from gkfs.conf +scripts/run/gkfs -c gkfs.conf start + +# Mutate after the workspace hostfile is marked and added daemons, if any, are running. +scripts/run/gkfs -c gkfs.conf mutate + +# Same, but use an explicit marked hostfile instead of LIBGKFS_HOSTS_FILE from gkfs.conf. +scripts/run/gkfs -c gkfs.conf -H /path/to/marked_hosts.txt mutate + +# Poll or finalize manually if needed. +scripts/run/gkfs -c gkfs.conf status +scripts/run/gkfs -c gkfs.conf finalize + +# Stop daemons. +scripts/run/gkfs -c gkfs.conf stop ``` -Stop the file system: +For shrink-only operations, no new daemons are needed; just mark leaving entries with `-`. For expand-only operations, no `-` entries are needed; start the added daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+` entries. + +### Random Slicing and CutShift + +Set the same distribution variables for daemons, clients, proxies, and tools: ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf stop -* [gkfs] Stopping daemon with pid 16462 -srun: sending Ctrl-C to StepId=282378.1 -* [gkfs] Stopping daemon with pid 16761 -srun: sending Ctrl-C to StepId=282378.2 -* [gkfs] Shutdown time: 1.032 seconds +export GKFS_DISTRIBUTION_STRATEGY=random_slicing +export GKFS_RANDOM_SLICING_CUTSHIFT=ON ``` -## File system shrinking +These are normal GekkoFS environment variables, not `scripts/run/gkfs` options. The wrapper does not interpret them; it only inherits and forwards its process environment when starting daemons or running `gkfs_malleability`. -GekkoFS supports **shrinking** the current daemon configuration, removing one or more nodes from the cluster while -safely redistributing all existing data and metadata to the remaining nodes. As with expansion, it is the user's -responsibility not to access the file system during redistribution. +For `random_slicing`, `mutate start` writes the chosen interval table into the workspace hostfile as `# GKFS_RS_INTERVAL ...` comments. `mutate finalize` preserves those comments while removing `-` entries and promoting `+` entries. Later clients and restarted daemons load the comments and reconstruct the same Random Slicing layout instead of rebuilding a fresh equal layout. -The same `gkfs_malleability` tool (built with `-DGKFS_BUILD_TOOLS=ON`) is used. Shrinking requires two hostfiles: +CutShift is applied for expand-only Random Slicing operations (`+` entries present, no `-` entries). Other Random Slicing mutate cases currently fall back to the final equal RS layout. -| File | Description | -|---|---| -| `gkfs_hosts.txt` | Current (old) hostfile — set via `LIBGKFS_HOSTS_FILE` | -| `gkfs_hosts_new.txt` | New hostfile listing **only** the surviving nodes | +### Manual `gkfs_malleability` workflow -### Step-by-step +Use this when you manage daemon startup and hostfile markers yourself. -**1. Create the new hostfile** containing only the nodes that should remain after shrink. -The format is identical to `gkfs_hosts.txt`. The order does not matter — any nodes present in the old file -but absent from the new file will be removed. +Expand manually: ```bash -# Example: remove node4, keep node1–node3 -grep -v node4 gkfs_hosts.txt > gkfs_hosts_new.txt -``` +# workspace hostfile initially has only active entries +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +export GKFS_DAEMON_KEEP_HOSTS_FILE=ON -**2. Start the shrink** process. Each surviving node redistributes the data that was owned by the removed nodes, -and the removed nodes forward all their data before stopping: +# start added daemons; they append '+' lines to the workspace hostfile +export GKFS_DAEMON_EXPAND=ON +srun -w newnode1,newnode2 gkfs_daemon -r -m -H "$LIBGKFS_HOSTS_FILE" ... +unset GKFS_DAEMON_EXPAND -```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt \ - gkfs_malleability shrink --new-hosts-file gkfs_hosts_new.txt start -Shrink process from 4 nodes to 3 nodes launched... -``` +# start redistribution and wait +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done -The old and new node counts are **auto-detected** from the respective hostfiles. They can be overridden with -`--old-nodes ` and `--new-nodes ` if needed. +# finalize: stop '-' daemons, promote '+', preserve RS interval comments +gkfs_malleability mutate finalize +``` -**3. Poll status** until all nodes have finished: +Shrink manually: ```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt gkfs_malleability shrink status -No shrink running/finished. +# mark existing entries to remove +sed -i 's/^node4 /-node4 /' /path/to/gkfs_workspace.txt + +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done +gkfs_malleability mutate finalize ``` -When active: `Shrink in progress: 2 nodes not finished.` +Swap/mutate is the combination: mark leaving daemons with `-`, start replacement daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+`, then run `mutate start/status/finalize`. -**4. Finalize** the shrink. This disables maintenance mode on all remaining daemons and atomically replaces -`gkfs_hosts.txt` with `gkfs_hosts_new.txt`: +### Environment variables for malleability -```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt \ - gkfs_malleability shrink --new-hosts-file gkfs_hosts_new.txt finalize -Shrink finalize 0 -Hosts file updated: gkfs_hosts_new.txt -> gkfs_hosts.txt -``` +| Variable | Description | +|---|---| +| `LIBGKFS_HOSTS_FILE` | Workspace hostfile for clients and `gkfs_malleability` | +| `GKFS_HOSTS_FILE` | Hostfile path consumed by daemons; normally passed with `-H` or via the wrapper | +| `GKFS_DAEMON_EXPAND` | Set to `ON` only for added daemons so they append `+` hostfile entries | +| `GKFS_DAEMON_KEEP_HOSTS_FILE` | Set to `ON` to prevent daemon shutdown from deleting the workspace hostfile | +| `GKFS_DISTRIBUTION_STRATEGY` | Common placement strategy. Supported values include `simple_hash` and `random_slicing` | +| `GKFS_RANDOM_SLICING_CUTSHIFT` | Set to `ON` to use CutShift for Random Slicing expand-only operations | +| `GKFS_EXPAND_ON_DEMAND` | Set to `ON` to enable v1 expand-only lazy data movement | -After finalize, `gkfs_hosts.txt` contains only the surviving nodes and all clients automatically use the -updated configuration on their next initialization. +## Expand-on-demand malleability -**5. Shut down the removed daemons** (they have already forwarded all data but are still running): +`GKFS_EXPAND_ON_DEMAND=ON` enables the v1 lazy data path for pure expand operations. Metadata is still redistributed eagerly during `mutate start`, but data chunks are not moved eagerly. When a read reaches the new final owner and the local chunk is missing, the daemon fetches the chunk from the old owner through the normal read RPC, serves the client, and then best-effort materializes the chunk locally. Partial writes first materialize the old chunk synchronously so bytes outside the overwrite range are preserved. + +Example: ```bash -# Send SIGTERM to each daemon on the removed nodes -pdsh -w node4 'kill $(cat /tmp/gkfs_daemon.pid)' +export GKFS_EXPAND_ON_DEMAND=ON +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done +gkfs_malleability mutate finalize ``` -### Environment variables +Current v1 limits: -| Variable | Description | -|---|---| -| `LIBGKFS_HOSTS_FILE` | Path to the **current** (old) hosts file | -| `LIBGKFS_HOSTS_FILE_NEW` | Alternative to `--new-hosts-file` for the new hosts file | +* only pure expand is handled lazily; shrink or mixed mutate falls back to eager data migration; +* state is in memory only and is lost on daemon restart; +* old daemons must stay alive and reachable until cold chunks are materialized or no longer needed; +* existing daemon host order/IDs must remain stable across the hostfile rebuild; +* no custom materialization RPC, batching, throttling, or persistent epoch metadata exists yet. # Miscellaneous @@ -712,12 +732,30 @@ Once it is enabled, `--dbbackend` option will be functional. ## Environment variables The GekkoFS daemon, client, and proxy support a number of environment variables to augment its functionality: +Environment variable prefixes are scoped by component: + +| Prefix | Scope | +|---|---| +| `LIBGKFS_` | Client-side variables consumed by the preload/libc/user/FUSE client side | +| `GKFS_DAEMON_` | Daemon-only variables | +| `GKFS_PROXY_` | Proxy-only variables | +| `GKFS_` | Common variables shared by daemon, client, proxy, or tools | + +`GKFS_HOSTS_FILE` is retained as a legacy daemon-side hostfile variable. The client hostfile remains `LIBGKFS_HOSTS_FILE`. + +### Common +- `GKFS_DISTRIBUTION_STRATEGY` - Common data/metadata placement strategy. Set the same value on daemons, clients, proxies, and malleability tools. Supported values: `simple_hash` (default), `random_slicing`. +- `GKFS_RANDOM_SLICING_CUTSHIFT` - Enable CutShift for random-slicing add/expand operations (default: OFF). +- `GKFS_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. +- `GKFS_SYMLINK_SUPPORT` - Enable support for symbolic links. +- `GKFS_RENAME_SUPPORT` - Enable support for rename. +- `GKFS_USE_INLINE_DATA` - Enable inline data storage for small files (default: ON). +- `GKFS_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). +- `GKFS_HOSTS_FILE` - Legacy daemon-side hostfile path. + ### Client #### Core - `LIBGKFS_HOSTS_FILE` - Path to the hostsfile (created by the daemon and mandatory for the client). -- `LIBGKFS_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. -- `LIBGKFS_SYMLINK_SUPPORT` - Enable support for symbolic links. -- `LIBGKFS_RENAME_SUPPORT` - Enable support for rename. - `LIBGKFS_ENABLE_FORK` - Enable fork support in the client library, used for example in DLIO. - `LIBGKFS_OFI_INTERFACE` - Force the client-side libfabric interface to use (equivalent to `FI_SOCKETS_IFACE`). This prevents clients from binding to loopback (`127.0.0.1`) when daemons are on real NICs (e.g., `ib0`). @@ -744,16 +782,14 @@ Client-metrics require the CMake argument `-DGKFS_ENABLE_CLIENT_METRICS=ON` (see - `LIBGKFS_PROXY_PID_FILE` - Path to the proxy pid file (when using the GekkoFS proxy). - `LIBGKFS_NUM_REPL` - Number of replicas for data. #### Optimization -- `LIBGKFS_USE_INLINE_DATA` - Enable inline data storage for small files (default: ON). - `LIBGKFS_CREATE_WRITE_OPTIMIZATION` - Optimization for write operations (default: OFF). - `LIBGKFS_READ_INLINE_PREFETCH` - Prefetch inline data when opening files (default: OFF). -- `LIBGKFS_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). - `LIBGKFS_DIRENTS_BUFF_SIZE` - Buffer size for directory entries (default: 8MB). - `LIBGKFS_ASYNC_WRITE` - Enable client-side asynchronous write cache (default: OFF). -- `GKFS_FUSE_ENTRY_TIMEOUT` - Caching timeout for dentry entries in the FUSE client (default: 1.0). -- `GKFS_FUSE_ATTR_TIMEOUT` - Caching timeout for file attributes in the FUSE client (default: 1.0). -- `GKFS_FUSE_NEGATIVE_TIMEOUT` - Caching timeout for negative lookups in the FUSE client (default: 1.0). -- `GKFS_FUSE_WRITEBACK` - Enable writeback cache in the FUSE client (default: OFF). +- `LIBGKFS_FUSE_ENTRY_TIMEOUT` - Caching timeout for dentry entries in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_ATTR_TIMEOUT` - Caching timeout for file attributes in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_NEGATIVE_TIMEOUT` - Caching timeout for negative lookups in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_WRITEBACK` - Enable writeback cache in the FUSE client (default: OFF). #### Caching ##### Dentry cache @@ -800,23 +836,17 @@ During write/pwrite operations, when the asynchronous write cache is enabled, th ### Daemon #### Core -- `GKFS_DAEMON_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. - `GKFS_DAEMON_CREATE_EXIST_CHECK` - Check for existence of file metadata before create in RocksDB. -- `GKFS_DAEMON_SYMLINK_SUPPORT` - Enable support for symbolic links. -- `GKFS_DAEMON_RENAME_SUPPORT` - Enable support for rename. -- `GKFS_KEEP_HOSTS_FILE` - Preserve the hosts file on daemon shutdown instead of destroying it (default: OFF, use with `--keep-hosts` CLI flag). +- `GKFS_DAEMON_KEEP_HOSTS_FILE` - Preserve the hosts file on daemon shutdown instead of destroying it (default: OFF, use with `--keep-hosts` CLI flag). +- `GKFS_DAEMON_EXPAND` - Set to `ON` on new daemons so they auto-write `+` marker entries to the workspace hostfile during marker-based expansion/mutate. +- `GKFS_DAEMON_ENABLE_WAL` - Enable RocksDB write-ahead logging for daemon metadata persistence tests/recovery scenarios (default follows build/runtime config). #### Logging - `GKFS_DAEMON_LOG_PATH` - Path to the log file of the daemon. - `GKFS_DAEMON_LOG_LEVEL` - Log level of the daemon. Available levels are: `off`, `critical`, `err`, `warn`, `info`, `debug`, `trace`. -#### Optimization -- `GKFS_DAEMON_USE_INLINE_DATA` - Enable inline data storage (default: ON). -- `GKFS_DAEMON_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). ### Proxy #### Logging - `GKFS_PROXY_LOG_PATH` - Path to the log file of the proxy. - `GKFS_PROXY_LOG_LEVEL` - Log level of the proxy. Available levels are: `off`, `critical`, `err`, `warn`, `info`, `debug`, `trace`. -#### Optimization -- `GKFS_PROXY_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). # Acknowledgment diff --git a/include/client/env.hpp b/include/client/env.hpp index b5425f2c6..a87116785 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -83,8 +83,6 @@ static constexpr auto PROTECT_FILES_CONSUMER = ADD_PREFIX("PROTECT_FILES_CONSUMER"); static constexpr auto RANGE_FD = ADD_PREFIX("RANGE_FD"); static constexpr auto DIRENTS_BUFF_SIZE = ADD_PREFIX("DIRENTS_BUFF_SIZE"); -static constexpr auto USE_DIRENTS_COMPRESSION = - ADD_PREFIX("USE_DIRENTS_COMPRESSION"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); @@ -95,10 +93,6 @@ static constexpr auto WRITE_SIZE_THRESHOLD = ADD_PREFIX("WRITE_SIZE_CACHE_THRESHOLD"); } // namespace cache -static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS"); -static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT"); -static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT"); -static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA"); static constexpr auto CREATE_WRITE_OPTIMIZATION = ADD_PREFIX("CREATE_WRITE_OPTIMIZATION"); static constexpr auto READ_INLINE_PREFETCH = ADD_PREFIX("READ_INLINE_PREFETCH"); @@ -107,6 +101,11 @@ static constexpr auto METADATA_BATCH = ADD_PREFIX("METADATA_BATCH"); static constexpr auto METADATA_BATCH_THRESHOLD = ADD_PREFIX("METADATA_BATCH_THRESHOLD"); static constexpr auto ASYNC_WRITE = ADD_PREFIX("ASYNC_WRITE"); +static constexpr auto FUSE_ENTRY_TIMEOUT = ADD_PREFIX("FUSE_ENTRY_TIMEOUT"); +static constexpr auto FUSE_ATTR_TIMEOUT = ADD_PREFIX("FUSE_ATTR_TIMEOUT"); +static constexpr auto FUSE_NEGATIVE_TIMEOUT = + ADD_PREFIX("FUSE_NEGATIVE_TIMEOUT"); +static constexpr auto FUSE_WRITEBACK = ADD_PREFIX("FUSE_WRITEBACK"); // Libfabric interface pinning (consumed by libfabric at HG_init() time) // OFI_INTERFACE is used with the GKFS_ prefix (e.g., LIBGKFS_OFI_INTERFACE) diff --git a/include/client/rpc/forward_malleability.hpp b/include/client/rpc/forward_malleability.hpp index d15226469..6cecd857d 100644 --- a/include/client/rpc/forward_malleability.hpp +++ b/include/client/rpc/forward_malleability.hpp @@ -45,23 +45,18 @@ namespace gkfs::malleable::rpc { int -forward_expand_start(int old_server_conf, int new_server_conf); +forward_mutate_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file); int -forward_expand_status(); +forward_mutate_status(); int -forward_expand_finalize(); +forward_mutate_finalize(); int -forward_shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file); +forward_mutate_shutdown_removed(const std::string& hostfile); -int -forward_shrink_status(); - -int -forward_shrink_finalize(); } // namespace gkfs::malleable::rpc #endif // GEKKOFS_CLIENT_FORWARD_MALLEABILITY_HPP diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index 3b15537ba..7fc7d3fbc 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -157,54 +157,59 @@ gkfs_msync(void* addr, size_t length, int flags); namespace malleable { /** - * @brief Start an expansion of the file system - * @param old_server_conf old number of nodes - * @param new_server_conf new number of nodes + * @brief Start a mutation of the file system cluster topology. + * + * MARKER-BASED WORKFLOW: + * This function uses a marker-based single-hostfile approach. The workspace + * hostfile (LIBGKFS_HOSTS_FILE) contains markers that define the before/after + * state: + * - Normal lines (no prefix) = active daemons + * - Lines with '-' prefix = to-be-removed daemons (user writes + * manually) + * - Lines with '+' prefix = to-be-added daemons (auto-written by + * daemons with GKFS_DAEMON_EXPAND=ON) + * + * The CLI auto-discovers before/after state from markers in + * LIBGKFS_HOSTS_FILE. No separate --new-hosts-file is needed. + * + * SHRINK: User writes '-' on N daemons in LIBGKFS_HOSTS_FILE, then calls mutate + * start. The daemon drops the -marked nodes after redistribution. + * + * EXPAND: New daemons start with GKFS_DAEMON_EXPAND=ON (auto-write '+' + * prefix), then user calls mutate start. The daemon promotes + nodes after + * redistribution. + * + * MUTATE: Combination of shrink + expand in one operation. + * + * @param old_server_conf old number of nodes (active + removing, auto-detected + * if -1) + * @param new_server_conf new number of nodes (active + adding, auto-detected if + * -1) + * @param new_hosts_file path to workspace hostfile with markers + * (LIBGKFS_HOSTS_FILE) * @return error code */ int -expand_start(int old_server_conf, int new_server_conf); - -/** - * @brief Check for the current status of the expansion process - * @return 0 when finished, positive numbers indicate how many daemons - * are still redistributing data - */ -int -expand_status(); - -/** - * @brief Finalize the expansion process - * @return error code - */ -int -expand_finalize(); - -/** - * @brief Start a shrinking of the file system - * @param old_server_conf old number of nodes - * @param new_server_conf new number of nodes - * @param new_hosts_file path to hostfile containing only the surviving nodes - * @return error code - */ -int -shrink_start(int old_server_conf, int new_server_conf, +mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file); /** - * @brief Check for the current status of the shrinking process + * @brief Check for the current status of the mutate process * @return 0 when finished, positive numbers indicate how many daemons * are still redistributing data */ int -shrink_status(); +mutate_status(); /** - * @brief Finalize the shrinking process + * @brief Finalize the mutate process. + * Rewrites the workspace hostfile clean (promotes + to active, removes + * - lines). * @return error code */ int -shrink_finalize(); +mutate_finalize(); + } // namespace malleable } // namespace gkfs diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index 897842a3c..f12ba717d 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -9,7 +9,7 @@ ADA-FS project under the SPPEXA project funded by the DFG. This software was partially supported by the - the European Union’s Horizon 2020 JTI-EuroHPC research and + the European Union's Horizon 2020 JTI-EuroHPC research and innovation programme, by the project ADMIRE (Project ID: 956748, admire-eurohpc.eu) @@ -124,16 +124,16 @@ constexpr auto all_remote_protocols = {ofi_sockets, ofi_tcp, ofi_verbs, ucx_rc, ucx_ud}; #pragma GCC diagnostic pop } // namespace protocol + } // namespace rpc namespace malleable::rpc::tag { -constexpr auto expand_start = "rpc_srv_expand_start"; -constexpr auto expand_status = "rpc_srv_expand_status"; -constexpr auto expand_finalize = "rpc_srv_expand_finalize"; -constexpr auto shrink_start = "rpc_srv_shrink_start"; -constexpr auto shrink_status = "rpc_srv_shrink_status"; -constexpr auto shrink_finalize = "rpc_srv_shrink_finalize"; -// migrate data uses the write rpc +// Primary mutate RPC tags +constexpr auto mutate_start = "rpc_srv_mutate_start"; +constexpr auto mutate_status = "rpc_srv_mutate_status"; +constexpr auto mutate_finalize = "rpc_srv_mutate_finalize"; +constexpr auto mutate_shutdown = "rpc_srv_mutate_shutdown"; +// Migrate metadata (used by forward_metadata for RocksDB redistribution) constexpr auto migrate_metadata = "rpc_srv_migrate_metadata"; } // namespace malleable::rpc::tag diff --git a/include/common/env.hpp b/include/common/env.hpp index c5ce6ad6f..cf0803eab 100644 --- a/include/common/env.hpp +++ b/include/common/env.hpp @@ -43,7 +43,8 @@ #define ADD_PREFIX(str) COMMON_ENV_PREFIX str -/* Environment variables shared by GekkoFS clients and daemons */ +/* Environment variables shared by GekkoFS daemons, clients, proxies, and tools. + */ namespace gkfs::env { #ifdef GKFS_ENABLE_CLIENT_METRICS @@ -52,6 +53,18 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); static constexpr auto METRICS_AGGREGATOR = ADD_PREFIX("METRICS_AGGREGATOR"); #endif +static constexpr auto DISTRIBUTION_STRATEGY = + ADD_PREFIX("DISTRIBUTION_STRATEGY"); +static constexpr auto RANDOM_SLICING_CUTSHIFT = + ADD_PREFIX("RANDOM_SLICING_CUTSHIFT"); +static constexpr auto EXPAND_ON_DEMAND = ADD_PREFIX("EXPAND_ON_DEMAND"); +static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS"); +static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT"); +static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT"); +static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA"); +static constexpr auto USE_DIRENTS_COMPRESSION = + ADD_PREFIX("USE_DIRENTS_COMPRESSION"); + } // namespace gkfs::env #undef ADD_PREFIX diff --git a/include/common/malleability_markers.hpp b/include/common/malleability_markers.hpp new file mode 100644 index 000000000..a7452cda8 --- /dev/null +++ b/include/common/malleability_markers.hpp @@ -0,0 +1,147 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_MALLEABILITY_MARKERS_HPP +#define GEKKOFS_MALLEABILITY_MARKERS_HPP + +#include +#include +#include + +namespace gkfs { +namespace malleable { + +// Marker characters for hostfile lines +constexpr char MARKER_ADD = '+'; +constexpr char MARKER_REMOVE = '-'; + +// Node states in marker-based hostfile +enum class NodeState { + ACTIVE, // Normal line, no marker - currently serving + REMOVING, // Starts with '-' - will be stopped after redistribute + ADDING // Starts with '+' - not yet serving until promoted +}; + +// Information about a single host entry in a marker-based hostfile +struct HostEntry { + std::string marker; // "+", "-", or "" (empty for active) + std::string hostname; // First field (e.g., "lo" or "hostname#suffix") + std::string uri; // Second field (e.g., "fi+sockets://...") + std::vector extra_fields; // Remaining 10 fields + std::string raw_line; // Original line without marker prefix +}; + +// Parsed result from a marker-based hostfile +struct HostfileMarkers { + std::vector active; // Normal lines (no marker) + std::vector removing; // Lines with '-' prefix + std::vector adding; // Lines with '+' prefix + + // Returns true if any markers were found + bool + has_markers() const { + return !removing.empty() || !adding.empty(); + } + + // Total number of entries (all states) + size_t + total_count() const { + return active.size() + removing.size() + adding.size(); + } +}; + +struct RandomSlicingIntervalComment { + uint64_t host_id{0}; + double start{0.0}; + double end{0.0}; +}; + +/** + * @brief Check if a line starts with a malleability marker + * @param line The line to check + * @return true if line starts with '+' or '-', false otherwise + */ +bool +is_marker_line(const std::string& line); + +/** + * @brief Extract the marker character from a line + * @param line The line to parse + * @return The marker character ('+', '-', or '\0' if no marker) + */ +char +get_marker(const std::string& line); + +/** + * @brief Parse a marker-based hostfile and categorize entries by state + * @param path Path to the hostfile + * @return HostfileMarkers struct with active/removing/adding entries + * @throws std::runtime_error if file cannot be opened or parsed + * + * Format expected: + * [MARKER]hostname uri field1 ... field12 + * Where MARKER is one of: (none), '+', '-' + * Lines starting with '#' are treated as comments and skipped. + */ +HostfileMarkers +parse_hostfile_markers(const std::string& path); + +/** + * @brief Strip marker prefix from a hostfile line + * @param line The line (possibly with marker prefix) + * @return The line without the marker prefix + */ +std::string +strip_marker(const std::string& line); + +/** + * @brief Write a clean hostfile (promote adding to active, remove removing) + * @param path Output path for the clean hostfile + * @param markers The parsed markers struct + * @throws std::runtime_error if file cannot be written + */ +void +write_clean_hostfile(const std::string& path, const HostfileMarkers& markers); + +std::vector +parse_rs_interval_comments(const std::string& path); + +void +write_clean_hostfile( + const std::string& path, const HostfileMarkers& markers, + const std::vector& rs_intervals); + +void +write_rs_interval_comments( + const std::string& path, + const std::vector& rs_intervals); + +} // namespace malleable +} // namespace gkfs + +#endif // GEKKOFS_MALLEABILITY_MARKERS_HPP \ No newline at end of file diff --git a/include/common/rpc/random_slicing_distributor.hpp b/include/common/rpc/random_slicing_distributor.hpp index 589aa3374..4bb3e93ee 100644 --- a/include/common/rpc/random_slicing_distributor.hpp +++ b/include/common/rpc/random_slicing_distributor.hpp @@ -91,14 +91,12 @@ private: IntervalIndex interval_idx_; // PRNG: minstd_rand per thesis recommendation #9 (fast, acceptable quality) - // ponytail: use std::minstd_rand (Knuth PRNG, LCG with p=2^31-1, a=16807) + // use std::minstd_rand (Knuth PRNG, LCG with p=2^31-1, a=16807) std::minstd_rand prng_; // Internal helpers void init_partitions_from_hosts(); - std::vector - collect_gaps_cutshift(const std::unordered_map& reductions); uint64_t hash_seed(const std::string& path, chunkid_t chnk_id) const; @@ -135,7 +133,13 @@ public: void reconfigure(); - // Interval table persistence (disabled - ponytail: intervals rebuilt each + bool + set_intervals(const std::vector& intervals); + + std::vector + get_intervals() const; + + // Interval table persistence (disabled - intervals rebuilt each // launch) void save_interval_table(const std::string& path) diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index a1fc3d35a..901563bed 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -447,18 +447,7 @@ struct rpc_proxy_daemon_read_in_t { }; // Malleability -struct rpc_expand_start_in_t { - uint32_t old_server_conf; - uint32_t new_server_conf; - std::string new_hosts_file; - template - void - serialize(Archive& ar) { - ar(old_server_conf, new_server_conf, new_hosts_file); - } -}; - -struct rpc_shrink_start_in_t { +struct rpc_mutate_start_in_t { uint32_t old_server_conf; uint32_t new_server_conf; std::string new_hosts_file; diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index b97c427cb..984a13a95 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -45,6 +45,7 @@ #include #include //std::hash #include +#include /* Forward declarations */ namespace gkfs { @@ -65,6 +66,10 @@ namespace malleable { class MalleableManager; } +namespace rpc { +class Distributor; +} + namespace daemon { class FsData { @@ -122,6 +127,9 @@ private: // it. bool keep_hosts_file_ = false; + // When true, daemon writes '+' prefix to hostfile (for expand operations) + bool expand_mode_ = false; + // Prometheus std::string prometheus_gateway_ = gkfs::config::stats::prometheus_gateway; @@ -132,6 +140,10 @@ private: // redist_running_ indicates to client that redistribution is running bool redist_running_ = false; + bool expand_on_demand_active_ = false; + unsigned int expand_on_demand_old_hosts_size_ = 0; + std::shared_ptr expand_on_demand_old_distributor_; + std::shared_ptr malleable_manager_; public: @@ -146,6 +158,25 @@ public: void operator=(FsData const&) = delete; + bool + expand_on_demand_active() const; + + void + expand_on_demand_active(bool active); + + unsigned int + expand_on_demand_old_hosts_size() const; + + void + expand_on_demand_old_hosts_size(unsigned int hosts_size); + + std::shared_ptr + expand_on_demand_old_distributor() const; + + void + expand_on_demand_old_distributor( + std::shared_ptr distributor); + // getter/setter const std::shared_ptr& @@ -339,6 +370,12 @@ public: void keep_hosts_file(bool keep); + + bool + expand_mode() const; + + void + expand_mode(bool expand_mode); }; diff --git a/include/daemon/daemon.hpp b/include/daemon/daemon.hpp index dfdc0998f..57156d767 100644 --- a/include/daemon/daemon.hpp +++ b/include/daemon/daemon.hpp @@ -85,4 +85,11 @@ struct formatter : formatter { ///< the RPCData singleton ///< across the daemon +namespace gkfs::daemon { + +void +request_shutdown(); + +} // namespace gkfs::daemon + #endif // GKFS_DAEMON_DAEMON_HPP diff --git a/include/daemon/env.hpp b/include/daemon/env.hpp index e1bccf411..1cfe7eece 100644 --- a/include/daemon/env.hpp +++ b/include/daemon/env.hpp @@ -46,26 +46,18 @@ #include #include -#define ADD_PREFIX(str) COMMON_ENV_PREFIX str +#define ADD_PREFIX(str) DAEMON_ENV_PREFIX str /* Environment variables for the GekkoFS daemon */ namespace gkfs::env { static constexpr auto METADATA_DB_PATH = ADD_PREFIX("METADATA_DB_PATH"); -static constexpr auto USE_DIRENTS_COMPRESSION = - ADD_PREFIX("DAEMON_USE_DIRENTS_COMPRESSION"); -static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -static constexpr auto DAEMON_CREATE_CHECK_PARENTS = - ADD_PREFIX("DAEMON_CREATE_CHECK_PARENTS"); +static constexpr auto HOSTS_FILE = COMMON_ENV_PREFIX "HOSTS_FILE"; static constexpr auto DAEMON_CREATE_EXIST_CHECK = - ADD_PREFIX("DAEMON_CREATE_EXIST_CHECK"); -static constexpr auto DAEMON_SYMLINK_SUPPORT = - ADD_PREFIX("DAEMON_SYMLINK_SUPPORT"); -static constexpr auto DAEMON_RENAME_SUPPORT = - ADD_PREFIX("DAEMON_RENAME_SUPPORT"); -static constexpr auto DAEMON_USE_INLINE_DATA = - ADD_PREFIX("DAEMON_USE_INLINE_DATA"); + ADD_PREFIX("CREATE_EXIST_CHECK"); static constexpr auto KEEP_HOSTS_FILE = ADD_PREFIX("KEEP_HOSTS_FILE"); +static constexpr auto DAEMON_EXPAND_MODE = ADD_PREFIX("EXPAND"); +static constexpr auto ENABLE_WAL = ADD_PREFIX("ENABLE_WAL"); } // namespace gkfs::env diff --git a/include/daemon/handler/rpc_defs.hpp b/include/daemon/handler/rpc_defs.hpp index 81d39011e..6fa00ddb5 100644 --- a/include/daemon/handler/rpc_defs.hpp +++ b/include/daemon/handler/rpc_defs.hpp @@ -44,6 +44,7 @@ #define GKFS_DAEMON_RPC_DEFS_HPP #include +#include // client <-> daemon RPCs @@ -135,31 +136,21 @@ rpc_srv_get_chunk_stat(const tl::request& req, // void proxy_rpc_srv_read(const tl::request& req, ...); // void proxy_rpc_srv_write(const tl::request& req, ...); -// malleability - -void -rpc_srv_expand_start(const tl::request& req, - const gkfs::rpc::rpc_expand_start_in_t& in); - -void -rpc_srv_expand_status(const tl::request& req); - void -rpc_srv_expand_finalize(const tl::request& req); - +rpc_srv_migrate_metadata(const tl::request& req, + const gkfs::rpc::rpc_migrate_metadata_in_t& in); void -rpc_srv_shrink_start(const tl::request& req, - const gkfs::rpc::rpc_shrink_start_in_t& in); +rpc_srv_mutate_start(const tl::request& req, + const gkfs::rpc::rpc_mutate_start_in_t& in); void -rpc_srv_shrink_status(const tl::request& req); +rpc_srv_mutate_status(const tl::request& req); void -rpc_srv_shrink_finalize(const tl::request& req); +rpc_srv_mutate_finalize(const tl::request& req); void -rpc_srv_migrate_metadata(const tl::request& req, - const gkfs::rpc::rpc_migrate_metadata_in_t& in); +rpc_srv_mutate_shutdown(const tl::request& req); // inline data operations void diff --git a/include/daemon/malleability/malleable_manager.hpp b/include/daemon/malleability/malleable_manager.hpp index 6999a7653..8593fb9fe 100644 --- a/include/daemon/malleability/malleable_manager.hpp +++ b/include/daemon/malleability/malleable_manager.hpp @@ -67,23 +67,10 @@ private: int redistribute_metadata(); - void - redistribute_data(); - - int - execute_migrations(const std::vector& jobs); - /// Migration job handler: called by DataMigrationExecutor for each job int do_migration(gkfs::rpc::MigrationJob& job); - static void - expand_abt(void* _arg); - - /// Legacy data redistribution using direct ChunkStorage I/O - void - redistribute_data_legacy(); - /// New data redistribution using DataMigrationExecutor pipeline void redistribute_data_v2(); @@ -91,13 +78,12 @@ private: static void expand_abt_v2(void* _arg); -public: - void - expand_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file); + static void + expand_on_demand_abt(void* _arg); +public: void - shrink_start(int old_server_conf, int new_server_conf, + mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file); }; } // namespace gkfs::malleable diff --git a/include/daemon/ops/data.hpp b/include/daemon/ops/data.hpp index 43b7fa900..99001022e 100644 --- a/include/daemon/ops/data.hpp +++ b/include/daemon/ops/data.hpp @@ -66,6 +66,14 @@ extern "C" { namespace gkfs::data { +std::pair +expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, + char* buf, size_t size, off64_t offset); + +int +expand_on_demand_materialize_for_partial_write(const std::string& path, + uint64_t chunk_id); + /** * @brief Internal Exception for all general chunk operations. */ diff --git a/include/daemon/util.hpp b/include/daemon/util.hpp index d557060cc..7f2787ebd 100644 --- a/include/daemon/util.hpp +++ b/include/daemon/util.hpp @@ -45,10 +45,11 @@ namespace gkfs::utils { /** * @brief Registers the daemon's RPC address to the shared hosts file. + * @param expand_mode If true, prepend '+' marker to the line * @throws std::runtime_error when file stream fails */ void -populate_hosts_file(); +populate_hosts_file(bool expand_mode = false); /** * @brief Attempts to remove the entire hosts file. diff --git a/scripts/run/gkfs b/scripts/run/gkfs index d59fd99a1..3e5aebd1f 100755 --- a/scripts/run/gkfs +++ b/scripts/run/gkfs @@ -18,6 +18,58 @@ C_AST_RED="${C_BRED}*${C_NONE} [gkfs] " # Important const globals FS_INSTANCE_MARKER_CONST="#FS_INSTANCE_END" + +count_host_lines() { + local file="$1" + grep -v '^#' "${file}" 2>/dev/null | grep -cv '^[[:space:]]*$' +} + +count_before_mutate_host_lines() { + local file="$1" + grep -v '^#' "${file}" 2>/dev/null | grep -v '^[[:space:]]*+' | grep -cv '^[[:space:]]*$' +} + +ensure_malleability_bin() { + if [[ -n ${GKFS_MALLEABILITY_BIN_} ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "${GKFS_MALLEABILITY_BIN_}") + else + GKFS_MALLEABILITY_BIN_=$(command -v gkfs_malleability) + fi + if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then + if [[ -f $(dirname "${DAEMON_BIN}")/gkfs_malleability ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "${DAEMON_BIN}")/gkfs_malleability") + elif [[ -f $(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability") + else + echo -e "${C_AST_RED}ERROR: gkfs_malleability binary not found. Exiting ..." + exit 1 + fi + fi +} + +run_malleability_workflow() { + local label="$1" + local old_count="$2" + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + + ${GKFS_MALLEABILITY_BIN_} mutate start + echo -e "${C_AST_GREEN}${label} progress: " + until MALLEABILITY_STATUS=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate status 2>&1); [ $((${MALLEABILITY_STATUS})) -eq 0 ] + do + sleep 1 + show_expand_progress ${MALLEABILITY_STATUS} ${old_count} + done + show_expand_progress ${MALLEABILITY_STATUS} ${old_count} + echo + echo -e "${C_AST_GREEN}Redistribution process done. Finalizing ..." + MALLEABILITY_FINALIZE=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate finalize 2>&1) + if [ $((${MALLEABILITY_FINALIZE})) -ne 0 ]; then + echo -e "${C_AST_RED}ERROR: ${label} finalize failed. This is not recoverable. Exiting ..." + exit 1 + fi + echo -e "${C_AST_GREEN}${label} done." +} ####################################### # Poll GekkoFS hostsfile until all daemons are started. # Exits with 1 if daemons cannot be started. @@ -38,11 +90,12 @@ wait_for_gkfs_daemons() { if [[ -n ${NODE_NUM} ]]; then nodes=${NODE_NUM} fi - # when expanding the total number of nodes is: initial nodelist + expand nodelist - if [[ ${COMMAND} == *"expand"* ]]; then + # when expanding/mutating with new daemons, the hostfile contains old entries + # plus '+' entries written by the added daemons + if [[ ${GKFS_DAEMON_EXPAND} == "ON" ]]; then nodes=${NODE_CNT_EXPAND} fi - until [ $(($(grep -cv '^#' "${HOSTSFILE}" 2> /dev/null | awk '{print $1}') + 0)) -eq "${nodes}" ] + until [ $(($(count_host_lines "${HOSTSFILE}" | awk '{print $1}') + 0)) -eq "${nodes}" ] do #echo "Waiting for all servers to report connection. Try $server_wait_cnt" sleep 2 @@ -362,70 +415,6 @@ stop_daemons() { fi } -####################################### -# Sets up expand progress for later operation -# Globals: -# RUN_FOREGROUND -# EXPAND_NODELIST -# HOSTSFILE -# DAEMON_NODELIST -# USE_PROXY -# GKFS_MALLEABILITY_BIN_ -# VERBOSE -# Outputs: -# sets GKFS_MALLEABILITY_BIN_ if not already given by config -####################################### -expand_setup() { - # sanity checks - if [[ ${RUN_FOREGROUND} == true ]]; then - echo -e "${C_AST_RED}ERROR: Cannot run in foreground for expansion. Exiting ..." - exit 1 - fi - if [[ -z ${EXPAND_NODELIST} ]]; then - echo -e "${C_AST_RED}ERROR: No expand host file given. We need to know which nodes should be used. Exiting ..." - exit 1 - fi - # if proxy is enabled error out - # to support proxy, all proxies need to be shutdown during expansion and started up after again - # to get the new configuration. - if [[ ${USE_PROXY} == true ]]; then - echo -e "${C_AST_RED}ERROR: Proxy not supported for expansion. Exiting ..." - exit 1 - fi - # check that gkfs host file exists - if [[ ! -f ${HOSTSFILE} ]]; then - echo -e "${C_AST_RED}ERROR: No GekkoFS hostfile for expansion found at ${HOSTSFILE}. Exiting ..." - exit 1 - fi - # check that daemon pid file exists - if [[ ! -f ${DAEMON_PID_FILE} ]]; then - echo -e "${C_AST_RED}ERROR: No daemon pid file found at ${DAEMON_PID_FILE}." - echo -e "${C_AST_RED} Existing daemon must run in background for extension. Exiting ..." - exit 1 - fi - # modify all necessary environment variables from the config file to fit expand - DAEMON_NODELIST_=${DAEMON_NODELIST} - # Set daemon node list based on given expand hostfile - DAEMON_NODELIST_=$(readlink -f ${EXPAND_NODELIST}) - # setup - # This must be equivalent to the line set in include/common/common_defs.hpp - echo "$FS_INSTANCE_MARKER_CONST" >> "${HOSTSFILE}" - # check that the gkfs_malleability binary exists in $PATH if not already set via config - if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then - GKFS_MALLEABILITY_BIN_=$(COMMAND -v gkfs_malleability) - fi - # if not found check if it exists in the parent directory of the daemon bin - if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then - # check that the gkfs_malleability binary exists somewhere in the parent directory where daemon bin is located - if [[ -f $(dirname ${DAEMON_BIN})/gkfs_malleability ]]; then - GKFS_MALLEABILITY_BIN_=$(readlink -f $(dirname ${DAEMON_BIN})/gkfs_malleability) - else - echo -e "${C_AST_RED}ERROR: gkfs_malleability binary not found. Exiting ..." - exit 1 - fi - fi -} - ####################################### # Prints expansion progress # Input: @@ -458,43 +447,33 @@ show_expand_progress() { printf "] %d/%d left" "$current" "$total" } -####################################### -# Adds GekkoFS daemons to an existing GekkoFS instance -# Globals: -# DAEMON_PID_FILE -# PROXY_PID_FILE -# VERBOSE -# Outputs: -# Writes status to stdout -####################################### -add_daemons() { - expand_setup - # get old and new node configuration - local node_cnt_initial=$(grep -v '^#' "${HOSTSFILE}" | wc -l) - NODE_CNT_EXPAND=$((${node_cnt_initial}+$(cat ${EXPAND_NODELIST} | wc -l))) - # start new set of daemons - start_daemons - export LIBGKFS_HOSTS_FILE=${HOSTSFILE} - # start expansion which redistributes metadata and data - ${GKFS_MALLEABILITY_BIN_} expand start - echo -e "${C_AST_GREEN}Expansion progress: " - # wait for expansion to finish - until EXPAND_STATUS=$(${GKFS_MALLEABILITY_BIN_} -m expand status); [ $((${EXPAND_STATUS})) -eq 0 ] - do - sleep 1 - show_expand_progress ${EXPAND_STATUS} ${node_cnt_initial} - done - show_expand_progress ${EXPAND_STATUS} ${node_cnt_initial} - echo - # finalize and remove marker - echo -e "${C_AST_GREEN}Redistribution process done. Finalizing ..." - sed -i '/^#/d' ${HOSTSFILE} - EXPAND_FINALIZE=$(${GKFS_MALLEABILITY_BIN_} -m expand finalize) - if [ $((${EXPAND_FINALIZE})) -ne 0 ]; then - echo -e "${C_AST_RED}ERROR: Expansion finalized failed. This is not recoverable. Exiting ..." +mutate_daemons() { + if [[ ${RUN_FOREGROUND} == true ]]; then + echo -e "${C_AST_RED}ERROR: Cannot run in foreground for mutate. Exiting ..." + exit 1 + fi + if [[ ${USE_PROXY} == true ]]; then + echo -e "${C_AST_RED}ERROR: Proxy not supported for mutate. Exiting ..." exit 1 fi - echo -e "${C_AST_GREEN}Expansion done." + if [[ ! -f ${HOSTSFILE} ]]; then + echo -e "${C_AST_RED}ERROR: No GekkoFS hostfile found at ${HOSTSFILE}. Exiting ..." + exit 1 + fi + local node_cnt_initial=$(count_before_mutate_host_lines "${HOSTSFILE}") + run_malleability_workflow "Mutate" "${node_cnt_initial}" +} + +malleability_status() { + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + ${GKFS_MALLEABILITY_BIN_} mutate status +} + +malleability_finalize() { + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + ${GKFS_MALLEABILITY_BIN_} mutate finalize } ####################################### @@ -504,9 +483,9 @@ add_daemons() { ####################################### usage_short() { echo " -usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args ] [--proxy ] [-f/--foreground ] - [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-v/--verbose ] - {start,expand,stop} +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} " } ####################################### @@ -522,7 +501,7 @@ help_msg() { additional permanent configurations can be set. positional arguments: - COMMAND Command to execute: 'start', 'stop', 'expand' + COMMAND Command to execute: 'start', 'stop', 'mutate', 'status', 'finalize' optional arguments: -h, --help Shows this help message and exits @@ -538,17 +517,19 @@ help_msg() { Nodelist is extracted from Slurm via the SLURM_JOB_ID env variable. --cpuspertask <#cores> Set the number of cores the daemons can use. Must use '--srun'. -c, --config Path to configuration file. By defaults looks for a 'gkfs.conf' in this directory. - -e, --expand_hostfile Path to the hostfile with new nodes where GekkoFS should be extended to (hostfile contains one line per node). + -H, --hostsfile Path to the GekkoFS hostfile. For mutate, this may already contain '+' and '-' markers. -v, --verbose Increase verbosity -t, --time Set a limit on the total run time of the slurm job allocation. -A, --account Account for the slurm job (only required for job allocation) -P, --partition Partition for the slurm job (only required for job allocation) + malleability examples: + gkfs mutate " } CONFIGPATH="" argv=("$@") # get config path first from argument list -for i in "${argv[@]}"; do +for i in "${!argv[@]}"; do if [[ "${argv[i]}" == "-c" || "${argv[i]}" == "--config" ]]; then CONFIGPATH=$(readlink -mn "${argv[i+1]}") break @@ -593,7 +574,6 @@ HOSTSFILE=$(readlink -f ${HOSTSFILE}) PROXY_LOCAL_PID_FILE=$(readlink -f ${PROXY_LOCAL_PID_FILE}) DAEMON_PID_FILE=$(readlink -f ${DAEMON_PID_FILE}) PROXY_PID_FILE=$(readlink -f ${PROXY_PID_FILE}) -EXPAND_NODELIST="" GKFS_MALLEABILITY_BIN_=${GKFS_MALLEABILITY_BIN} # parse input @@ -659,8 +639,8 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; - -e | --expand_hostfile) - EXPAND_NODELIST=$2 + -H | --hostsfile) + HOSTSFILE=$2 shift # past argument shift # past value ;; @@ -694,6 +674,7 @@ while [[ $# -gt 0 ]]; do esac done set -- "${POSITIONAL[@]}" # restore positional parameters +HOSTSFILE=$(readlink -f ${HOSTSFILE}) # positional arguments if [[ -z ${1+x} ]]; then @@ -703,7 +684,7 @@ if [[ -z ${1+x} ]]; then fi COMMAND="${1}" # checking input -if [[ ${COMMAND} != *"start"* ]] && [[ ${COMMAND} != *"stop"* ]] && [[ ${COMMAND} != *"expand"* ]]; then +if [[ ${COMMAND} != "start" ]] && [[ ${COMMAND} != "stop" ]] && [[ ${COMMAND} != "mutate" ]] && [[ ${COMMAND} != "status" ]] && [[ ${COMMAND} != "finalize" ]]; then echo -e "${C_AST_RED}ERROR: COMMAND ${COMMAND} not supported" usage_short exit 1 @@ -713,8 +694,12 @@ if [[ ${COMMAND} == "start" ]]; then start_daemons elif [[ ${COMMAND} == "stop" ]]; then stop_daemons -elif [[ ${COMMAND} == "expand" ]]; then - add_daemons +elif [[ ${COMMAND} == "mutate" ]]; then + mutate_daemons +elif [[ ${COMMAND} == "status" ]]; then + malleability_status +elif [[ ${COMMAND} == "finalize" ]]; then + malleability_finalize fi if [[ ${VERBOSE} == true ]]; then echo -e "${C_AST_GREEN}Nothing left to do. Exiting :)" diff --git a/scripts/run/gkfs.conf b/scripts/run/gkfs.conf index 3650e682e..7c32449fd 100644 --- a/scripts/run/gkfs.conf +++ b/scripts/run/gkfs.conf @@ -8,6 +8,12 @@ PROXY_BIN=../../build/src/proxy/gkfs_proxy # client configuration (needs to be set for all clients) LIBGKFS_HOSTS_FILE=/home/XXX/workdir/gkfs_hosts.txt +# distribution configuration shared by clients, daemons, and tools +# supported examples: simple_hash, random_slicing +export GKFS_DISTRIBUTION_STRATEGY=simple_hash +# enable Random Slicing CutShift during expand-only malleability operations +export GKFS_RANDOM_SLICING_CUTSHIFT=OFF + # tools (if build) GKFS_MALLEABILITY_BIN=../../build/tools/gkfs_malleability diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 8485a590f..e6fb81b8f 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -50,6 +50,7 @@ target_sources(gkfs_common rpc/forward_malleability.cpp cache.cpp syscalls/detail/syscall_info.c syscalls/util.S + ../common/malleability_markers.cpp ) target_link_libraries( diff --git a/src/client/fuse/fuse_client.cpp b/src/client/fuse/fuse_client.cpp index 87ff84e86..8571e4b40 100644 --- a/src/client/fuse/fuse_client.cpp +++ b/src/client/fuse/fuse_client.cpp @@ -38,6 +38,7 @@ */ #include +#include #ifdef GKFS_ENABLE_CLIENT_METRICS #include #endif @@ -1405,20 +1406,20 @@ main(int argc, char* argv[]) { ud.attr_timeout = ud.timeout; ud.negative_timeout = ud.timeout; - if(const char* env_e = std::getenv("GKFS_FUSE_ENTRY_TIMEOUT")) { + if(const char* env_e = std::getenv(gkfs::env::FUSE_ENTRY_TIMEOUT)) { ud.entry_timeout = std::stod(env_e); LOG(INFO, "FUSE entry_timeout set to {}", ud.entry_timeout); } - if(const char* env_a = std::getenv("GKFS_FUSE_ATTR_TIMEOUT")) { + if(const char* env_a = std::getenv(gkfs::env::FUSE_ATTR_TIMEOUT)) { ud.attr_timeout = std::stod(env_a); LOG(INFO, "FUSE attr_timeout set to {}", ud.attr_timeout); } - if(const char* env_n = std::getenv("GKFS_FUSE_NEGATIVE_TIMEOUT")) { + if(const char* env_n = std::getenv(gkfs::env::FUSE_NEGATIVE_TIMEOUT)) { ud.negative_timeout = std::stod(env_n); LOG(INFO, "FUSE negative_timeout set to {}", ud.negative_timeout); } - if(const char* env_w = std::getenv("GKFS_FUSE_WRITEBACK")) { + if(const char* env_w = std::getenv(gkfs::env::FUSE_WRITEBACK)) { ud.writeback = (std::string(env_w) == "ON" || std::string(env_w) == "1"); LOG(INFO, "FUSE writeback set to {}", ud.writeback); diff --git a/src/client/malleability.cpp b/src/client/malleability.cpp index aaa0378c5..9019d29c4 100644 --- a/src/client/malleability.cpp +++ b/src/client/malleability.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -49,81 +50,95 @@ using namespace std; namespace gkfs::malleable { int -expand_start(int old_server_conf, int new_server_conf) { - LOG(INFO, "{}() Expand operation enter", __func__); - // sanity checks - if(old_server_conf == new_server_conf) { - auto err_str = - "ERR: Old server configuration is the same as the new one"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; - } - if(CTX->hosts().size() != static_cast(old_server_conf)) { - auto err_str = - "ERR: Old server configuration does not match the number of hosts in hostsfile"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; - } - // TODO check that hostsfile contains endmarker - return gkfs::malleable::rpc::forward_expand_start(old_server_conf, - new_server_conf); -} - -int -expand_status() { - LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_expand_status(); - LOG(INFO, "{}() '{}' nodes working on extend operation.", __func__, res); - return res; -} - -int -expand_finalize() { - LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_expand_finalize(); - LOG(INFO, "{}() extend operation finalized. ", __func__); - return res; -} - -int -shrink_start(int old_server_conf, int new_server_conf, +mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file) { - LOG(INFO, "{}() Shrink operation enter", __func__); + LOG(INFO, "{}() Mutate operation enter", __func__); // sanity checks + auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); if(old_server_conf == new_server_conf) { - auto err_str = - "ERR: Old server configuration is the same as the new one"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; + bool same_count_marker_mode = false; + if(hf) { + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + same_count_marker_mode = markers.has_markers() && + !markers.adding.empty() && + !markers.removing.empty(); + } catch(...) { + // Keep the old error below for malformed/non-marker hostfiles. + } + } + if(!same_count_marker_mode) { + auto err_str = + "ERR: Old server configuration is the same as the new one"; + cerr << err_str << endl; + LOG(ERROR, "{}() {}", __func__, err_str); + return -1; + } } - if(CTX->hosts().size() != static_cast(old_server_conf)) { + // For marker-based workflow: the CLI tool computes old_nodes = active + + // removing. But CTX->hosts() only contains active hosts (loaded at + // gkfs_init time from LIBGKFS_HOSTS_FILE). For shrink operations, the + // removing nodes are marked with '-' prefix and are not loaded into CTX. + // + // Key insight: We can't lookup removing hosts at the client side because + // they may be shutting down. Instead, the daemon handles data migration + // from its own state (which includes removing hosts parsed from markers). + // We just need to validate that the marker file has the right number of + // hosts. + uint64_t detected_hosts = CTX->hosts().size(); + bool marker_mode = false; + if(hf && detected_hosts != static_cast(old_server_conf)) { + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + uint64_t total_hosts = static_cast( + markers.active.size() + markers.removing.size()); + if(total_hosts == static_cast(old_server_conf)) { + // Marker file has the right count - this is a shrink operation + // The daemon will handle data migration using its own host list + marker_mode = true; + } + } catch(...) { + // Ignore parse errors + } + } + if(!marker_mode && + CTX->hosts().size() != static_cast(old_server_conf)) { auto err_str = "ERR: Old server configuration does not match the number of hosts in hostsfile"; cerr << err_str << endl; LOG(ERROR, "{}() {}", __func__, err_str); return -1; } - return gkfs::malleable::rpc::forward_shrink_start( + return gkfs::malleable::rpc::forward_mutate_start( old_server_conf, new_server_conf, new_hosts_file); } int -shrink_status() { +mutate_status() { LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_shrink_status(); - LOG(INFO, "{}() '{}' nodes working on shrink operation.", __func__, res); + auto res = gkfs::malleable::rpc::forward_mutate_status(); + LOG(INFO, "{}() '{}' nodes working on mutate operation.", __func__, res); return res; } int -shrink_finalize() { +mutate_finalize() { LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_shrink_finalize(); - LOG(INFO, "{}() shrink operation finalized. ", __func__); + auto res = gkfs::malleable::rpc::forward_mutate_finalize(); + if(res == 0) { + if(const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE")) { + auto shutdown_res = + gkfs::malleable::rpc::forward_mutate_shutdown_removed(hf); + if(shutdown_res != 0) { + LOG(ERROR, + "{}() failed to gracefully shutdown removed daemons: {}", + __func__, shutdown_res); + return shutdown_res; + } + } + } + LOG(INFO, "{}() mutate operation finalized. ", __func__); return res; } -} // namespace gkfs::malleable \ No newline at end of file +} // namespace gkfs::malleable diff --git a/src/client/preload.cpp b/src/client/preload.cpp index b68f169ab..e658a858b 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -50,8 +51,10 @@ #include #include +#include #include #include +#include #ifdef GKFS_ENABLE_CLIENT_METRICS #include #endif @@ -292,7 +295,7 @@ init_environment() { // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var // (defaults to simple_hash for backward compatibility) gkfs::rpc::DistributionConfig config; - const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); if(env_val != nullptr && env_val[0] != '\0') { config.set_strategy(env_val); } @@ -305,6 +308,27 @@ init_environment() { if(!distributor) { exit_error_msg(EXIT_FAILURE, "Failed to create distributor"); } + if(config.is_random_slicing()) { + if(const auto* hostfile = std::getenv(gkfs::env::HOSTS_FILE)) { + auto rs = dynamic_cast( + distributor.get()); + auto comments = + gkfs::malleable::parse_rs_interval_comments(hostfile); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + if(rs && !intervals.empty() && rs->set_intervals(intervals)) { + LOG(INFO, + "{}() Loaded {} random-slicing intervals from hostfile", + __func__, intervals.size()); + } + } + } CTX->distributor(std::move(distributor)); } diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 27733afb4..61efc9b08 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include // #include @@ -174,19 +175,14 @@ load_hostfile(const std::string& path) { path, strerror(errno))); } vector> hosts; + vector> removing_hosts; const regex line_re( "^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)$", regex::ECMAScript | regex::optimize); - - string line; std::smatch match; - while(getline(lf, line)) { - // if line starts with #, it indicates the end of current FS instance - // Further hosts are not part of the file system instance yet and are - // therefore skipped The hostfile is ordered, so nothgin below this line - // can contain valid hosts - if(line.find(gkfs::client::hostsfile_end_str) != string::npos) - break; + + auto parse_host_line = [&](const string& line, + vector>& out) { if(!regex_match(line, match, line_re)) { LOG(ERROR, "Unrecognized line format: [path: '{}', line: '{}']", path, line); @@ -197,7 +193,7 @@ load_hostfile(const std::string& path) { string host = match[1]; string uri = match[2]; // match[3] that is the proxy (not used here) - hosts.emplace_back(host, uri); + out.emplace_back(host, uri); // info will be repeated line per line: CTX->mountdir(match[4]); @@ -212,6 +208,41 @@ load_hostfile(const std::string& path) { // convert match[11] and match[12] to unsigned integers. CTX->fs_conf()->uid = std::stoi(match[11]); CTX->fs_conf()->gid = std::stoi(match[12]); + }; + + string line; + while(getline(lf, line)) { + if(line.empty()) + continue; + // if line starts with #, it indicates the end of current FS instance + // Further hosts are not part of the file system instance yet and are + // therefore skipped. The hostfile is ordered, so nothing below this + // line can contain valid hosts + if(line.find(gkfs::client::hostsfile_end_str) != string::npos) + break; + // Skip comment lines (lines starting with #) + if(line[0] == '#') + continue; + + if(line[0] == '+') { + // Adding daemons are reachable, but they are not part of the old + // distribution at client bootstrap. mutate_start contacts them via + // marker parsing in forward_malleability.cpp. + continue; + } + + if(line[0] == '-') { + parse_host_line(gkfs::malleable::strip_marker(line), + removing_hosts); + continue; + } + + parse_host_line(line, hosts); + } + if(hosts.empty() && !removing_hosts.empty()) { + LOG(INFO, + "Hosts file contains no unmarked daemons; bootstrapping from '-' marked daemons for mutate"); + hosts = std::move(removing_hosts); } if(hosts.empty()) { throw runtime_error( diff --git a/src/client/rpc/forward_malleability.cpp b/src/client/rpc/forward_malleability.cpp index 0fedf4aa6..3c63d0b11 100644 --- a/src/client/rpc/forward_malleability.cpp +++ b/src/client/rpc/forward_malleability.cpp @@ -9,7 +9,7 @@ ADA-FS project under the SPPEXA project funded by the DFG. This software was partially supported by the - the European Union’s Horizon 2020 JTI-EuroHPC research and + the European Union's Horizon 2020 JTI-EuroHPC research and innovation programme, by the project ADMIRE (Project ID: 956748, admire-eurohpc.eu) @@ -27,141 +27,165 @@ #include #include #include +#include -namespace gkfs::malleable::rpc { - -int -forward_expand_start(int old_server_conf, int new_server_conf) { - LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); +#include +#include +#include +#include - auto err = 0; - std::vector waiters; - waiters.reserve(targets.size()); - std::vector waiter_targets; - waiter_targets.reserve(targets.size()); +namespace { - // define rpc - auto expand_start_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::expand_start); +std::vector> +mutate_endpoints_from_loaded_hosts() { + std::vector> endpoints; + const auto& targets = CTX->distributor()->locate_directory_metadata(); + endpoints.reserve(targets.size()); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(auto target : targets) { try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - - gkfs::rpc::rpc_expand_start_in_t in; - in.old_server_conf = old_server_conf; - in.new_server_conf = new_server_conf; - - waiters.push_back( - expand_start_rpc.on(CTX->hosts().at(target)).async(in)); - waiter_targets.push_back(target); + endpoints.emplace_back(std::to_string(target), + CTX->hosts().at(target)); } catch(const std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); - // Continue to try others? Or fail? - // Margo code continued. + LOG(ERROR, "Failed to get mutate endpoint for host {}: {}", target, + ex.what()); } } - LOG(INFO, "{}() send expand_start rpc to '{}' targets", __func__, - waiters.size()); + return endpoints; +} - // wait for RPC responses - for(std::size_t i = 0; i < waiters.size(); ++i) { +std::vector> +mutate_endpoints_from_markers(const std::string& hostfile, + bool include_removing) { + std::vector> endpoints; + std::set seen_uris; + auto markers = gkfs::malleable::parse_hostfile_markers(hostfile); + + if(!markers.has_markers()) { + return endpoints; + } + + auto add_entry = [&](const gkfs::malleable::HostEntry& entry) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + return; + } try { - gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); - if(out.err != 0) { - err = out.err; - } + LOG(DEBUG, "Looking up mutate endpoint '{}'", entry.uri); + endpoints.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], + LOG(ERROR, "Failed to lookup mutate endpoint '{}': {}", entry.uri, ex.what()); - err = EBUSY; + } + }; + + for(const auto& entry : markers.active) { + add_entry(entry); + } + for(const auto& entry : markers.adding) { + add_entry(entry); + } + if(include_removing) { + for(const auto& entry : markers.removing) { + add_entry(entry); } } - return err; + + return endpoints; } -int -forward_expand_status() { - LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); +std::vector> +mutate_removed_endpoints_from_markers(const std::string& hostfile) { + std::vector> endpoints; + std::set seen_uris; + auto markers = gkfs::malleable::parse_hostfile_markers(hostfile); - auto err = 0; - std::vector waiters; - waiters.reserve(targets.size()); - std::vector waiter_targets; - waiter_targets.reserve(targets.size()); + for(const auto& entry : markers.removing) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + try { + LOG(DEBUG, "Looking up removed mutate endpoint '{}'", entry.uri); + endpoints.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to lookup removed mutate endpoint '{}': {}", + entry.uri, ex.what()); + } + } - auto expand_status_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::expand_status); + return endpoints; +} - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; +std::vector> +mutate_endpoints(const std::string* hostfile, bool include_removing) { + if(hostfile && !hostfile->empty()) { try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - expand_status_rpc.on(CTX->hosts().at(target)).async()); - waiter_targets.push_back(target); + auto endpoints = + mutate_endpoints_from_markers(*hostfile, include_removing); + if(!endpoints.empty()) { + return endpoints; + } } catch(const std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); + LOG(DEBUG, "No marker mutate endpoint list from '{}': {}", + *hostfile, ex.what()); } } - LOG(INFO, "{}() send expand_status rpc to '{}' targets", __func__, - waiters.size()); - - // wait for RPC responses - for(std::size_t i = 0; i < waiters.size(); ++i) { + if(const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE")) { try { - gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); - if(out.err > 0) { - LOG(DEBUG, "{}() Host '{}' not done yet.", __func__, - waiter_targets[i]); - err += out.err; - } else if(out.err < 0) { - LOG(ERROR, "{}() Host '{}' error.", __func__, - waiter_targets[i]); - // Margo logic didn't update global err? "err += - // mercury_out.err" only if > 0. But it logged error. + auto endpoints = + mutate_endpoints_from_markers(hf, include_removing); + if(!endpoints.empty()) { + return endpoints; } } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], + LOG(DEBUG, "No marker mutate endpoint list from '{}': {}", hf, ex.what()); - err = EBUSY; } } - return err; + + return mutate_endpoints_from_loaded_hosts(); } +} // namespace + +namespace gkfs::malleable::rpc { + int -forward_expand_finalize() { +forward_mutate_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file) { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(&new_hosts_file, true); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto expand_finalize_rpc = CTX->rpc_engine()->define( - gkfs::malleable::rpc::tag::expand_finalize); + // define rpc + auto mutate_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_start); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - expand_finalize_rpc.on(CTX->hosts().at(target)).async()); + + gkfs::rpc::rpc_mutate_start_in_t in; + in.old_server_conf = old_server_conf; + in.new_server_conf = new_server_conf; + in.new_hosts_file = new_hosts_file; + + waiters.push_back(mutate_rpc.on(endpoint).async(in)); waiter_targets.push_back(target); - } catch(std::exception& ex) { + } catch(const std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); } } - LOG(INFO, "{}() send expand_finalize rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_start rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses @@ -169,7 +193,6 @@ forward_expand_finalize() { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); if(out.err != 0) { - LOG(ERROR, "Failed finalize on host '{}'", waiter_targets[i]); err = out.err; } } catch(const std::exception& ex) { @@ -182,48 +205,43 @@ forward_expand_finalize() { } int -forward_shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file) { +forward_mutate_status() { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(nullptr, true); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - // define rpc - auto shrink_start_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::shrink_start); + auto mutate_status_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_status); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - - gkfs::rpc::rpc_shrink_start_in_t in; - in.old_server_conf = old_server_conf; - in.new_server_conf = new_server_conf; - in.new_hosts_file = new_hosts_file; - - waiters.push_back( - shrink_start_rpc.on(CTX->hosts().at(target)).async(in)); + waiters.push_back(mutate_status_rpc.on(endpoint).async()); waiter_targets.push_back(target); } catch(const std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); } } - LOG(INFO, "{}() send shrink_start rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_status rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses for(std::size_t i = 0; i < waiters.size(); ++i) { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); - if(out.err != 0) { - err = out.err; + if(out.err > 0) { + LOG(DEBUG, "{}() Host '{}' not done yet.", __func__, + waiter_targets[i]); + err += out.err; + } else if(out.err < 0) { + LOG(ERROR, "{}() Host '{}' error.", __func__, + waiter_targets[i]); } } catch(const std::exception& ex) { LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], @@ -235,45 +253,39 @@ forward_shrink_start(int old_server_conf, int new_server_conf, } int -forward_shrink_status() { +forward_mutate_finalize() { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(nullptr, true); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto shrink_status_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::shrink_status); + auto mutate_finalize_rpc = CTX->rpc_engine()->define( + gkfs::malleable::rpc::tag::mutate_finalize); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - shrink_status_rpc.on(CTX->hosts().at(target)).async()); + waiters.push_back(mutate_finalize_rpc.on(endpoint).async()); waiter_targets.push_back(target); - } catch(const std::exception& ex) { + } catch(std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); } } - LOG(INFO, "{}() send shrink_status rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_finalize rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses for(std::size_t i = 0; i < waiters.size(); ++i) { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); - if(out.err > 0) { - LOG(DEBUG, "{}() Host '{}' not done yet.", __func__, - waiter_targets[i]); - err += out.err; - } else if(out.err < 0) { - LOG(ERROR, "{}() Host '{}' error.", __func__, - waiter_targets[i]); + if(out.err != 0) { + LOG(ERROR, "Failed finalize on host '{}'", waiter_targets[i]); + err = out.err; } } catch(const std::exception& ex) { LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], @@ -285,49 +297,69 @@ forward_shrink_status() { } int -forward_shrink_finalize() { +forward_mutate_shutdown_removed(const std::string& hostfile) { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + + std::vector> targets; + try { + targets = mutate_removed_endpoints_from_markers(hostfile); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to parse removed hosts from '{}': {}", hostfile, + ex.what()); + return EINVAL; + } + + if(targets.empty()) { + LOG(INFO, "{}() no removed daemons to shutdown", __func__); + return 0; + } auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto shrink_finalize_rpc = CTX->rpc_engine()->define( - gkfs::malleable::rpc::tag::shrink_finalize); + auto mutate_shutdown_rpc = CTX->rpc_engine()->define( + gkfs::malleable::rpc::tag::mutate_shutdown); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - shrink_finalize_rpc.on(CTX->hosts().at(target)).async()); + LOG(DEBUG, "{}() Sending graceful shutdown RPC to host: '{}'", + __func__, target); + waiters.push_back(mutate_shutdown_rpc.on(endpoint).async()); waiter_targets.push_back(target); - } catch(std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to send shutdown RPC to host {}: {}", target, + ex.what()); + err = EBUSY; } } - LOG(INFO, "{}() send shrink_finalize rpc to '{}' targets", __func__, - waiters.size()); + LOG(INFO, "{}() sent graceful shutdown RPC to '{}' removed targets", + __func__, waiters.size()); - // wait for RPC responses for(std::size_t i = 0; i < waiters.size(); ++i) { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); if(out.err != 0) { - LOG(ERROR, "Failed finalize on host '{}'", waiter_targets[i]); + LOG(ERROR, "Failed graceful shutdown on host '{}'", + waiter_targets[i]); err = out.err; } } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], - ex.what()); + LOG(ERROR, "Shutdown RPC wait failed for target {}: {}", + waiter_targets[i], ex.what()); err = EBUSY; } } + return err; } -} // namespace gkfs::malleable::rpc \ No newline at end of file +// Legacy forwarding functions (the headers already provide inline aliases +// above, so these are no-op stubs kept to avoid symbol conflicts on some +// linkers. They delegate to the new unified mutate functions. +// If the header already defines them as inline, these are no-ops. + +} // namespace gkfs::malleable::rpc diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 67f61ec3d..4e9eae88f 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -50,6 +50,7 @@ target_sources(distributor ${INCLUDE_DIR}/common/rpc/cutshift_sorted.hpp ${INCLUDE_DIR}/common/rpc/data_migration_executor.hpp ${INCLUDE_DIR}/common/rpc/distribution_config.hpp + ${INCLUDE_DIR}/common/env.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor.cpp ${CMAKE_CURRENT_LIST_DIR}/rpc/random_slicing_distributor.cpp diff --git a/src/common/malleability_markers.cpp b/src/common/malleability_markers.cpp new file mode 100644 index 000000000..5c5c959f3 --- /dev/null +++ b/src/common/malleability_markers.cpp @@ -0,0 +1,278 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace gkfs { +namespace malleable { + +bool +is_marker_line(const std::string& line) { + if(line.empty()) + return false; + return line[0] == MARKER_ADD || line[0] == MARKER_REMOVE; +} + +char +get_marker(const std::string& line) { + if(line.empty()) + return '\0'; + if(line[0] == MARKER_ADD) + return MARKER_ADD; + if(line[0] == MARKER_REMOVE) + return MARKER_REMOVE; + return '\0'; +} + +std::string +strip_marker(const std::string& line) { + if(!is_marker_line(line)) + return line; + + auto first = line.find_first_not_of(" \t", 1); + if(first == std::string::npos) + return ""; + return line.substr(first); +} + +HostfileMarkers +parse_hostfile_markers(const std::string& path) { + HostfileMarkers result; + + std::ifstream file(path); + if(!file.is_open()) { + throw std::runtime_error( + fmt::format("Failed to open hostfile with markers: '{}': {}", + path, strerror(errno))); + } + + std::string line; + while(std::getline(file, line)) { + // Skip empty lines + if(line.empty()) + continue; + + // Trim trailing CR if present (Windows line endings) + if(!line.empty() && line.back() == '\r') { + line.pop_back(); + } + + // Skip comments + if(line[0] == '#') + continue; + + HostEntry entry; + char marker = get_marker(line); + + if(marker != '\0') { + // Has marker prefix + entry.marker = std::string(1, marker); + // Strip marker to get the actual content + std::string content = strip_marker(line); + if(content.empty()) + continue; // Skip empty entries after strip + + // Parse the content: hostname uri field1 ... field12 + std::istringstream iss(content); + if(!(iss >> entry.hostname >> entry.uri)) { + continue; // Skip malformed entries + } + + // Read remaining fields + std::string field; + while(std::getline(iss, field, ' ')) { + if(!field.empty()) { + entry.extra_fields.push_back(field); + } + } + } else { + // No marker - active entry + entry.marker = ""; + std::istringstream iss(line); + if(!(iss >> entry.hostname >> entry.uri)) { + continue; // Skip malformed entries + } + + std::string field; + while(std::getline(iss, field, ' ')) { + if(!field.empty()) { + entry.extra_fields.push_back(field); + } + } + } + + entry.raw_line = line; + + // Categorize by marker + if(marker == MARKER_ADD) { + result.adding.push_back(entry); + } else if(marker == MARKER_REMOVE) { + result.removing.push_back(entry); + } else { + result.active.push_back(entry); + } + } + + return result; +} + +void +write_clean_hostfile(const std::string& path, const HostfileMarkers& markers) { + write_clean_hostfile(path, markers, {}); +} + +std::vector +parse_rs_interval_comments(const std::string& path) { + std::vector intervals; + + std::ifstream file(path); + if(!file.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to open hostfile for RS intervals: '{}': {}", path, + strerror(errno))); + } + + std::string line; + while(std::getline(file, line)) { + if(line.rfind("# GKFS_RS_INTERVAL ", 0) != 0) { + continue; + } + + RandomSlicingIntervalComment interval; + std::istringstream iss( + line.substr(std::string("# GKFS_RS_INTERVAL ").size())); + std::string token; + bool have_host = false; + bool have_start = false; + bool have_end = false; + while(iss >> token) { + const auto pos = token.find('='); + if(pos == std::string::npos) { + continue; + } + const auto key = token.substr(0, pos); + const auto value = token.substr(pos + 1); + if(key == "host") { + interval.host_id = static_cast(std::stoull(value)); + have_host = true; + } else if(key == "start") { + interval.start = std::stod(value); + have_start = true; + } else if(key == "end") { + interval.end = std::stod(value); + have_end = true; + } + } + if(have_host && have_start && have_end && + interval.start < interval.end) { + intervals.push_back(interval); + } + } + + return intervals; +} + +void +write_clean_hostfile( + const std::string& path, const HostfileMarkers& markers, + const std::vector& rs_intervals) { + std::ofstream file(path, std::ios::trunc); + if(!file.is_open()) { + throw std::runtime_error( + fmt::format("Failed to write clean hostfile: '{}': {}", path, + strerror(errno))); + } + + // Write active entries (unchanged) + for(const auto& entry : markers.active) { + file << strip_marker(entry.raw_line) << "\n"; + } + + // Promote adding entries (remove marker prefix) + for(const auto& entry : markers.adding) { + file << strip_marker(entry.raw_line) << "\n"; + } + + // Removing entries are not written (they are dropped) + + for(const auto& interval : rs_intervals) { + file << "# GKFS_RS_INTERVAL host=" << interval.host_id + << " start=" << std::fixed << std::setprecision(9) + << interval.start << " end=" << interval.end << "\n"; + } + + file.close(); +} + +void +write_rs_interval_comments( + const std::string& path, + const std::vector& rs_intervals) { + std::ifstream in(path); + if(!in.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to read hostfile for RS intervals: '{}': {}", path, + strerror(errno))); + } + + std::vector lines; + std::string line; + while(std::getline(in, line)) { + if(line.rfind("# GKFS_RS_INTERVAL ", 0) == 0) { + continue; + } + lines.push_back(line); + } + in.close(); + + std::ofstream out(path, std::ios::trunc); + if(!out.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to write hostfile for RS intervals: '{}': {}", path, + strerror(errno))); + } + for(const auto& kept_line : lines) { + out << kept_line << "\n"; + } + for(const auto& interval : rs_intervals) { + out << "# GKFS_RS_INTERVAL host=" << interval.host_id + << " start=" << std::fixed << std::setprecision(9) << interval.start + << " end=" << interval.end << "\n"; + } +} + +} // namespace malleable +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/cutshift_sorted.cpp b/src/common/rpc/cutshift_sorted.cpp index fc9c7b543..e608f0019 100644 --- a/src/common/rpc/cutshift_sorted.cpp +++ b/src/common/rpc/cutshift_sorted.cpp @@ -25,76 +25,193 @@ namespace gkfs { namespace rpc { -// ponytail: CutShift+Sorted algorithm from thesis Chapter 4 -// Simplification: uniform capacity per node (heterogeneous weights are Phase 4) +namespace { + +constexpr float epsilon = 1.0e-6f; + +float +interval_size(const Interval& interval) { + return interval.end - interval.start; +} + +void +merge_adjacent_gaps(std::vector& gaps) { + if(gaps.empty()) { + return; + } + + std::sort(gaps.begin(), gaps.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + + std::vector merged; + merged.reserve(gaps.size()); + merged.push_back(gaps.front()); + + for(size_t i = 1; i < gaps.size(); ++i) { + auto& last = merged.back(); + const auto& current = gaps[i]; + if(current.start <= last.end + epsilon) { + last.end = std::max(last.end, current.end); + } else { + merged.push_back(current); + } + } + + gaps = std::move(merged); +} + +void +sort_gaps_largest_first(std::vector& gaps) { + std::sort(gaps.begin(), gaps.end(), + [](const Interval& a, const Interval& b) { + const auto size_a = interval_size(a); + const auto size_b = interval_size(b); + if(std::abs(size_a - size_b) > epsilon) { + return size_a > size_b; + } + return a.start < b.start; + }); +} std::vector -collect_gaps_cutshift(const std::vector& old_partitions, - const std::unordered_map& reductions) { +collect_gaps_cutshift_in_place( + std::vector& partitions, + const std::unordered_map& reductions) { std::vector gaps; + bool cut_from_beginning = true; - // Sort old partitions by host_id for deterministic iteration - std::vector sorted_parts(old_partitions.begin(), - old_partitions.end()); - std::sort(sorted_parts.begin(), sorted_parts.end(), - [](const Partition& a, const Partition& b) { - return a.host_id < b.host_id; + std::vector partition_order(partitions.size()); + for(size_t i = 0; i < partition_order.size(); ++i) { + partition_order[i] = i; + } + std::sort(partition_order.begin(), partition_order.end(), + [&partitions](auto a, auto b) { + return partitions[a].host_id < partitions[b].host_id; }); - for(const auto& part : sorted_parts) { - auto it = reductions.find(part.host_id); - if(it == reductions.end()) - continue; // no reduction needed - float remaining_reduction = it->second; - if(remaining_reduction <= 0.0f) + for(const auto partition_index : partition_order) { + auto& part = partitions[partition_index]; + auto reduction_it = reductions.find(part.host_id); + if(reduction_it == reductions.end()) { continue; + } + + auto remaining_reduction = reduction_it->second; + if(remaining_reduction <= epsilon || part.intervals.empty()) { + continue; + } + + std::vector interval_order(part.intervals.size()); + for(size_t i = 0; i < interval_order.size(); ++i) { + interval_order[i] = i; + } + + // CutShift first tries to release complete intervals. Taking smaller + // intervals first maximizes the number of complete gaps before one + // partial split is required. + std::sort(interval_order.begin(), interval_order.end(), + [&part](auto a, auto b) { + const auto size_a = interval_size(part.intervals[a]); + const auto size_b = interval_size(part.intervals[b]); + if(std::abs(size_a - size_b) > epsilon) { + return size_a < size_b; + } + return part.intervals[a].start < part.intervals[b].start; + }); - // ponytail: iterate intervals and shrink from the end (alternating - // would be more complex) For Phase 1, we shrink from the end of the - // last interval - if(!part.intervals.empty()) { - auto& last_iv = - const_cast&>(part.intervals).back(); - float shrink_amount = - std::min(remaining_reduction, last_iv.end - last_iv.start); - if(shrink_amount > 0.0f) { - // Create a gap from the end - Interval gap; - gap.start = last_iv.end - shrink_amount; - gap.end = last_iv.end; - gap.host_id = 0; // 0 = empty space - gaps.push_back(gap); - - // Shrink the original interval - last_iv.end -= shrink_amount; + std::vector remove_interval(part.intervals.size(), false); + for(const auto interval_index : interval_order) { + if(remaining_reduction <= epsilon) { + break; + } + + const auto& interval = part.intervals[interval_index]; + const auto size = interval_size(interval); + if(size <= epsilon || size > remaining_reduction + epsilon) { + continue; + } + + gaps.push_back({interval.start, interval.end, 0}); + remove_interval[interval_index] = true; + remaining_reduction -= size; + } + + if(remaining_reduction > epsilon) { + auto split_it = std::find_if( + interval_order.rbegin(), interval_order.rend(), + [&part, &remove_interval](auto interval_index) { + return !remove_interval[interval_index] && + interval_size(part.intervals[interval_index]) > + epsilon; + }); + + if(split_it != interval_order.rend()) { + auto& interval = part.intervals[*split_it]; + const auto shrink_amount = + std::min(remaining_reduction, interval_size(interval)); + + if(cut_from_beginning) { + gaps.push_back({interval.start, + interval.start + shrink_amount, 0}); + interval.start += shrink_amount; + } else { + gaps.push_back( + {interval.end - shrink_amount, interval.end, 0}); + interval.end -= shrink_amount; + } + + cut_from_beginning = !cut_from_beginning; remaining_reduction -= shrink_amount; } } - } - // Sort gaps by size (largest first) for greedy packing - std::sort(gaps.begin(), gaps.end(), - [](const Interval& a, const Interval& b) { - return (a.end - a.start) > (b.end - b.start); - }); + std::vector kept; + kept.reserve(part.intervals.size()); + for(size_t i = 0; i < part.intervals.size(); ++i) { + if(remove_interval[i] || + interval_size(part.intervals[i]) <= epsilon) { + continue; + } + part.intervals[i].host_id = part.host_id; + kept.push_back(part.intervals[i]); + } + std::sort(kept.begin(), kept.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + part.intervals = std::move(kept); + part.total_capacity = part.coverage(); + } + merge_adjacent_gaps(gaps); + sort_gaps_largest_first(gaps); return gaps; } +} // namespace + +// CutShift+Sorted algorithm from thesis Chapter 4 +// Simplification: uniform capacity per node (heterogeneous weights are Phase 4) + +std::vector +collect_gaps_cutshift(const std::vector& old_partitions, + const std::unordered_map& reductions) { + auto scratch_partitions = old_partitions; + return collect_gaps_cutshift_in_place(scratch_partitions, reductions); +} + std::vector assign_gaps_to_new_nodes(std::vector gaps, const std::vector& new_hosts, const std::vector& old_partitions) { - // ponytail: compute per-node capacity as average of old node capacities - float avg_capacity = 0.0f; - for(const auto& p : old_partitions) { - for(const auto& iv : p.intervals) { - avg_capacity += (iv.end - iv.start); - } + float new_node_capacity = 0.0f; + for(const auto& gap : gaps) { + new_node_capacity += (gap.end - gap.start); + } + if(!new_hosts.empty()) { + new_node_capacity /= static_cast(new_hosts.size()); } - avg_capacity /= old_partitions.size(); std::vector new_partitions; size_t gap_idx = 0; @@ -102,15 +219,21 @@ assign_gaps_to_new_nodes(std::vector gaps, for(host_t host : new_hosts) { Partition p; p.host_id = host; - p.total_capacity = avg_capacity; + p.total_capacity = new_node_capacity; - float needed = avg_capacity; - while(needed > 0.0f && gap_idx < gaps.size()) { + float needed = new_node_capacity; + while(needed > epsilon && gap_idx < gaps.size()) { Interval g = gaps[gap_idx]; - float gap_size = g.end - g.start; + float gap_size = interval_size(g); + + if(gap_size <= epsilon) { + gap_idx++; + continue; + } - if(gap_size <= needed) { + if(gap_size <= needed + epsilon) { // Take the entire gap + g.host_id = host; p.intervals.push_back(g); needed -= gap_size; gap_idx++; @@ -118,7 +241,10 @@ assign_gaps_to_new_nodes(std::vector gaps, // Take part of the gap Interval partial = g; partial.end = partial.start + needed; - p.intervals.push_back(partial); + partial.host_id = host; + if(interval_size(partial) > epsilon) { + p.intervals.push_back(partial); + } // Leave remaining gap for next node g.start = partial.end; gaps[gap_idx] = g; @@ -130,7 +256,7 @@ assign_gaps_to_new_nodes(std::vector gaps, } // Add remaining gaps back (they'll be distributed among old nodes) - // ponytail: for Phase 1, we discard remaining gaps — they represent + // for Phase 1, we discard remaining gaps — they represent // capacity that was removed from the cluster return new_partitions; @@ -162,14 +288,16 @@ expand_with_cutshift(std::vector current_partitions, } } - // Collect gaps from shrinking nodes - auto gaps = collect_gaps_cutshift(current_partitions, reductions); + // Shrink old partitions and collect the released intervals. This is the + // CutShift phase: complete intervals are assimilated first; when a split is + // required, split side alternates between beginning and end. + auto gaps = collect_gaps_cutshift_in_place(current_partitions, reductions); // Create new partitions for new nodes from gaps auto new_partitions = assign_gaps_to_new_nodes(gaps, new_hosts, current_partitions); - // Shrink old partitions + // Finalize old partition capacities after the in-place CutShift shrink. for(auto& p : current_partitions) { auto it = reductions.find(p.host_id); if(it != reductions.end()) { diff --git a/src/common/rpc/data_migrator.cpp b/src/common/rpc/data_migrator.cpp index ed6ef337e..b750a333c 100644 --- a/src/common/rpc/data_migrator.cpp +++ b/src/common/rpc/data_migrator.cpp @@ -26,7 +26,7 @@ namespace gkfs { namespace rpc { -// ponytail: reuse RandomSlicingDistributor's hashing for consistent host +// reuse RandomSlicingDistributor's hashing for consistent host // lookups host_t find_host_for(const std::vector& partitions, const std::string& path, @@ -76,7 +76,7 @@ DataMigrator::compute_migrations(const std::vector& old_partitions, int chunk_sample_size) { std::vector jobs; - // ponytail: sample a fixed set of chunk IDs per file path + // sample a fixed set of chunk IDs per file path // This is an estimation — in practice the MDS knows exact chunk mappings for(int chunk_id = 0; chunk_id < chunk_sample_size; ++chunk_id) { // Use a synthetic path for estimation diff --git a/src/common/rpc/distribution_config.cpp b/src/common/rpc/distribution_config.cpp index 8b10d6ad8..520eb90c5 100644 --- a/src/common/rpc/distribution_config.cpp +++ b/src/common/rpc/distribution_config.cpp @@ -19,6 +19,7 @@ */ #include "common/rpc/distribution_config.hpp" +#include #include namespace gkfs { @@ -33,7 +34,7 @@ create_from_config(const DistributionConfig& config, host_t localhost, DistributionStrategy read_strategy_from_env() { - const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); if(env_val == nullptr || env_val[0] == '\0') { return DistributionConfig::default_strategy(); } diff --git a/src/common/rpc/distributor.cpp b/src/common/rpc/distributor.cpp index 8fd771284..9d9c0265a 100644 --- a/src/common/rpc/distributor.cpp +++ b/src/common/rpc/distributor.cpp @@ -65,6 +65,8 @@ SimpleHashDistributor::hosts_size() const { void SimpleHashDistributor::hosts_size(unsigned int size) { hosts_size_ = size; + all_hosts_ = std::vector(size); + ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } host_t diff --git a/src/common/rpc/distributor_factory.cpp b/src/common/rpc/distributor_factory.cpp index c366bb0d2..6551962f0 100644 --- a/src/common/rpc/distributor_factory.cpp +++ b/src/common/rpc/distributor_factory.cpp @@ -38,12 +38,12 @@ strategy_to_string(DistributionStrategy strategy) { case DistributionStrategy::Forwarder: return "forwarder"; } - return "unknown"; // ponytail: unreachable default + return "unknown"; // unreachable default } DistributionStrategy string_to_strategy(const std::string& str) { - // ponytail: case-insensitive comparison + // case-insensitive comparison std::string lower = str; std::transform(lower.begin(), lower.end(), lower.begin(), [](unsigned char c) { return std::tolower(c); }); @@ -75,7 +75,7 @@ create_distributor(DistributionStrategy strategy, host_t localhost, case DistributionStrategy::Forwarder: return std::make_unique(localhost, fwd_host); } - return nullptr; // ponytail: should never happen + return nullptr; // should never happen } std::unique_ptr diff --git a/src/common/rpc/random_slicing_distributor.cpp b/src/common/rpc/random_slicing_distributor.cpp index ed484a383..e54d50b3b 100644 --- a/src/common/rpc/random_slicing_distributor.cpp +++ b/src/common/rpc/random_slicing_distributor.cpp @@ -19,16 +19,36 @@ */ #include "common/rpc/random_slicing_distributor.hpp" +#include "common/env.hpp" +#include "common/rpc/cutshift_sorted.hpp" #include #include #include #include #include +#include +#include namespace gkfs { namespace rpc { +namespace { + +bool +random_slicing_cutshift_enabled() { + const auto* env_value = std::getenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + if(env_value == nullptr || env_value[0] == '\0') { + return false; + } + + const std::string value(env_value); + return value == "1" || value == "ON" || value == "on" || value == "TRUE" || + value == "true" || value == "YES" || value == "yes"; +} + +} // namespace + // ===== IntervalIndex implementation ===== void @@ -77,7 +97,7 @@ uint64_t RandomSlicingDistributor::hash_seed(const std::string& path, chunkid_t chnk_id) const { // Combine path and chunk_id into a 64-bit seed using FNV-1a hash - // ponytail: FNV-1a is faster than SHA1 and provides adequate distribution + // FNV-1a is faster than SHA1 and provides adequate distribution // for random slicing with minstd_rand uint64_t hash = 14695981039346656037ULL; // FNV offset basis for(char c : path) { @@ -98,7 +118,7 @@ RandomSlicingDistributor::init_partitions_from_hosts() { if(all_hosts_.empty()) return; - // ponytail: assume uniform capacity (weight=1) for now. + // assume uniform capacity (weight=1) for now. // Future: read weight from host metadata in RPCData unsigned int n = static_cast(all_hosts_.size()); float capacity_per_host = 1.0f / static_cast(n); @@ -124,7 +144,7 @@ RandomSlicingDistributor::init_partitions_from_hosts() { RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, unsigned int hosts_size) : localhost_(localhost), hosts_size_(hosts_size), - prng_(42) { // ponytail: fixed seed for determinism + prng_(42) { // fixed seed for determinism all_hosts_.resize(hosts_size); for(unsigned int i = 0; i < hosts_size; ++i) { all_hosts_[i] = i; @@ -147,7 +167,7 @@ RandomSlicingDistributor::hosts_size() const { void RandomSlicingDistributor::hosts_size(unsigned int size) { hosts_size_ = size; - // ponytail: hosts_size() setter doesn't auto-reconfigure. + // hosts_size() setter doesn't auto-reconfigure. // Caller must call reconfigure() or add_nodes()/remove_nodes() explicitly. } @@ -155,15 +175,13 @@ host_t RandomSlicingDistributor::locate_data(const std::string& path, const chunkid_t& chnk_id, const int num_copy) const { - // ponytail: for num_copy > 1, we'd need to find distinct hosts. // Simplification: return primary host only, same as current // SimpleHashDistributor. The replication is handled at a higher level by // the MDS. uint64_t seed = hash_seed(path, chnk_id); // Use mt19937 for better distribution quality (FISHER-YATES recommendation - // #4 from thesis) ponytail: mt19937 has excellent distribution properties - // and is the gold standard PRNG + // #4 from thesis) std::mt19937_64 prng(static_cast(seed ^ (seed >> 16))); std::uniform_real_distribution dist(0.0f, 1.0f); float x = dist(prng); @@ -171,7 +189,7 @@ RandomSlicingDistributor::locate_data(const std::string& path, host_t host = interval_idx_.find_host(x); // Fallback should never happen if intervals cover [0, 1) - return (host != 0) ? host : (localhost_ != 0 ? localhost_ : 0); + return host; } host_t @@ -185,13 +203,13 @@ RandomSlicingDistributor::locate_data(const std::string& path, host_t RandomSlicingDistributor::locate_file_metadata(const std::string& path, const int num_copy) const { - // ponytail: use same algorithm as locate_data for consistency + // use same algorithm as locate_data for consistency return locate_data(path, 0, num_copy); } std::vector RandomSlicingDistributor::locate_directory_metadata() const { - // ponytail: distribute directory metadata across all nodes + // distribute directory metadata across all nodes std::vector result; result.reserve(all_hosts_.size()); for(auto h : all_hosts_) { @@ -207,32 +225,86 @@ RandomSlicingDistributor::reconfigure() { init_partitions_from_hosts(); } +bool +RandomSlicingDistributor::set_intervals( + const std::vector& intervals) { + if(intervals.empty()) { + return false; + } + + std::vector partitions; + partitions.reserve(hosts_size_); + for(auto host : all_hosts_) { + Partition partition; + partition.host_id = host; + partitions.push_back(partition); + } + + for(auto interval : intervals) { + if(interval.start < 0.0f || interval.end > 1.0f || + interval.start >= interval.end || interval.host_id >= hosts_size_) { + return false; + } + interval.host_id = static_cast(interval.host_id); + partitions[interval.host_id].intervals.push_back(interval); + } + + for(auto& partition : partitions) { + std::sort( + partition.intervals.begin(), partition.intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + partition.total_capacity = partition.coverage(); + } + + auto sorted = intervals; + std::sort(sorted.begin(), sorted.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + constexpr float epsilon = 1.0e-5f; + if(std::abs(sorted.front().start) > epsilon || + std::abs(sorted.back().end - 1.0f) > epsilon) { + return false; + } + for(size_t i = 1; i < sorted.size(); ++i) { + if(std::abs(sorted[i].start - sorted[i - 1].end) > epsilon) { + return false; + } + } + + partitions_ = std::move(partitions); + interval_idx_.build(partitions_); + return true; +} + +std::vector +RandomSlicingDistributor::get_intervals() const { + return interval_idx_.intervals(); +} + void RandomSlicingDistributor::add_nodes(std::vector new_nodes) { if(new_nodes.empty()) return; - // Append new hosts and recompute + const auto old_host_count = all_hosts_.size(); + for(auto h : new_nodes) { all_hosts_.push_back(h); } hosts_size_ = static_cast(all_hosts_.size()); - init_partitions_from_hosts(); - // ponytail: full reconfiguration on add. - // The thesis CutShift+Sorted algorithm would be more efficient, - // but full reconfig is correct and simpler for Phase 1. - // TODO: implement CutShift+Sorted for minimal disruption -} + if(random_slicing_cutshift_enabled() && old_host_count > 0 && + !partitions_.empty()) { + auto updated_partitions = + expand_with_cutshift(partitions_, new_nodes, 1.0f, 1.0f); + if(updated_partitions.size() == all_hosts_.size()) { + partitions_ = std::move(updated_partitions); + interval_idx_.build(partitions_); + return; + } + } -std::vector -RandomSlicingDistributor::collect_gaps_cutshift( - const std::unordered_map& reductions) { - // ponycail: placeholder for Phase 2. - // This implements the CutShift+Sorted algorithm from the thesis. - std::vector gaps; - // TODO: implement gap collection per Chapter 4 of the thesis - return gaps; + // Default: full reconfiguration on add. + init_partitions_from_hosts(); } void @@ -249,12 +321,12 @@ RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { // Recompute partitions init_partitions_from_hosts(); - // ponytail: chunks that were on removed nodes are now mapped to remaining + // chunks that were on removed nodes are now mapped to remaining // nodes automatically via the new interval table. Migration is handled by // DataMigrator (Phase 3). } -// ponytail: interval table persistence stubs — intervals are rebuilt from hosts +// interval table persistence stubs — intervals are rebuilt from hosts // on each daemon/proxy startup. This is intentional: random slicing depends on // the live host list and capacities, so stale interval files would be // incorrect. diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 1ba881612..998c67b5f 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -41,6 +41,7 @@ target_sources( PRIVATE daemon.cpp handler/srv_data.cpp ../common/rpc/rpc_util.cpp + ../common/malleability_markers.cpp util.cpp ops/metadentry.cpp ops/data.cpp diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 17b5a5989..f9883ee69 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -155,7 +156,7 @@ RocksDBBackend::RocksDBBackend(const std::string& path) { // Enable WAL if requested via environment bool use_wal = gkfs::config::rocksdb::use_write_ahead_log; - char* env_wal = std::getenv("GKFS_DAEMON_ENABLE_WAL"); + const auto* env_wal = std::getenv(gkfs::env::ENABLE_WAL); if(env_wal != nullptr) { use_wal = (std::string(env_wal) == "ON"); } diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index d78028987..a107c3f44 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -38,6 +38,7 @@ #include #include +#include #include @@ -53,6 +54,37 @@ FsData::~FsData() { // getter/setter +bool +FsData::expand_on_demand_active() const { + return expand_on_demand_active_; +} + +void +FsData::expand_on_demand_active(bool active) { + expand_on_demand_active_ = active; +} + +unsigned int +FsData::expand_on_demand_old_hosts_size() const { + return expand_on_demand_old_hosts_size_; +} + +void +FsData::expand_on_demand_old_hosts_size(unsigned int hosts_size) { + expand_on_demand_old_hosts_size_ = hosts_size; +} + +std::shared_ptr +FsData::expand_on_demand_old_distributor() const { + return expand_on_demand_old_distributor_; +} + +void +FsData::expand_on_demand_old_distributor( + std::shared_ptr distributor) { + expand_on_demand_old_distributor_ = std::move(distributor); +} + const std::shared_ptr& FsData::spdlogger() const { return spdlogger_; @@ -382,4 +414,14 @@ FsData::keep_hosts_file(bool keep) { keep_hosts_file_ = keep; } +bool +FsData::expand_mode() const { + return expand_mode_; +} + +void +FsData::expand_mode(bool expand_mode) { + expand_mode_ = expand_mode; +} + } // namespace gkfs::daemon diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 4c7153157..4c99809b1 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,8 @@ #include #include #include +#include +#include #ifdef GKFS_ENABLE_AGIOS #include @@ -92,6 +95,14 @@ using namespace std; namespace fs = std::filesystem; namespace tl = thallium; +// Forward declarations for malleability RPC handlers (from +// srv_malleability.cpp) +namespace gkfs::rpc { +struct rpc_mutate_start_in_t; +struct rpc_migrate_metadata_in_t; +struct rpc_err_out_t; +} // namespace gkfs::rpc + static condition_variable shutdown_please; // handler for shutdown signaling static mutex mtx; // mutex to wait on shutdown conditional variable static bool keep_rootdir = true; @@ -325,18 +336,14 @@ register_server_rpcs(std::shared_ptr engine) { rpc_srv_create_write_inline); // Malleability RPCs - engine->define(gkfs::malleable::rpc::tag::expand_start, - rpc_srv_expand_start); - engine->define(gkfs::malleable::rpc::tag::expand_status, - rpc_srv_expand_status); - engine->define(gkfs::malleable::rpc::tag::expand_finalize, - rpc_srv_expand_finalize); - engine->define(gkfs::malleable::rpc::tag::shrink_start, - rpc_srv_shrink_start); - engine->define(gkfs::malleable::rpc::tag::shrink_status, - rpc_srv_shrink_status); - engine->define(gkfs::malleable::rpc::tag::shrink_finalize, - rpc_srv_shrink_finalize); + engine->define(gkfs::malleable::rpc::tag::mutate_start, + rpc_srv_mutate_start); + engine->define(gkfs::malleable::rpc::tag::mutate_status, + rpc_srv_mutate_status); + engine->define(gkfs::malleable::rpc::tag::mutate_finalize, + rpc_srv_mutate_finalize); + engine->define(gkfs::malleable::rpc::tag::mutate_shutdown, + rpc_srv_mutate_shutdown); engine->define(gkfs::malleable::rpc::tag::migrate_metadata, rpc_srv_migrate_metadata); } @@ -383,16 +390,7 @@ init_rpc_server() { */ void register_client_rpcs(std::shared_ptr engine) { - // Commented out - /* - RPC_DATA->rpc_client_ids().migrate_metadata_id = - MARGO_REGISTER(mid, gkfs::malleable::rpc::tag::migrate_metadata, - rpc_migrate_metadata_in_t, rpc_err_out_t, NULL); - // this is just a write - RPC_DATA->rpc_client_ids().migrate_data_id = - MARGO_REGISTER(mid, gkfs::rpc::tag::write, rpc_write_data_in_t, - rpc_data_out_t, NULL); - */ + (void) engine; } /** @@ -483,7 +481,7 @@ init_environment() { // Check for environment variables for configuration gkfs::config::metadata::use_inline_data = - gkfs::env::get_var(gkfs::env::DAEMON_USE_INLINE_DATA, + gkfs::env::get_var(gkfs::env::USE_INLINE_DATA, gkfs::config::metadata::use_inline_data ? "ON" : "OFF") == "ON"; @@ -493,7 +491,7 @@ init_environment() { ? "ON" : "OFF") == "ON"; gkfs::config::metadata::create_check_parents = - gkfs::env::get_var(gkfs::env::DAEMON_CREATE_CHECK_PARENTS, + gkfs::env::get_var(gkfs::env::CREATE_CHECK_PARENTS, gkfs::config::metadata::create_check_parents ? "ON" : "OFF") == "ON"; @@ -503,18 +501,22 @@ init_environment() { ? "ON" : "OFF") == "ON"; gkfs::config::metadata::symlink_support = - gkfs::env::get_var(gkfs::env::DAEMON_SYMLINK_SUPPORT, + gkfs::env::get_var(gkfs::env::SYMLINK_SUPPORT, gkfs::config::metadata::symlink_support ? "ON" : "OFF") == "ON"; gkfs::config::metadata::rename_support = - gkfs::env::get_var(gkfs::env::DAEMON_RENAME_SUPPORT, + gkfs::env::get_var(gkfs::env::RENAME_SUPPORT, gkfs::config::metadata::rename_support ? "ON" : "OFF") == "ON"; GKFS_DATA->keep_hosts_file( gkfs::env::get_var(gkfs::env::KEEP_HOSTS_FILE, "OFF") == "ON"); + // Read GKFS_DAEMON_EXPAND env var for marker-based hostfile + GKFS_DATA->expand_mode( + gkfs::env::get_var(gkfs::env::DAEMON_EXPAND_MODE, "OFF") == "ON"); + GKFS_DATA->spdlogger()->info( "{}() Inline data: {} / Dirents compression: {} / Create check parents: {} / Create exist check: {} / Symlink support: {} / Rename support: {} / Keep hosts file: {}", __func__, gkfs::config::metadata::use_inline_data, @@ -620,7 +622,7 @@ init_environment() { } // setup hostfile to let clients know that a daemon is running on this host if(!GKFS_DATA->hosts_file().empty()) { - gkfs::utils::populate_hosts_file(); + gkfs::utils::populate_hosts_file(GKFS_DATA->expand_mode()); } // Init margo client @@ -642,7 +644,7 @@ init_environment() { // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var // (defaults to simple_hash for backward compatibility) gkfs::rpc::DistributionConfig config; - const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); if(env_val != nullptr && env_val[0] != '\0') { config.set_strategy(env_val); } @@ -656,6 +658,25 @@ init_environment() { if(!distributor) { throw std::runtime_error("Failed to create distributor"); } + if(config.is_random_slicing() && !GKFS_DATA->hosts_file().empty()) { + auto rs = dynamic_cast( + distributor.get()); + auto comments = gkfs::malleable::parse_rs_interval_comments( + GKFS_DATA->hosts_file()); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + if(rs && !intervals.empty() && rs->set_intervals(intervals)) { + GKFS_DATA->spdlogger()->info( + "{}() Loaded {} random-slicing intervals from hostfile", + __func__, intervals.size()); + } + } RPC_DATA->distributor(std::move(distributor)); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( @@ -768,6 +789,15 @@ shutdown_handler(int dummy) { shutdown_please.notify_all(); } +namespace gkfs::daemon { + +void +request_shutdown() { + shutdown_please.notify_all(); +} + +} // namespace gkfs::daemon + /** * @brief Initializes the daemon logging environment. * @internal diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index aebe546bf..536ae4a0a 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -103,581 +103,695 @@ using namespace std; void rpc_srv_write(const std::shared_ptr& engine, const tl::request& req, const gkfs::rpc::rpc_write_data_in_t& in) { - gkfs::rpc:: - run_rpc_handler(req, in, - [&engine, &req](const gkfs::rpc:: - rpc_write_data_in_t& - in, - gkfs::rpc::rpc_data_out_t& out) { - /* - * 1. Setup - */ - out.err = EIO; - out.io_size = 0; - - size_t bulk_size = - in.bulk_handle - .size(); - - GKFS_DATA - ->spdlogger() - ->debug("{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", - __func__, - in.path, - in.chunk_start, - in.chunk_end, - in.chunk_n, - in.total_chunk_size, - bulk_size, - in.offset); - - std::vector write_ops_vect = - gkfs::rpc::decompress_bitset( - in.wbitset); - - // Calculate the - // number of chunks - // hashing to this - // host - uint64_t - host_chunk_n = + gkfs:: + rpc:: + run_rpc_handler(req, in, + [&engine, + &req](const gkfs::rpc:: + rpc_write_data_in_t& + in, + gkfs::rpc::rpc_data_out_t& + out) { + /* + * 1. Setup + */ + out.err = + EIO; + out.io_size = 0; - for(uint64_t chnk_id_file = - in.chunk_start; - chnk_id_file <= - in.chunk_end; - chnk_id_file++) { - if(gkfs::rpc::get_bitset( - write_ops_vect, - chnk_id_file - - in.chunk_start)) { - host_chunk_n++; - } - } - - GKFS_DATA - ->spdlogger() - ->debug("{}() host_chunk_n {}", - __func__, - host_chunk_n); -#ifdef GKFS_ENABLE_AGIOS - int* data; - ABT_eventual eventual = - ABT_EVENTUAL_NULL; - - /* creating eventual - */ - ABT_eventual_create( - sizeof(int64_t), - &eventual); - - unsigned long long int request_id = - generate_unique_id(); - char* agios_path = const_cast< - char*>( - in.path.c_str()); - - // We should call - // AGIOS before - // chunking (as that - // is an internal way - // to handle the - // requests) - if(!agios_add_request( - agios_path, - AGIOS_WRITE, - in.offset, - in.total_chunk_size, - request_id, - AGIOS_SERVER_ID_IGNORE, - agios_eventual_callback, - eventual)) { - GKFS_DATA - ->spdlogger() - ->error("{}() Failed to send request to AGIOS", - __func__); - } else { - GKFS_DATA - ->spdlogger() - ->debug("{}() request {} was sent to AGIOS", - __func__, - request_id); - } - - /* Block until the - * eventual is - * signaled */ - ABT_eventual_wait( - eventual, - reinterpret_cast< - void**>( - &data)); - - unsigned long long int - result = - *data; - GKFS_DATA - ->spdlogger() - ->debug("{}() request {} was unblocked (offset = {})!", - __func__, - result, - in.offset); - - ABT_eventual_free( - &eventual); - - // Let AGIOS knows it - // can release the - // request, as it is - // completed - if(!agios_release_request( - agios_path, - AGIOS_WRITE, - in.total_chunk_size, - in.offset)) { - GKFS_DATA - ->spdlogger() - ->error("{}() Failed to release request from AGIOS", - __func__); - } -#endif + size_t bulk_size = + in.bulk_handle + .size(); - /* - * 2. Set up buffers - * for pull bulk - * transfers - */ - - // Allocate memory - // for bulk transfer - // using vector - std::vector bulk_buf( - in.total_chunk_size); - - // Expose the local - // buffer - std::vector> - segments; - segments.emplace_back( - bulk_buf.data(), - bulk_buf.size()); - - tl::bulk local_bulk = engine->expose( - segments, - tl::bulk_mode:: - write_only); - - auto const host_id = - in.host_id; - [[maybe_unused]] auto const - host_size = - in.host_size; - - // Use string - // directly chnk_ids - // used by this host - vector chnk_ids_host( - host_chunk_n); - // counter to track - // how many chunks - // have been assigned - auto chnk_id_curr = - static_cast< - uint64_t>( - 0); - // chnk sizes per - // chunk for this - // host - vector chnk_sizes( - host_chunk_n); - // how much size is - // left to assign - // chunks for writing - auto chnk_size_left_host = - in.total_chunk_size; - // temporary - // traveling pointer - char* chnk_ptr = - bulk_buf.data(); - - // temporary - // variables - auto transfer_size = - (bulk_size <= - gkfs::config:: - rpc::chunksize) - ? bulk_size - : gkfs::config:: - rpc::chunksize; - uint64_t - origin_offset; - uint64_t - local_offset; - // object for - // asynchronous disk - // IO - gkfs::data::ChunkWriteOperation - chunk_op{ - in.path, - host_chunk_n}; - - /* - * 3. Calculate chunk - * sizes that - * correspond to this - * host, transfer - * data, and start - * tasks to write to - * disk - */ - // Start to look for - // a chunk that - // hashes to this - // host with the - // first chunk in the - // buffer - for(auto chnk_id_file = - in.chunk_start; - chnk_id_file <= - in.chunk_end && - chnk_id_curr < - host_chunk_n; - chnk_id_file++) { - // Continue if - // chunk does not - // hash to this - // host - - if(!(gkfs::rpc::get_bitset( - write_ops_vect, - chnk_id_file - - in.chunk_start))) { GKFS_DATA ->spdlogger() - ->trace("{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", + ->debug("{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", __func__, - chnk_id_file, - host_id, - chnk_id_curr); - continue; - } - - if(GKFS_DATA - ->enable_chunkstats()) { - GKFS_DATA - ->stats() - ->add_write( in.path, - chnk_id_file); - } - - chnk_ids_host[chnk_id_curr] = - chnk_id_file; // save this id to host chunk list - // offset case. - // Only relevant - // in the first - // iteration of - // the loop and - // if the chunk - // hashes to this - // host - if(chnk_id_file == - in.chunk_start && - in.offset > - 0) { - // if only 1 - // destination - // and 1 - // chunk - // (small - // write) the - // transfer_size - // == - // bulk_size - size_t offset_transfer_size = + in.chunk_start, + in.chunk_end, + in.chunk_n, + in.total_chunk_size, + bulk_size, + in.offset); + + std::vector write_ops_vect = + gkfs::rpc::decompress_bitset( + in.wbitset); + + // Calculate + // the number + // of chunks + // hashing to + // this host + uint64_t host_chunk_n = 0; - if(in.offset + - bulk_size <= - gkfs::config:: - rpc::chunksize) - offset_transfer_size = - bulk_size; - else - offset_transfer_size = static_cast< - size_t>( - gkfs::config:: - rpc::chunksize - - in.offset); - - // PULL - // transfer - try { - size_t current_buf_offset = - chnk_ptr - - bulk_buf.data(); - - local_bulk( - current_buf_offset, - offset_transfer_size) - << in.bulk_handle - .on(req.get_endpoint())( - 0, - offset_transfer_size); - - } catch(const std::exception& - e) { + for(uint64_t chnk_id_file = + in.chunk_start; + chnk_id_file <= + in.chunk_end; + chnk_id_file++) { + if(gkfs::rpc::get_bitset( + write_ops_vect, + chnk_id_file - + in.chunk_start)) { + host_chunk_n++; + } + } + + GKFS_DATA + ->spdlogger() + ->debug("{}() host_chunk_n {}", + __func__, + host_chunk_n); + +#ifdef GKFS_ENABLE_AGIOS + int* data; + ABT_eventual eventual = + ABT_EVENTUAL_NULL; + + /* creating + * eventual + */ + ABT_eventual_create( + sizeof(int64_t), + &eventual); + + unsigned long long int request_id = + generate_unique_id(); + char* agios_path = const_cast< + char*>( + in.path.c_str()); + + // We should + // call AGIOS + // before + // chunking + // (as that + // is an + // internal + // way to + // handle the + // requests) + if(!agios_add_request( + agios_path, + AGIOS_WRITE, + in.offset, + in.total_chunk_size, + request_id, + AGIOS_SERVER_ID_IGNORE, + agios_eventual_callback, + eventual)) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to send request to AGIOS", + __func__); + } else { GKFS_DATA ->spdlogger() - ->error("{}() Failed to pull data from client for chunk {} (startchunk {}; endchunk {})", + ->debug("{}() request {} was sent to AGIOS", __func__, - chnk_id_file, - in.chunk_start, - in.chunk_end - - 1); - out.err = - EBUSY; - return; + request_id); } - chnk_sizes[chnk_id_curr] = - offset_transfer_size; - chnk_ptr += - offset_transfer_size; - chnk_size_left_host -= - offset_transfer_size; - } else { - local_offset = - in.total_chunk_size - - chnk_size_left_host; - // origin - // offset of - // a chunk is - // dependent - // on a given - // offset in - // a write - // operation - if(in.offset > - 0) - origin_offset = - (gkfs::config:: - rpc::chunksize - - in.offset) + - ((chnk_id_file - - in.chunk_start) - - 1) * gkfs::config::rpc:: - chunksize; - else - origin_offset = - (chnk_id_file - - in.chunk_start) * - gkfs::config:: - rpc::chunksize; - // last chunk - // might have - // different - // transfer_size - if(chnk_id_curr == - in.chunk_n - - 1) - transfer_size = - chnk_size_left_host; + /* Block + * until the + * eventual + * is + * signaled + */ + ABT_eventual_wait( + eventual, + reinterpret_cast< + void**>( + &data)); + + unsigned long long int + result = + *data; GKFS_DATA ->spdlogger() - ->trace("{}() BULK_TRANSFER_PULL hostid {} file {} chnkid {} total_Csize {} Csize_left {} origin offset {} local offset {} transfersize {}", + ->debug("{}() request {} was unblocked (offset = {})!", __func__, - host_id, - in.path, - chnk_id_file, - in.total_chunk_size, - chnk_size_left_host, - origin_offset, - local_offset, - transfer_size); - - // RDMA PULL - try { - // margo: - // PULL, - // in.bulk, - // origin_offset, - // bulk_handle, - // local_offset, - // transfer_size - local_bulk( - local_offset, - transfer_size) - << in.bulk_handle - .on(req.get_endpoint())( - origin_offset, - transfer_size); - } catch(const std::exception& - e) { + result, + in.offset); + + ABT_eventual_free( + &eventual); + + // Let AGIOS + // knows it + // can + // release + // the + // request, + // as it is + // completed + if(!agios_release_request( + agios_path, + AGIOS_WRITE, + in.total_chunk_size, + in.offset)) { GKFS_DATA ->spdlogger() - ->error("{}() Failed to pull data from client. file {} chunk {} (startchunk {}; endchunk {}). Error: {}", - __func__, - in.path, - chnk_id_file, - in.chunk_start, - (in.chunk_end - - 1), - e.what()); - out.err = - EBUSY; - return; + ->error("{}() Failed to release request from AGIOS", + __func__); } +#endif - chnk_sizes[chnk_id_curr] = - transfer_size; - chnk_ptr += - transfer_size; - chnk_size_left_host -= - transfer_size; - } - try { - // start - // tasklet - // for + /* + * 2. Set up + * buffers + * for pull + * bulk + * transfers + */ + + // Allocate + // memory for + // bulk + // transfer + // using + // vector + std::vector bulk_buf( + in.total_chunk_size); + + // Expose the + // local + // buffer + std::vector> + segments; + segments.emplace_back( + bulk_buf.data(), + bulk_buf.size()); + + tl::bulk local_bulk = engine->expose( + segments, + tl::bulk_mode:: + write_only); + + auto const host_id = + in.host_id; + [[maybe_unused]] auto const + host_size = + in.host_size; + + // Use string + // directly + // chnk_ids + // used by + // this host + vector chnk_ids_host( + host_chunk_n); + // counter to + // track how + // many + // chunks + // have been + // assigned + auto chnk_id_curr = static_cast< + uint64_t>( + 0); + // chnk sizes + // per chunk + // for this + // host + vector chnk_sizes( + host_chunk_n); + // how much + // size is + // left to + // assign + // chunks for // writing - // chunk + auto chnk_size_left_host = + in.total_chunk_size; + // temporary + // traveling + // pointer + char* chnk_ptr = + bulk_buf.data(); + + // temporary + // variables + auto transfer_size = + (bulk_size <= + gkfs::config:: + rpc::chunksize) + ? bulk_size + : gkfs::config:: + rpc::chunksize; + uint64_t + origin_offset; + uint64_t + local_offset; + // object for + // asynchronous + // disk IO + gkfs::data::ChunkWriteOperation chunk_op{ + in.path, + host_chunk_n}; - if(chnk_id_file == - in.chunk_start && - in.offset > - 0) { - chunk_op.write_nonblock( - chnk_id_curr, - chnk_ids_host - [chnk_id_curr], - bulk_buf.data(), - chnk_sizes - [chnk_id_curr], - in.offset); - } else { - size_t computed_offset = - in.total_chunk_size - - (chnk_size_left_host + - chnk_sizes - [chnk_id_curr]); - // chnk_size_left_host - // was - // just - // decremented. - // so - // previous - // left - // was - // chnk_size_left_host - // + - // size. + /* + * 3. + * Calculate + * chunk + * sizes that + * correspond + * to this + * host, + * transfer + * data, and + * start + * tasks to + * write to + * disk + */ + // Start to + // look for + // a chunk + // that + // hashes to + // this host + // with the + // first + // chunk in + // the buffer + for(auto chnk_id_file = + in.chunk_start; + chnk_id_file <= + in.chunk_end && + chnk_id_curr < + host_chunk_n; + chnk_id_file++) { + // Continue + // if + // chunk + // does + // not + // hash + // to + // this + // host + + if(!(gkfs::rpc::get_bitset( + write_ops_vect, + chnk_id_file - + in.chunk_start))) { + GKFS_DATA + ->spdlogger() + ->trace("{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", + __func__, + chnk_id_file, + host_id, + chnk_id_curr); + continue; + } + + if(GKFS_DATA + ->enable_chunkstats()) { + GKFS_DATA + ->stats() + ->add_write( + in.path, + chnk_id_file); + } + + chnk_ids_host[chnk_id_curr] = + chnk_id_file; // save this id to host chunk list // offset - // = - // total - // - - // (prev_left) - // = - // total - // - - // (left - // + - // size) - // This - // matches - // local_offset. - - chunk_op.write_nonblock( - chnk_id_curr, - chnk_ids_host - [chnk_id_curr], - bulk_buf.data() + - computed_offset, - chnk_sizes - [chnk_id_curr], - 0); + // case. + // Only + // relevant + // in the + // first + // iteration + // of the + // loop + // and if + // the + // chunk + // hashes + // to + // this + // host + if(chnk_id_file == + in.chunk_start && + in.offset > + 0) { + // if + // only + // 1 + // destination + // and + // 1 + // chunk + // (small + // write) + // the + // transfer_size + // == + // bulk_size + size_t offset_transfer_size = + 0; + if(in.offset + + bulk_size <= + gkfs::config:: + rpc::chunksize) + offset_transfer_size = + bulk_size; + else + offset_transfer_size = static_cast< + size_t>( + gkfs::config:: + rpc::chunksize - + in.offset); + + // PULL + // transfer + try { + size_t current_buf_offset = + chnk_ptr - + bulk_buf.data(); + + local_bulk( + current_buf_offset, + offset_transfer_size) + << in.bulk_handle + .on(req.get_endpoint())( + 0, + offset_transfer_size); + + } catch(const std::exception& + e) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to pull data from client for chunk {} (startchunk {}; endchunk {})", + __func__, + chnk_id_file, + in.chunk_start, + in.chunk_end - + 1); + out.err = + EBUSY; + return; + } + + chnk_sizes[chnk_id_curr] = + offset_transfer_size; + chnk_ptr += + offset_transfer_size; + chnk_size_left_host -= + offset_transfer_size; + } else { + local_offset = + in.total_chunk_size - + chnk_size_left_host; + // origin + // offset + // of + // a + // chunk + // is + // dependent + // on + // a + // given + // offset + // in + // a + // write + // operation + if(in.offset > + 0) + origin_offset = + (gkfs::config:: + rpc::chunksize - + in.offset) + + ((chnk_id_file - + in.chunk_start) - + 1) * gkfs::config::rpc:: + chunksize; + else + origin_offset = + (chnk_id_file - + in.chunk_start) * + gkfs::config:: + rpc::chunksize; + // last + // chunk + // might + // have + // different + // transfer_size + if(chnk_id_curr == + in.chunk_n - + 1) + transfer_size = + chnk_size_left_host; + GKFS_DATA + ->spdlogger() + ->trace("{}() BULK_TRANSFER_PULL hostid {} file {} chnkid {} total_Csize {} Csize_left {} origin offset {} local offset {} transfersize {}", + __func__, + host_id, + in.path, + chnk_id_file, + in.total_chunk_size, + chnk_size_left_host, + origin_offset, + local_offset, + transfer_size); + + // RDMA + // PULL + try { + // margo: + // PULL, + // in.bulk, + // origin_offset, + // bulk_handle, + // local_offset, + // transfer_size + local_bulk( + local_offset, + transfer_size) + << in.bulk_handle + .on(req.get_endpoint())( + origin_offset, + transfer_size); + } catch(const std::exception& + e) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to pull data from client. file {} chunk {} (startchunk {}; endchunk {}). Error: {}", + __func__, + in.path, + chnk_id_file, + in.chunk_start, + (in.chunk_end - + 1), + e.what()); + out.err = + EBUSY; + return; + } + + chnk_sizes[chnk_id_curr] = + transfer_size; + chnk_ptr += + transfer_size; + chnk_size_left_host -= + transfer_size; + } + const bool partial_chunk_write = + (chnk_id_file == + in.chunk_start && + in.offset > + 0) || + chnk_sizes[chnk_id_curr] < + gkfs::config:: + rpc::chunksize; + if(partial_chunk_write) { + auto mat_err = gkfs::data::expand_on_demand_materialize_for_partial_write( + in.path, + chnk_ids_host + [chnk_id_curr]); + if(mat_err != + 0) { + GKFS_DATA + ->spdlogger() + ->error("{}() expand-on-demand partial-write materialization failed for '{}' chunk '{}': err '{}'", + __func__, + in.path, + chnk_ids_host + [chnk_id_curr], + mat_err); + out.err = + mat_err; + return; + } + } + + try { + // start + // tasklet + // for + // writing + // chunk + + if(chnk_id_file == + in.chunk_start && + in.offset > + 0) { + chunk_op.write_nonblock( + chnk_id_curr, + chnk_ids_host + [chnk_id_curr], + bulk_buf.data(), + chnk_sizes + [chnk_id_curr], + in.offset); + } else { + size_t computed_offset = + in.total_chunk_size - + (chnk_size_left_host + + chnk_sizes + [chnk_id_curr]); + // chnk_size_left_host + // was + // just + // decremented. + // so + // previous + // left + // was + // chnk_size_left_host + // + + // size. + // offset + // = + // total + // - + // (prev_left) + // = + // total + // - + // (left + // + + // size) + // This + // matches + // local_offset. + + chunk_op.write_nonblock( + chnk_id_curr, + chnk_ids_host + [chnk_id_curr], + bulk_buf.data() + + computed_offset, + chnk_sizes + [chnk_id_curr], + 0); + } + + } catch(const gkfs::data::ChunkWriteOpException& + e) { + // This + // exception + // is + // caused + // by + // setup + // of + // Argobots + // variables. + // If + // this + // fails, + // something + // is + // really + // wrong + GKFS_DATA + ->spdlogger() + ->error("{}() while write_nonblock err '{}'", + __func__, + e.what()); + out.err = + EIO; + return; + } + // next + // chunk + chnk_id_curr++; } - - } catch(const gkfs::data::ChunkWriteOpException& - e) { - // This - // exception - // is caused - // by setup - // of - // Argobots - // variables. - // If this - // fails, - // something - // is really - // wrong - GKFS_DATA - ->spdlogger() - ->error("{}() while write_nonblock err '{}'", - __func__, - e.what()); + // Sanity + // check that + // all chunks + // where + // detected + // in + // previous + // loop + // TODO don't + // proceed if + // that + // happens. + if(chnk_size_left_host != + 0) + GKFS_DATA + ->spdlogger() + ->warn("{}() Not all chunks were detected!!! Size left {}", + __func__, + chnk_size_left_host); + /* + * 4. Read + * task + * results + * and + * accumulate + * in + * out.io_size + */ + auto write_result = + chunk_op.wait_for_tasks(); out.err = - EIO; - return; - } - // next chunk - chnk_id_curr++; - } - // Sanity check that - // all chunks where - // detected in - // previous loop - // TODO don't proceed - // if that happens. - if(chnk_size_left_host != - 0) - GKFS_DATA - ->spdlogger() - ->warn("{}() Not all chunks were detected!!! Size left {}", - __func__, - chnk_size_left_host); - /* - * 4. Read task - * results and - * accumulate in - * out.io_size - */ - auto write_result = - chunk_op.wait_for_tasks(); - out.err = - write_result - .first; - out.io_size = - write_result - .second; - - // Sanity check to - // see if all data - // has been written - if(in.total_chunk_size != - out.io_size) { - GKFS_DATA - ->spdlogger() - ->warn("{}() total chunk size {} and out.io_size {} mismatch!", - __func__, - in.total_chunk_size, - out.io_size); - } - - if(GKFS_DATA - ->enable_stats()) { - GKFS_DATA - ->stats() - ->add_value_size( - gkfs::utils::Stats:: - SizeOp::write_size, - bulk_size); - } - }); + write_result + .first; + out.io_size = + write_result + .second; + + // Sanity + // check to + // see if all + // data has + // been + // written + if(in.total_chunk_size != + out.io_size) { + GKFS_DATA + ->spdlogger() + ->warn("{}() total chunk size {} and out.io_size {} mismatch!", + __func__, + in.total_chunk_size, + out.io_size); + } + + if(GKFS_DATA + ->enable_stats()) { + GKFS_DATA + ->stats() + ->add_value_size( + gkfs::utils::Stats:: + SizeOp::write_size, + bulk_size); + } + }); } /** diff --git a/src/daemon/handler/srv_malleability.cpp b/src/daemon/handler/srv_malleability.cpp index 5e86518dd..02bde6764 100644 --- a/src/daemon/handler/srv_malleability.cpp +++ b/src/daemon/handler/srv_malleability.cpp @@ -43,6 +43,9 @@ #include #include +#include +#include + extern "C" { #include } @@ -52,21 +55,21 @@ using namespace std; // namespace { void -rpc_srv_expand_start(const tl::request& req, - const gkfs::rpc::rpc_expand_start_in_t& in) { +rpc_srv_mutate_start(const tl::request& req, + const gkfs::rpc::rpc_mutate_start_in_t& in) { gkfs::rpc::rpc_err_out_t out; GKFS_DATA->spdlogger()->debug( - "{}() Got RPC with old conf '{}' new conf '{}'", __func__, - in.old_server_conf, in.new_server_conf); + "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", + __func__, in.old_server_conf, in.new_server_conf, + in.new_hosts_file); try { - // if maintenance mode is already set, error is thrown -- not allowed GKFS_DATA->maintenance_mode(true); - GKFS_DATA->malleable_manager()->expand_start( + GKFS_DATA->malleable_manager()->mutate_start( in.old_server_conf, in.new_server_conf, in.new_hosts_file); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to start expansion: '{}' ", + GKFS_DATA->spdlogger()->error("{}() Failed to start mutate: '{}' ", __func__, e.what()); GKFS_DATA->maintenance_mode(false); out.err = -1; @@ -78,15 +81,14 @@ rpc_srv_expand_start(const tl::request& req, } void -rpc_srv_expand_status(const tl::request& req) { +rpc_srv_mutate_status(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); try { - // return 1 if redistribution is running, 0 otherwise. out.err = GKFS_DATA->redist_running() ? 1 : 0; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( - "{}() Failed to check status for expansion: '{}'", __func__, + "{}() Failed to check status for mutate: '{}'", __func__, e.what()); out.err = -1; } @@ -96,14 +98,15 @@ rpc_srv_expand_status(const tl::request& req) { } void -rpc_srv_expand_finalize(const tl::request& req) { +rpc_srv_mutate_finalize(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); try { GKFS_DATA->maintenance_mode(false); + GKFS_DATA->keep_hosts_file(true); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to finalize expansion: '{}'", + GKFS_DATA->spdlogger()->error("{}() Failed to finalize mutate: '{}'", __func__, e.what()); out.err = -1; } @@ -114,94 +117,44 @@ rpc_srv_expand_finalize(const tl::request& req) { } void -rpc_srv_shrink_start(const tl::request& req, - const gkfs::rpc::rpc_shrink_start_in_t& in) { +rpc_srv_mutate_shutdown(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; + GKFS_DATA->spdlogger()->info( + "{}() Got graceful shutdown RPC after malleability finalize", + __func__); - GKFS_DATA->spdlogger()->debug( - "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", - __func__, in.old_server_conf, in.new_server_conf, - in.new_hosts_file); try { - GKFS_DATA->maintenance_mode(true); - GKFS_DATA->malleable_manager()->shrink_start( - in.old_server_conf, in.new_server_conf, in.new_hosts_file); + GKFS_DATA->maintenance_mode(false); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to start shrink: '{}' ", + GKFS_DATA->spdlogger()->error("{}() Failed to prepare shutdown: '{}'", __func__, e.what()); - GKFS_DATA->maintenance_mode(false); out.err = -1; } - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); gkfs::utils::safe_respond(req, out); -} -void -rpc_srv_shrink_status(const tl::request& req) { - gkfs::rpc::rpc_err_out_t out; - GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); - try { - out.err = GKFS_DATA->redist_running() ? 1 : 0; - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to check status for shrink: '{}'", __func__, - e.what()); - out.err = -1; + if(out.err == 0) { + std::thread([]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + gkfs::daemon::request_shutdown(); + }).detach(); } - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); - gkfs::utils::safe_respond(req, out); -} - -void -rpc_srv_shrink_finalize(const tl::request& req) { - gkfs::rpc::rpc_err_out_t out; - GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); - try { - GKFS_DATA->maintenance_mode(false); - out.err = 0; - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to finalize shrink: '{}'", - __func__, e.what()); - out.err = -1; - } - - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); - gkfs::utils::safe_respond(req, out); } void rpc_srv_migrate_metadata(const tl::request& req, const gkfs::rpc::rpc_migrate_metadata_in_t& in) { - gkfs::rpc::rpc_err_out_t out{}; - - GKFS_DATA->spdlogger()->debug("{}() Got RPC with key '{}' value '{}'", - __func__, in.key, in.value); + gkfs::rpc::rpc_err_out_t out; try { - // create metadentry GKFS_DATA->mdb()->put(in.key, in.value); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to create KV entry: '{}'", + GKFS_DATA->spdlogger()->error("{}() Failed to write metadata: '{}'", __func__, e.what()); out.err = -1; } - - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); gkfs::utils::safe_respond(req, out); } // } // namespace - -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_start) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_status) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_finalize) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_start) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_status) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_finalize) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_migrate_metadata) diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index 12e1d7a83..f59d79d80 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -42,13 +42,21 @@ #include #include +#include #include +#include +#include +#include #include #include #include #include #include +#include +#include +#include +#include extern "C" { #include @@ -62,6 +70,113 @@ namespace fs = std::filesystem; namespace gkfs::malleable { +static string +host_key(pair host) { + if(auto idx = host.first.rfind("#"); idx != string::npos) { + host.first.erase(idx); + } + return host.first + "\n" + host.second; +} + +static bool +env_truthy(const char* value) { + if(value == nullptr || value[0] == '\0') { + return false; + } + string v(value); + transform(v.begin(), v.end(), v.begin(), + [](unsigned char c) { return static_cast(tolower(c)); }); + return v == "1" || v == "on" || v == "true" || v == "yes"; +} + +static vector +to_interval_comments(const vector& intervals) { + vector comments; + comments.reserve(intervals.size()); + for(const auto& interval : intervals) { + comments.push_back({static_cast(interval.host_id), + static_cast(interval.start), + static_cast(interval.end)}); + } + return comments; +} + +static vector +make_equal_rs_partitions_for_before_hosts( + const vector>& before_hosts, + const vector>& after_hosts) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + vector partitions; + partitions.reserve(before_hosts.size()); + const auto n = static_cast(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + const auto it = after_ids.find(host_key(before_hosts[old_id])); + if(it == after_ids.end()) { + continue; + } + const auto start = static_cast(old_id) / n; + const auto end = static_cast(old_id + 1) / n; + gkfs::rpc::Partition partition; + partition.host_id = it->second; + partition.total_capacity = end - start; + partition.intervals.push_back({start, end, it->second}); + partitions.push_back(partition); + } + return partitions; +} + +static vector +remap_existing_rs_comments( + const vector& comments, + const vector>& before_hosts, + const vector>& after_hosts) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + if(comment.host_id >= before_hosts.size()) { + return {}; + } + const auto it = after_ids.find(host_key(before_hosts[comment.host_id])); + if(it == after_ids.end()) { + return {}; + } + intervals.push_back({static_cast(comment.start), + static_cast(comment.end), it->second}); + } + return intervals; +} + +static vector +intervals_to_partitions(const vector& intervals) { + map grouped; + for(const auto& interval : intervals) { + auto& partition = grouped[interval.host_id]; + partition.host_id = interval.host_id; + partition.intervals.push_back(interval); + } + + vector partitions; + partitions.reserve(grouped.size()); + for(auto& [_, partition] : grouped) { + sort(partition.intervals.begin(), partition.intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + partition.total_capacity = partition.coverage(); + partitions.push_back(std::move(partition)); + } + return partitions; +} + // TODO The following three functions are almost identical to the proxy code // They should be moved to a common and shared between the proxy and the daemon vector> @@ -89,6 +204,9 @@ MalleableManager::load_hostfile(const std::string& path) { // It is therefore skipped if(line[0] == '#') continue; + // Skip marker lines (for backward compatibility) + if(gkfs::malleable::is_marker_line(line)) + continue; if(!regex_match(line, match, line_re)) { GKFS_DATA->spdlogger()->error( "{}() Unrecognized line format: [path: '{}', line: '{}']", @@ -148,6 +266,8 @@ MalleableManager::connect_to_hosts( bool local_host_found = false; RPC_DATA->hosts_size(hosts.size()); + RPC_DATA->rpc_endpoints().clear(); + RPC_DATA->rpc_endpoints_str().clear(); vector host_ids(hosts.size()); // populate vector with [0, ..., host_size - 1] ::iota(::begin(host_ids), ::end(host_ids), 0); @@ -170,6 +290,7 @@ MalleableManager::connect_to_hosts( try { auto svr_addr = RPC_DATA->client_rpc_engine()->lookup(uri); RPC_DATA->rpc_endpoints().insert(make_pair(id, svr_addr)); + RPC_DATA->rpc_endpoints_str().insert(make_pair(id, uri)); break; } catch(const std::exception& e) { // still not working after 5 tries. @@ -234,22 +355,40 @@ MalleableManager::redistribute_metadata() { if(key == "/") { continue; } - auto dest_id = RPC_DATA->distributor()->locate_file_metadata(key, 0); + const auto dest_id = + RPC_DATA->distributor()->locate_file_metadata(key, 0); + const auto local_id = RPC_DATA->local_host_id(); GKFS_DATA->spdlogger()->trace( - "{}() Migration: key {} and value {}. From host {} to host {}", - __func__, key, value, RPC_DATA->local_host_id(), dest_id); - if(dest_id == RPC_DATA->local_host_id()) { - GKFS_DATA->spdlogger()->trace("{}() SKIP", __func__); - continue; + "{}() Migration: key {} and value {}. From host {} primary target {}", + __func__, key, value, local_id, dest_id); + + bool copied_to_primary = (dest_id == local_id); + for(const auto& [target_id, _] : RPC_DATA->rpc_endpoints()) { + if(target_id == local_id) { + continue; + } + + auto err = gkfs::malleable::rpc::forward_metadata(key, value, + target_id); + if(err != 0) { + GKFS_DATA->spdlogger()->error( + "{}() Failed to copy metadata for key '{}' to host '{}'", + __func__, key, target_id); + migration_err++; + } else if(target_id == dest_id) { + copied_to_primary = true; + } } - auto err = gkfs::malleable::rpc::forward_metadata(key, value, dest_id); - if(err != 0) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to migrate metadata for key '{}'", __func__, - key); - migration_err++; + + // Keep the source metadata. This avoids metadata loss if source and + // target overlap in single-machine tests and keeps mutate conservative: + // after finalize, clients route to the new primary, while stale source + // copies are harmless. + if(!copied_to_primary) { + GKFS_DATA->spdlogger()->warn( + "{}() Primary metadata target '{}' for key '{}' was not copied", + __func__, dest_id, key); } - GKFS_DATA->mdb()->remove(key); count++; if(percent_interval > 0 && count % percent_interval == 0) { GKFS_DATA->spdlogger()->info( @@ -262,91 +401,19 @@ MalleableManager::redistribute_metadata() { return migration_err; } -void -MalleableManager::redistribute_data() { - GKFS_DATA->spdlogger()->info("{}() Starting data redistribution...", - __func__); - - auto chunk_dir = fs::path(GKFS_DATA->storage()->get_chunk_directory()); - auto dir_iterator = GKFS_DATA->storage()->get_all_chunk_files(); - - // TODO this can be parallelized, e.g., async chunk I/O - for(const auto& entry : dir_iterator) { - if(!entry.is_regular_file()) { - continue; - } - // path under chunkdir as placed in the rootdir - auto rel_chunk_dir = fs::relative(entry, chunk_dir); - // chunk id from this entry used for determining destination - uint64_t chunk_id = stoul(rel_chunk_dir.filename().string()); - // mountdir gekkofs path used for determining destination - auto gkfs_path = rel_chunk_dir.parent_path().string(); - ::replace(gkfs_path.begin(), gkfs_path.end(), ':', '/'); - gkfs_path = "/" + gkfs_path; - auto dest_id = - RPC_DATA->distributor()->locate_data(gkfs_path, chunk_id, 0); - GKFS_DATA->spdlogger()->trace( - "{}() Migrating chunkfile: {} for gkfs file {} chnkid {} destid {}", - __func__, rel_chunk_dir.string(), gkfs_path, chunk_id, dest_id); - if(dest_id == RPC_DATA->local_host_id()) { - GKFS_DATA->spdlogger()->trace("{}() SKIPPERS", __func__); - continue; - } - auto fd = open(entry.path().c_str(), O_RDONLY); - if(fd < 0) { - GKFS_DATA->spdlogger()->error("{}() Failed to open chunkfile: {}", - __func__, entry.path().c_str()); - continue; - } - std::vector buf(entry.file_size()); - auto bytes_read = read(fd, buf.data(), entry.file_size()); - if(bytes_read < 0) { - GKFS_DATA->spdlogger()->error("{}() Failed to read chunkfile: {}", - __func__, entry.path().c_str()); - continue; - } - auto err = gkfs::malleable::rpc::forward_data( - gkfs_path, buf.data(), bytes_read, chunk_id, dest_id); - if(err != 0) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to migrate data for chunkfile: {}", __func__, - entry.path().c_str()); - } - close(fd); - GKFS_DATA->spdlogger()->trace( - "{}() Data migration completed for chunkfile: {}. Removing ...", - __func__, entry.path().c_str()); - // remove file after migration - auto entry_dir = entry.path().parent_path(); - try { - fs::remove(entry); - if(fs::is_empty(entry_dir)) { - fs::remove(entry_dir); - } - } catch(const fs::filesystem_error& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to remove chunkfile: {}", - __func__, entry.path().c_str()); - } - GKFS_DATA->spdlogger()->trace("{}() Done for chunkfile: {}", __func__, - entry.path().c_str()); - } - - GKFS_DATA->spdlogger()->info("{}() Data redistribution completed.", - __func__); -} - int MalleableManager::do_migration(gkfs::rpc::MigrationJob& job) { // Read chunk data from local storage std::string chunk_path; try { - // Build absolute chunk file path from gkfs path and chunk id std::string internal_path = job.path; - // Convert slashes to colons for chunk directory naming std::replace(internal_path.begin(), internal_path.end(), '/', ':'); - chunk_path = fmt::format("{}/chunks/{}/{}", + if(!internal_path.empty() && internal_path.front() == ':') { + internal_path.erase(0, 1); + } + chunk_path = fmt::format("{}/{}/{}", GKFS_DATA->storage()->get_chunk_directory(), - job.chunk_id, internal_path); + internal_path, job.chunk_id); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() Failed to build chunk path for {} chnk {}: {}", __func__, @@ -393,13 +460,12 @@ MalleableManager::do_migration(gkfs::rpc::MigrationJob& job) { GKFS_DATA->spdlogger()->trace( "{}() Migrated: {} chnk {} {} bytes to host {}", __func__, job.path, job.chunk_id, bytes_read, job.target_node); - // Remove local chunk file after successful migration - auto chunk_path = fmt::format( - "{}/chunks/{}/{}", GKFS_DATA->storage()->get_chunk_directory(), - job.chunk_id, std::string(job.path.begin(), job.path.end())); - std::replace(chunk_path.begin(), chunk_path.end(), '/', ':'); - std::error_code ec; - fs::remove(chunk_path, ec); + // Keep the source chunk after a successful copy. This preserves data + // in single-machine/test deployments where several daemons + // intentionally share the same backend root and source/target chunk + // paths alias. Reads after finalize use the new distributor and + // therefore use the migrated target copy; the old copy is harmless + // stale storage. } else { GKFS_DATA->spdlogger()->error( "{}() Failed to migrate data for {} chnk {} to host {}: err {}", @@ -409,19 +475,11 @@ MalleableManager::do_migration(gkfs::rpc::MigrationJob& job) { return err == 0 ? 0 : -1; } -void -MalleableManager::redistribute_data_legacy() { - GKFS_DATA->spdlogger()->info("{}() Starting legacy data redistribution...", - __func__); - // delegate to the original implementation - redistribute_data(); -} - void MalleableManager::redistribute_data_v2() { - GKFS_DATA->spdlogger()->info("{}() Starting v2 data redistribution " - "(DataMigrationExecutor pipeline)...", - __func__); + GKFS_DATA->spdlogger()->info( + "{}() Starting data redistribution from local chunk inventory...", + __func__); auto distributor = RPC_DATA->distributor(); if(!distributor) { @@ -430,56 +488,58 @@ MalleableManager::redistribute_data_v2() { return; } - // Get old partitions: cast to RandomSlicingDistributor since Distributor - // base class doesn't expose partitions directly - std::vector old_partitions; - std::vector current_partitions; + std::vector jobs; + const auto local_host = RPC_DATA->local_host_id(); + const auto chunk_root = + fs::path(GKFS_DATA->storage()->get_chunk_directory()); - auto* rs_dist = dynamic_cast( - distributor.get()); - if(rs_dist) { - old_partitions = {}; // Will be built from old_hosts_size - current_partitions = rs_dist->get_partitions_copy(); - } else { - // Fallback: build old partitions from old_hosts_size - // Use the same approach as init_partitions_from_hosts - GKFS_DATA->spdlogger()->warn( - "{}() Cannot cast to RandomSlicingDistributor, using simplified approach", - __func__); - current_partitions = {}; - old_partitions = {}; + if(!fs::exists(chunk_root)) { + GKFS_DATA->spdlogger()->info( + "{}() Chunk directory '{}' does not exist.", __func__, + chunk_root.native()); + return; } - // If we can't get old partitions dynamically, build them from - // old_hosts_size by creating a temporary distributor with the old config - if(rs_dist && old_hosts_size_ > 0) { - // Build old partitions using a temporary RandomSlicingDistributor - gkfs::rpc::RandomSlicingDistributor old_dist(rs_dist->localhost(), - old_hosts_size_); - old_dist.reconfigure(); - old_partitions = old_dist.get_partitions_copy(); - } + for(const auto& entry : fs::recursive_directory_iterator(chunk_root)) { + if(!entry.is_regular_file()) { + continue; + } + + const auto chunk_name = entry.path().filename().string(); + if(chunk_name.empty() || + !std::all_of(chunk_name.begin(), chunk_name.end(), ::isdigit)) { + continue; + } + + auto chunk_id = + static_cast(std::stoul(chunk_name)); + auto rel_parent = + fs::relative(entry.path().parent_path(), chunk_root).string(); + std::replace(rel_parent.begin(), rel_parent.end(), ':', '/'); + auto file_path = fmt::format("/{}", rel_parent); - // 1. Compute migration jobs using DataMigrator - gkfs::rpc::DataMigrator migrator; - auto migration_stats = migrator.get_migration_stats( - old_partitions, current_partitions, 256); + auto target = distributor->locate_data(file_path, chunk_id, 0); + if(target == local_host) { + continue; + } + + jobs.push_back(gkfs::rpc::MigrationJob{ + file_path, chunk_id, static_cast(local_host), + target}); + } - if(migration_stats.migrating_chunks == 0) { - GKFS_DATA->spdlogger()->info("{}() No data migration needed.", + if(jobs.empty()) { + GKFS_DATA->spdlogger()->info("{}() No local chunks need migration.", __func__); return; } - GKFS_DATA->spdlogger()->info( - "{}() DataMigrator computed {} migration jobs ({} chunks)", - __func__, migration_stats.jobs.size(), - migration_stats.migrating_chunks); + GKFS_DATA->spdlogger()->info("{}() Computed {} local chunk migration jobs", + __func__, jobs.size()); - // 2. Execute jobs using DataMigrationExecutor with our handler gkfs::rpc::DataMigrationExecutor executor; auto status = executor.execute( - migration_stats.jobs, + jobs, // Progress callback [](size_t done, size_t total) { // Could log progress here if needed @@ -516,153 +576,83 @@ MalleableManager::redistribute_data_v2() { } } -int -MalleableManager::execute_migrations( - const std::vector& jobs) { - GKFS_DATA->spdlogger()->info("{}() Executing {} migration jobs...", - __func__, jobs.size()); - - int success_count = 0; - int fail_count = 0; - size_t total_bytes = 0; - - for(const auto& job : jobs) { - // Skip if source and target are the same - if(job.source_node == job.target_node) { - GKFS_DATA->spdlogger()->debug( - "{}() Skipping job (same node): {} chnk {} src {} tgt {}", - __func__, job.path, job.chunk_id, job.source_node, - job.target_node); - continue; - } - - // Read chunk data from local storage - std::string chunk_path; - try { - // Build absolute chunk file path from gkfs path and chunk id - std::string internal_path = job.path; - // Convert slashes to colons for chunk directory naming - std::replace(internal_path.begin(), internal_path.end(), '/', ':'); - chunk_path = - fmt::format("{}/chunks/{}/{}", - GKFS_DATA->storage()->get_chunk_directory(), - job.chunk_id, internal_path); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to build chunk path for {} chnk {}: {}", - __func__, job.path, job.chunk_id, e.what()); - ++fail_count; - continue; - } - - // Open and read the chunk file - int fd = open(chunk_path.c_str(), O_RDONLY); - if(fd < 0) { - GKFS_DATA->spdlogger()->warn( - "{}() Chunk file not found, skipping: {} (err: {})", - __func__, chunk_path, strerror(errno)); - ++fail_count; - continue; - } - - struct stat st; - if(fstat(fd, &st) < 0) { - close(fd); - GKFS_DATA->spdlogger()->warn( - "{}() Failed to stat chunk file: {} (err: {})", __func__, - chunk_path, strerror(errno)); - ++fail_count; - continue; - } - - std::vector buf(st.st_size); - ssize_t bytes_read = read(fd, buf.data(), st.st_size); - close(fd); - - if(bytes_read < 0) { - GKFS_DATA->spdlogger()->warn( - "{}() Failed to read chunk file: {} (err: {})", __func__, - chunk_path, strerror(errno)); - ++fail_count; - continue; - } - - // Forward data to target node using existing RPC infrastructure - auto err = gkfs::malleable::rpc::forward_data( - job.path, buf.data(), static_cast(bytes_read), - static_cast(job.chunk_id), - static_cast(job.target_node)); - - if(err == 0) { - ++success_count; - total_bytes += static_cast(bytes_read); - GKFS_DATA->spdlogger()->trace( - "{}() Migrated: {} chnk {} {} bytes to host {}", __func__, - job.path, job.chunk_id, bytes_read, job.target_node); - } else { - ++fail_count; - GKFS_DATA->spdlogger()->error( - "{}() Failed to migrate data for {} chnk {} to host {}: err {}", - __func__, job.path, job.chunk_id, job.target_node, err); - } - } - - GKFS_DATA->spdlogger()->info( - "{}() Migration completed: {} succeeded, {} failed, {} bytes transferred", - __func__, success_count, fail_count, total_bytes); - - return fail_count > 0 ? -1 : 0; -} - void -MalleableManager::expand_abt(void* _arg) { - GKFS_DATA->spdlogger()->info("{}() Starting expansion process...", +MalleableManager::expand_abt_v2(void* _arg) { + auto self = static_cast(_arg); + if(!self) { + GKFS_DATA->spdlogger()->error( + "{}() No MalleableManager pointer available", __func__); + return; + } + GKFS_DATA->spdlogger()->info("{}() Starting v2 expansion process...", __func__); GKFS_DATA->redist_running(true); GKFS_DATA->malleable_manager()->redistribute_metadata(); try { - GKFS_DATA->malleable_manager()->redistribute_data_legacy(); + GKFS_DATA->malleable_manager()->redistribute_data_v2(); } catch(const gkfs::data::ChunkStorageException& e) { GKFS_DATA->spdlogger()->error("{}() Failed to redistribute data: '{}'", __func__, e.what()); } GKFS_DATA->redist_running(false); GKFS_DATA->spdlogger()->info( - "{}() Expansion process successfully finished.", __func__); + "{}() V2 expansion process successfully finished.", __func__); } void -MalleableManager::expand_abt_v2(void* _arg) { +MalleableManager::expand_on_demand_abt(void* _arg) { auto self = static_cast(_arg); if(!self) { GKFS_DATA->spdlogger()->error( "{}() No MalleableManager pointer available", __func__); return; } - GKFS_DATA->spdlogger()->info("{}() Starting v2 expansion process...", - __func__); + GKFS_DATA->spdlogger()->info( + "{}() Starting expand-on-demand process: metadata eager, data lazy.", + __func__); GKFS_DATA->redist_running(true); GKFS_DATA->malleable_manager()->redistribute_metadata(); - try { - GKFS_DATA->malleable_manager()->redistribute_data_v2(); - } catch(const gkfs::data::ChunkStorageException& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to redistribute data: '{}'", - __func__, e.what()); - } GKFS_DATA->redist_running(false); GKFS_DATA->spdlogger()->info( - "{}() V2 expansion process successfully finished.", __func__); + "{}() Expand-on-demand metadata redistribution finished. Data chunks remain lazy.", + __func__); } // PUBLIC void -MalleableManager::expand_start(int old_server_conf, int new_server_conf, +MalleableManager::mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file) { - // Capture old hosts_size BEFORE distributor update - old_hosts_size_ = RPC_DATA->distributor()->hosts_size(); + // Use the control-plane old server count as old topology size. Added + // daemons start with an empty local distributor, so RPC_DATA alone is not a + // valid old-layout snapshot on those nodes. + old_hosts_size_ = static_cast(old_server_conf); + + GKFS_DATA->spdlogger()->info("{}() Loading new hosts file '{}' for mutate " + "(old={} nodes, new={} nodes)", + __func__, new_hosts_file, old_server_conf, + new_server_conf); + + // MARKER-BASED WORKFLOW: Parse markers to get the "after" state (active + + // adding). load_hostfile() strips all markers, which loses '+' daemons for + // expand. We must use parse_hostfile_markers to include adding nodes in the + // after state. + auto markers = gkfs::malleable::parse_hostfile_markers(new_hosts_file); + vector> hosts; + for(const auto& e : markers.active) { + hosts.emplace_back(e.hostname, e.uri); + } + for(const auto& e : markers.adding) { + hosts.emplace_back(e.hostname, e.uri); + } + sort(hosts.begin(), hosts.end()); + // Remove rootdir suffix + for(auto& h : hosts) { + auto idx = h.first.rfind("#"); + if(idx != string::npos) + h.first.erase(idx, h.first.length()); + } - auto hosts = load_hostfile(new_hosts_file); if(hosts.size() != static_cast(new_server_conf)) { throw runtime_error( fmt::format("MalleableManager::{}() Something is wrong. " @@ -670,42 +660,151 @@ MalleableManager::expand_start(int old_server_conf, int new_server_conf, "does not match new server configuration ({})", __func__, hosts.size(), new_server_conf)); } - connect_to_hosts(hosts); + // On mutate, any old node not in the new set will be removed (pass true to + // connect_to_hosts so it handles missing local host gracefully) + connect_to_hosts(hosts, true); - // Update distributor with new host count (this triggers partition recalc) - RPC_DATA->distributor()->hosts_size(hosts.size()); + const bool expand_on_demand = + env_truthy(std::getenv(gkfs::env::EXPAND_ON_DEMAND)); + const bool pure_expand = + old_server_conf < new_server_conf && markers.removing.empty(); + if(expand_on_demand && !pure_expand) { + GKFS_DATA->spdlogger()->warn( + "{}() {} requested but disabled: only pure expand is supported (old={}, new={}, removing={}). Falling back to eager data migration.", + __func__, gkfs::env::EXPAND_ON_DEMAND, old_server_conf, + new_server_conf, markers.removing.size()); + } - // Use v2 pipeline for new expansion - auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, - ABT_THREAD_ATTR_NULL, &redist_thread_); - if(abt_err != ABT_SUCCESS) { - auto err_str = fmt::format( - "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", - __func__, abt_err); - throw runtime_error(err_str); + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); } -} -void -MalleableManager::shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file) { - // Capture old hosts_size BEFORE distributor update - old_hosts_size_ = RPC_DATA->distributor()->hosts_size(); + auto distributor = gkfs::rpc::create_from_config( + config, static_cast(RPC_DATA->local_host_id()), + static_cast(hosts.size()), 0); + if(!distributor) { + throw runtime_error("Failed to recreate distributor for mutate"); + } - GKFS_DATA->spdlogger()->info("{}() Loading new hosts file '{}' for shrink", - __func__, new_hosts_file); - auto hosts = load_hostfile(new_hosts_file); - if(hosts.size() != static_cast(new_server_conf)) { - throw runtime_error( - fmt::format("MalleableManager::{}() Something is wrong. " - "Number of hosts in new hosts file ({}) " - "does not match new server configuration ({})", - __func__, hosts.size(), new_server_conf)); + auto old_distributor = gkfs::rpc::create_from_config( + config, static_cast(RPC_DATA->local_host_id()), + old_hosts_size_, 0); + if(!old_distributor) { + throw runtime_error("Failed to recreate old distributor for mutate"); + } + + if(config.is_random_slicing()) { + vector> before_hosts; + before_hosts.reserve(markers.active.size() + markers.removing.size()); + for(const auto& e : markers.active) { + before_hosts.emplace_back(e.hostname, e.uri); + } + for(const auto& e : markers.removing) { + before_hosts.emplace_back(e.hostname, e.uri); + } + sort(before_hosts.begin(), before_hosts.end()); + for(auto& h : before_hosts) { + auto idx = h.first.rfind("#"); + if(idx != string::npos) { + h.first.erase(idx, h.first.length()); + } + } + + unordered_map before_keys; + for(const auto& host : before_hosts) { + before_keys.emplace(host_key(host), true); + } + + vector added_ids; + for(size_t i = 0; i < hosts.size(); ++i) { + if(before_keys.find(host_key(hosts[i])) == before_keys.end()) { + added_ids.push_back(static_cast(i)); + } + } + + auto* rs = dynamic_cast( + distributor.get()); + vector final_intervals; + const bool cutshift = + env_truthy(std::getenv(gkfs::env::RANDOM_SLICING_CUTSHIFT)); + + if(rs && cutshift && markers.removing.empty() && !added_ids.empty() && + !before_hosts.empty()) { + auto comments = + gkfs::malleable::parse_rs_interval_comments(new_hosts_file); + auto old_intervals = + comments.empty() ? vector{} + : remap_existing_rs_comments( + comments, before_hosts, hosts); + auto old_partitions = + old_intervals.empty() + ? make_equal_rs_partitions_for_before_hosts( + before_hosts, hosts) + : intervals_to_partitions(old_intervals); + + auto updated_partitions = gkfs::rpc::expand_with_cutshift( + old_partitions, added_ids, 1.0f, 1.0f); + for(const auto& partition : updated_partitions) { + for(const auto& interval : partition.intervals) { + final_intervals.push_back(interval); + } + } + } + + if(rs) { + if(final_intervals.empty()) { + final_intervals = rs->get_intervals(); + } else if(!rs->set_intervals(final_intervals)) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to install CutShift RS intervals; falling back to rebuilt RS layout", + __func__); + final_intervals = rs->get_intervals(); + } + + if(RPC_DATA->local_host_id() == 0) { + try { + gkfs::malleable::write_rs_interval_comments( + new_hosts_file, + to_interval_comments(final_intervals)); + GKFS_DATA->spdlogger()->info( + "{}() Wrote {} random-slicing intervals to hostfile", + __func__, final_intervals.size()); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to write random-slicing intervals to hostfile: {}", + __func__, e.what()); + } + } + } + } + + GKFS_DATA->spdlogger()->info( + "{}() Recreated '{}' distributor for new topology with {} hosts", + __func__, config.get_strategy_string(), hosts.size()); + RPC_DATA->distributor(std::move(distributor)); + + if(expand_on_demand && pure_expand) { + GKFS_DATA->expand_on_demand_active(true); + GKFS_DATA->expand_on_demand_old_hosts_size(old_hosts_size_); + GKFS_DATA->expand_on_demand_old_distributor(std::move(old_distributor)); + GKFS_DATA->spdlogger()->info( + "{}() {} active: old_hosts={}, new_hosts={}. Skipping eager data migration.", + __func__, gkfs::env::EXPAND_ON_DEMAND, old_hosts_size_, + hosts.size()); + auto abt_err = + ABT_thread_create(RPC_DATA->io_pool(), expand_on_demand_abt, + this, ABT_THREAD_ATTR_NULL, &redist_thread_); + if(abt_err != ABT_SUCCESS) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", + __func__, abt_err)); + } + return; } - connect_to_hosts(hosts, true); - RPC_DATA->distributor()->hosts_size(hosts.size()); - // Use v2 pipeline for shrink + // Use v2 pipeline for mutate auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index a2397f8e0..13d4a73cd 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -42,7 +42,11 @@ #include #include #include +#include +#include +#include #include +#include extern "C" { #include @@ -52,6 +56,165 @@ using namespace std; namespace gkfs::data { +namespace { + +bool +expand_on_demand_owns_final_missing_candidate(const std::string& path, + uint64_t chunk_id, + uint64_t& old_owner) { + if(!GKFS_DATA->expand_on_demand_active()) { + return false; + } + auto old_dist = GKFS_DATA->expand_on_demand_old_distributor(); + auto final_dist = RPC_DATA->distributor(); + if(!old_dist || !final_dist) { + return false; + } + + const auto local = RPC_DATA->local_host_id(); + const auto final_owner = final_dist->locate_data( + path, static_cast(chunk_id), 0); + if(final_owner != local) { + return false; + } + + old_owner = old_dist->locate_data( + path, static_cast(chunk_id), 0); + return old_owner != local && + old_owner < GKFS_DATA->expand_on_demand_old_hosts_size() && + RPC_DATA->rpc_endpoints().find(old_owner) != + RPC_DATA->rpc_endpoints().end(); +} + +void +async_materialize_local_full_read(const std::string& path, uint64_t chunk_id, + const char* buf, ssize_t size, + off64_t offset) { + if(size <= 0 || offset != 0) { + return; + } + + std::vector materialized(buf, buf + size); + std::thread([path, chunk_id, data = std::move(materialized)]() { + try { + GKFS_DATA->storage()->write_chunk( + path, static_cast(chunk_id), + data.data(), data.size(), 0); + GKFS_DATA->spdlogger()->debug( + "{}() expand-on-demand materialized chunk '{}' for '{}' ({} bytes)", + __func__, chunk_id, path, data.size()); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->warn( + "{}() expand-on-demand async materialization failed for '{}' chunk '{}': {}", + __func__, path, chunk_id, e.what()); + } + }).detach(); +} + +} // namespace + +std::pair +expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, + char* buf, size_t size, off64_t offset) { + uint64_t old_owner = 0; + if(!expand_on_demand_owns_final_missing_candidate(path, chunk_id, + old_owner)) { + return {ENOENT, 0}; + } + + std::vector> segments = { + std::make_pair(buf, size)}; + tl::bulk bulk_handle; + try { + bulk_handle = RPC_DATA->client_rpc_engine()->expose( + segments, tl::bulk_mode::write_only); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() failed to expose fallback read bulk for '{}' chunk '{}': {}", + __func__, path, chunk_id, e.what()); + return {EBUSY, 0}; + } + + gkfs::rpc::rpc_read_data_in_t in{}; + in.path = path; + in.offset = offset; + in.host_id = old_owner; + in.host_size = GKFS_DATA->expand_on_demand_old_hosts_size(); + std::vector bitset(1, 1); + in.wbitset = gkfs::rpc::compress_bitset(bitset); + in.chunk_n = 1; + in.chunk_start = chunk_id; + in.chunk_end = chunk_id; + in.total_chunk_size = size; + in.bulk_handle = bulk_handle; + + try { + auto read_data = + RPC_DATA->client_rpc_engine()->define(gkfs::rpc::tag::read); + auto out = read_data.on(RPC_DATA->rpc_endpoints().at(old_owner))(in) + .as(); + if(out.err == 0 && out.io_size > 0) { + async_materialize_local_full_read(path, chunk_id, buf, + static_cast(out.io_size), + offset); + } + GKFS_DATA->spdlogger()->debug( + "{}() expand-on-demand fallback read '{}' chunk '{}' from old owner '{}': err={} size={}", + __func__, path, chunk_id, old_owner, out.err, out.io_size); + return {out.err, static_cast(out.io_size)}; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() expand-on-demand fallback read failed for '{}' chunk '{}' from old owner '{}': {}", + __func__, path, chunk_id, old_owner, e.what()); + return {EBUSY, 0}; + } +} + +int +expand_on_demand_materialize_for_partial_write(const std::string& path, + uint64_t chunk_id) { + uint64_t old_owner = 0; + if(!expand_on_demand_owns_final_missing_candidate(path, chunk_id, + old_owner)) { + return 0; + } + + char probe = 0; + try { + auto ret = GKFS_DATA->storage()->read_chunk( + path, static_cast(chunk_id), &probe, 1, + 0); + if(ret >= 0) { + return 0; + } + } catch(const gkfs::data::ChunkStorageException& e) { + if(e.code().value() != ENOENT) { + return e.code().value(); + } + } + + std::vector chunk(gkfs::config::rpc::chunksize); + auto [err, read_size] = expand_on_demand_read_remote( + path, chunk_id, chunk.data(), chunk.size(), 0); + if(err == ENOENT) { + return 0; + } + if(err != 0) { + return err; + } + if(read_size <= 0) { + return 0; + } + try { + GKFS_DATA->storage()->write_chunk( + path, static_cast(chunk_id), chunk.data(), + static_cast(read_size), 0); + } catch(const gkfs::data::ChunkStorageException& e) { + return e.code().value(); + } + return 0; +} + /* ------------------------------------------------------------------------ * -------------------------- TRUNCATE ------------------------------------ * ------------------------------------------------------------------------*/ @@ -456,6 +619,29 @@ ChunkReadOperation::wait_for_tasks_and_push_back(const bulk_args& args) { // sparse regions do not have chunk files and are therefore // skipped if(-(*task_size) == ENOENT) { + auto fallback = expand_on_demand_read_remote( + path_, args.chunk_ids->at(idx), + task_args_[idx].buf, task_args_[idx].size, + task_args_[idx].off); + if(fallback.first == 0 && fallback.second > 0) { + try { + args.local_bulk_handle( + args.local_offsets->at(idx), + fallback.second) >> + args.origin_bulk_handle.on( + args.endpoint)( + args.origin_offsets->at(idx), + fallback.second); + total_read += fallback.second; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "ChunkReadOperation::{}() Failed to push expand-on-demand fallback data to client with thallium err: '{}'", + __func__, e.what()); + io_err = EBUSY; + } + } else if(fallback.first != ENOENT) { + io_err = fallback.first; + } task_args_[idx].bulk_transfer_done = true; bulk_transfer_cnt++; ABT_eventual_free(&task_eventuals_[idx]); @@ -533,6 +719,26 @@ ChunkReadOperation::wait_for_tasks_and_push_back(const bulk_args& args) { // sparse regions do not have chunk files and are therefore // skipped if(-(*task_size) == ENOENT) { + auto fallback = expand_on_demand_read_remote( + path_, args.chunk_ids->at(idx), task_args_[idx].buf, + task_args_[idx].size, task_args_[idx].off); + if(fallback.first == 0 && fallback.second > 0) { + try { + args.local_bulk_handle(args.local_offsets->at(idx), + fallback.second) >> + args.origin_bulk_handle.on(args.endpoint)( + args.origin_offsets->at(idx), + fallback.second); + total_read += fallback.second; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "ChunkReadOperation::{}() Failed to push expand-on-demand fallback data to client with thallium err: '{}'", + __func__, e.what()); + io_err = EBUSY; + } + } else if(fallback.first != ENOENT) { + io_err = fallback.first; + } ABT_eventual_free(&task_eventuals_[idx]); continue; } diff --git a/src/daemon/util.cpp b/src/daemon/util.cpp index f53a98194..7acfb2da7 100644 --- a/src/daemon/util.cpp +++ b/src/daemon/util.cpp @@ -101,13 +101,14 @@ namespace gkfs::utils { void -populate_hosts_file() { +populate_hosts_file(bool expand_mode) { const auto& hosts_file = GKFS_DATA->hosts_file(); const auto& daemon_addr = RPC_DATA->self_addr_str(); const auto& proxy_addr = RPC_DATA->self_proxy_addr_str(); - GKFS_DATA->spdlogger()->debug("{}() Populating hosts file: '{}'", __func__, - hosts_file); + GKFS_DATA->spdlogger()->debug( + "{}() Populating hosts file: '{}' (expand_mode={})", __func__, + hosts_file, expand_mode); // if rootdir_suffix is used, append it to hostname auto hostname = GKFS_DATA->rootdir_suffix().empty() @@ -127,6 +128,11 @@ populate_hosts_file() { (int) GKFS_DATA->link_cnt_state(), (int) GKFS_DATA->blocks_state(), getuid(), getgid()); + // Prepend '+' marker if in expand mode + if(expand_mode) { + line_out = "+" + line_out; + } + // Constants for retry mechanism const int MAX_RETRIES = 5; // Maximum number of retry attempts const std::chrono::milliseconds RETRY_DELAY( diff --git a/src/proxy/proxy.cpp b/src/proxy/proxy.cpp index 706002719..c00a64f81 100644 --- a/src/proxy/proxy.cpp +++ b/src/proxy/proxy.cpp @@ -25,6 +25,7 @@ #include #include +#include #include #include #include @@ -204,7 +205,7 @@ init_environment(const string& hostfile_path, const string& rpc_protocol) { // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var // (defaults to simple_hash for backward compatibility) gkfs::rpc::DistributionConfig config; - const char* env_val = std::getenv("GKFS_DISTRIBUTION_STRATEGY"); + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); if(env_val != nullptr && env_val[0] != '\0') { config.set_strategy(env_val); } @@ -311,7 +312,7 @@ main(int argc, const char* argv[]) { // Check for environment variables for configuration gkfs::config::rpc::use_dirents_compression = - gkfs::env::get_var("GKFS_PROXY_USE_DIRENTS_COMPRESSION", + gkfs::env::get_var(gkfs::env::USE_DIRENTS_COMPRESSION, gkfs::config::rpc::use_dirents_compression ? "ON" : "OFF") == "ON"; diff --git a/tests/integration/directories/test_packing_order.py b/tests/integration/directories/test_packing_order.py index d650f330f..bd0d8d3f5 100644 --- a/tests/integration/directories/test_packing_order.py +++ b/tests/integration/directories/test_packing_order.py @@ -14,7 +14,7 @@ def test_packing_order_all_fields(test_workspace, request): # Start Daemon with compression ON daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "ON" + "GKFS_USE_DIRENTS_COMPRESSION": "ON" } daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() @@ -22,7 +22,7 @@ def test_packing_order_all_fields(test_workspace, request): try: client = ShellClient(test_workspace) mount_dir = test_workspace.mountdir - client_env = {"LIBGKFS_USE_DIRENTS_COMPRESSION": "ON"} + client_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} # 1. Create files with specific properties # file_a: size 100 diff --git a/tests/integration/directories/test_sfind.py b/tests/integration/directories/test_sfind.py index 1fd17873f..033dc0bc9 100644 --- a/tests/integration/directories/test_sfind.py +++ b/tests/integration/directories/test_sfind.py @@ -31,7 +31,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): log.info("--- Phase 1: Population (Safe Mode) ---") pop_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "OFF" + "GKFS_USE_DIRENTS_COMPRESSION": "OFF" } daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=pop_daemon_env) daemon_pop.run() @@ -39,7 +39,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): try: # Client Env for Population (Safe Mode) pop_client_env = { - "LIBGKFS_USE_DIRENTS_COMPRESSION": "OFF", + "GKFS_USE_DIRENTS_COMPRESSION": "OFF", "LIBGKFS_DENTRY_CACHE": "OFF", "LIBGKFS_LOG": "info" } @@ -69,7 +69,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): test_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": conf["compress"] + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"] } daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=test_daemon_env) daemon_test.run() @@ -77,7 +77,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): try: # Client Env for Test test_client_env = { - "LIBGKFS_USE_DIRENTS_COMPRESSION": conf["compress"], + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], "LIBGKFS_DENTRY_CACHE": conf["cache"], "LIBGKFS_DIRENTS_BUFF_SIZE": buff_size, "LIBGKFS_LOG": "info" diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index a70746361..a3e608aee 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -64,9 +64,9 @@ gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' gkfwd_daemon_log_file = 'gkfs_daemon.log' -gkfwd_daemon_log_level = '100' +gkfwd_daemon_log_level = '20' gkfwd_client_log_file = 'gkfs_client.log' -gkfwd_client_log_level = 'all' +gkfwd_client_log_level = 'debug' gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -243,16 +243,24 @@ class FwdDaemonCreator: self._interface = interface self._workspace = workspace - def create(self): + def create(self, expand_mode=False, enable_forwarding=True): """ Create a forwarding daemon in the tests workspace. + Parameters + ---------- + expand_mode: `bool` + If True, sets GKFS_DAEMON_EXPAND=ON in the daemon environment. + Returns ------- The `FwdDaemon` object to interact with the daemon. """ - daemon = FwdDaemon(self._interface, self._workspace) + daemon = FwdDaemon(self._interface, + self._workspace, + expand_mode=expand_mode, + enable_forwarding=enable_forwarding) daemon.run() return daemon @@ -692,7 +700,13 @@ class Client: stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - out_stdout, out_stderr = proc.communicate() + try: + out_stdout, out_stderr = proc.communicate(timeout=kwargs.get('timeout')) + except subprocess.TimeoutExpired: + proc.kill() + out_stdout, out_stderr = proc.communicate() + logger.error(f"Command timed out after {kwargs.get('timeout')}s") + out_stdout = b'{"errnum": 110, "retval": -1}' out_returncode = proc.returncode if out_stdout: @@ -1388,13 +1402,26 @@ class ShellClientLibc: return self._workspace.twd class FwdDaemon: - def __init__(self, interface, workspace): + def __init__(self, interface, workspace, expand_mode=False, enable_forwarding=True): self._address = get_ephemeral_address(interface) self._workspace = workspace self._hostfile = str(self.cwd / gkfwd_hosts_file) self._cmd = find_command(gkfwd_daemon_cmd, self._workspace.bindirs) self._env = os.environ.copy() + self._expand_mode = expand_mode + self._enable_forwarding = enable_forwarding + self._rootdir = self._workspace.rootdir + self._metadir = self._workspace.metadir + self._logdir = self._workspace.logdir + if not enable_forwarding: + suffix = self._address.replace(':', '_') + self._rootdir = self._workspace.twd / f"root_{suffix}" + self._metadir = self._workspace.twd / f"meta_{suffix}" + self._logdir = self._workspace.logdir / f"daemon_{suffix}" + self._rootdir.mkdir(parents=True, exist_ok=True) + self._metadir.mkdir(parents=True, exist_ok=True) + self._logdir.mkdir(parents=True, exist_ok=True) libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + @@ -1404,8 +1431,12 @@ class FwdDaemon: 'LD_LIBRARY_PATH' : libdirs, 'GKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'GKFS_DAEMON_LOG_PATH' : str(self.logdir / gkfwd_daemon_log_file), - 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level + 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level, + 'GKFS_DAEMON_KEEP_HOSTS_FILE': 'ON' } + if expand_mode: + self._patched_env['GKFS_DAEMON_EXPAND'] = 'ON' + self._env.update(self._patched_env) def run(self): @@ -1413,8 +1444,9 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address, - '--enable-forwarding'] + '-l', self._address] + if self._enable_forwarding: + args.append('--enable-forwarding') logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -1512,11 +1544,11 @@ class FwdDaemon: @property def rootdir(self): - return self._workspace.rootdir + return self._rootdir @property def metadir(self): - return self._workspace.metadir + return self._metadir @property def mountdir(self): @@ -1524,7 +1556,7 @@ class FwdDaemon: @property def logdir(self): - return self._workspace.logdir + return self._logdir @property def interface(self): diff --git a/tests/integration/malleability/test_client_disconnect_during_rpc.py b/tests/integration/malleability/test_client_disconnect_during_rpc.py index c682483f4..f8aec82af 100644 --- a/tests/integration/malleability/test_client_disconnect_during_rpc.py +++ b/tests/integration/malleability/test_client_disconnect_during_rpc.py @@ -63,7 +63,7 @@ def test_daemon_survives_client_abort_during_write(gkfwd_daemon_factory, gkfs_sh cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"Daemon not responding before client abort: {cmd.stderr.decode()}" @@ -140,7 +140,7 @@ if fd >= 0: cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ diff --git a/tests/integration/malleability/test_expand_on_demand.py b/tests/integration/malleability/test_expand_on_demand.py new file mode 100644 index 000000000..fffddfe39 --- /dev/null +++ b/tests/integration/malleability/test_expand_on_demand.py @@ -0,0 +1,163 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +import hashlib +import os +import shutil +import stat +import time +from pathlib import Path + +import pytest + + +FILE_COUNT = 8 +FILE_SIZE = 1024 * 1024 +PARTIAL_OVERWRITE_OFFSET = 128 +PARTIAL_OVERWRITE = b"expand-on-demand-partial" + + +def _get_malleability_bin(gkfs_shell): + search_path = ":".join(str(p) for p in gkfs_shell._search_paths) + malleability_bin = shutil.which("gkfs_malleability", path=search_path) + assert malleability_bin is not None, "gkfs_malleability not found in PATH" + return malleability_bin + + +def _run_mutate_cmd(gkfs_shell, bin_path, hosts_file, args, timeout=120): + libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") + cmd_str = ( + f'LD_LIBRARY_PATH="{libdirs}" ' + f'LIBGKFS_HOSTS_FILE="{hosts_file}" ' + f'{bin_path} {args}' + ) + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + assert cmd.exit_code == 0, ( + f"gkfs_malleability {args} failed\n" + f"stdout: {cmd.stdout.decode()}\n" + f"stderr: {cmd.stderr.decode()}" + ) + return cmd + + +def _mark_added_daemon(hostfile, daemon): + port = daemon.address.rsplit(":", 1)[-1] + lines = Path(hostfile).read_text().splitlines(keepends=True) + with open(hostfile, "w") as hf_out: + for line in lines: + stripped = line.strip() + if stripped and not stripped.startswith("#") and f":{port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + +def _read_md5(client, path): + ret = client.stat(path) + assert ret.retval == 0, f"stat failed for {path}" + size = ret.statbuf.st_size + ret = client.open(path, os.O_RDONLY, 0) + assert ret.retval != -1, f"open read failed for {path}" + ret = client.read(path, size) + assert ret.retval == size, f"read failed for {path}: {ret.retval}/{size}" + assert ret.buf is not None, f"read returned no buffer for {path}" + return hashlib.md5(ret.buf).hexdigest() + + +def _expected_payload_with_partial_overwrite(): + payload = bytearray((ord("0") + (i % 10)) for i in range(FILE_SIZE)) + payload[ + PARTIAL_OVERWRITE_OFFSET : PARTIAL_OVERWRITE_OFFSET + len(PARTIAL_OVERWRITE) + ] = PARTIAL_OVERWRITE + return bytes(payload) + + +def _write_file(client, path): + ret = client.open( + path, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open write failed for {path}" + ret = client.write_validate(path, FILE_SIZE) + assert ret.retval == 0, f"write_validate failed for {path}" + return _read_md5(client, path) + + +def _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile): + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + if "No mutate running/finished." in cmd.stderr.decode(): + return + time.sleep(2) + pytest.fail("Mutate did not finish within 120 s") + + +def test_expand_on_demand_skips_eager_data_and_keeps_reads_correct( + monkeypatch, gkfwd_daemon_factory, gkfs_client, gkfs_shell +): + monkeypatch.setenv("GKFS_EXPAND_ON_DEMAND", "ON") + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", "simple_hash") + + daemons = [] + try: + d00 = gkfwd_daemon_factory.create() + daemons.append(d00) + time.sleep(2) + + md5_map = {} + for i in range(FILE_COUNT): + path = d00.mountdir / f"expand_on_demand_{i:03d}.dat" + md5_map[path] = _write_file(gkfs_client, path) + + d01 = gkfwd_daemon_factory.create(expand_mode=True) + daemons.append(d01) + time.sleep(2) + + hostfile = Path(d00.hostfile) + _mark_added_daemon(hostfile, d01) + malleability_bin = _get_malleability_bin(gkfs_shell) + + _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate start", 340) + _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile) + _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate finalize") + + daemon_log = Path(d00.logdir) / "gkfs_daemon.log" + log_text = daemon_log.read_text() if daemon_log.exists() else "" + assert "GKFS_EXPAND_ON_DEMAND active" in log_text + assert "Skipping eager data migration" in log_text + + expected_after_partial = _expected_payload_with_partial_overwrite() + expected_after_partial_md5 = hashlib.md5(expected_after_partial).hexdigest() + partial_path = next(iter(md5_map.keys())) + ret = gkfs_client.pwrite( + partial_path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + + for path, expected_md5 in md5_map.items(): + if path == partial_path: + assert _read_md5(gkfs_client, path) == expected_after_partial_md5 + else: + assert _read_md5(gkfs_client, path) == expected_md5 + + for path in list(md5_map.keys())[1:]: + ret = gkfs_client.pwrite( + path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + assert _read_md5(gkfs_client, path) == expected_after_partial_md5 + + finally: + for daemon in daemons: + daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/malleability/test_malleability_error_handling.py b/tests/integration/malleability/test_malleability_error_handling.py index f63c649a6..5e97dd6bd 100644 --- a/tests/integration/malleability/test_malleability_error_handling.py +++ b/tests/integration/malleability/test_malleability_error_handling.py @@ -46,8 +46,6 @@ def test_expand_status_with_no_running_expansion(gkfwd_daemon_factory, gkfs_shel time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") search_path = ":".join(str(p) for p in gkfs_shell._search_paths) @@ -58,10 +56,10 @@ def test_expand_status_with_no_running_expansion(gkfwd_daemon_factory, gkfs_shel cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"expand status failed: {cmd.stderr.decode()}" + assert cmd.exit_code == 0, f"mutate status failed: {cmd.stderr.decode()}" d00.shutdown() @@ -72,8 +70,13 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") + + # Create a new-hosts-file with the same content (same node count) + new_hostfile = hostfile.parent / "gkfs_hosts_same.txt" + with open(hostfile, "r") as hf_in, open(new_hostfile, "w") as hf_out: + for line in hf_in: + if not line.startswith("#"): + hf_out.write(line) libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") search_path = ":".join(str(p) for p in gkfs_shell._search_paths) @@ -81,10 +84,21 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): assert malleability_bin is not None, "gkfs_malleability not found in PATH" + # Use marker-based approach: no --new-hosts-file needed + # Mark all entries with '+' to simulate adding same nodes (will fail gracefully) + with open(hostfile, 'r') as hf_in: + original_lines = hf_in.readlines() + with open(hostfile, 'w') as hf_out: + for line in original_lines: + if not line.startswith('#') and line.strip(): + hf_out.write('+' + line) + else: + hf_out.write(line) + cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" + f"{malleability_bin} mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) @@ -93,7 +107,7 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ @@ -108,8 +122,13 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") + + # Create a new-hosts-file with the same content (will fail) + new_hostfile = hostfile.parent / "gkfs_hosts_fail.txt" + with open(hostfile, "r") as hf_in, open(new_hostfile, "w") as hf_out: + for line in hf_in: + if not line.startswith("#"): + hf_out.write(line) libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") search_path = ":".join(str(p) for p in gkfs_shell._search_paths) @@ -117,20 +136,29 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Try expand (may fail) + # Try mutate with markers (may fail due to same node count) + with open(hostfile, 'r') as hf_in: + original_lines = hf_in.readlines() + with open(hostfile, 'w') as hf_out: + for line in original_lines: + if not line.startswith('#') and line.strip(): + hf_out.write('+' + line) + else: + hf_out.write(line) + cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" + f"{malleability_bin} mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) - # Verify shrink status works + # Verify mutate status works cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} shrink status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ @@ -159,8 +187,6 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s assert ret.retval == 0, f"write_validate failed for {f}" hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") search_path = ":".join(str(p) for p in gkfs_shell._search_paths) @@ -168,20 +194,29 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Verify no running expansion + # Verify no running mutation cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - # Start expansion + # Start mutation with markers (fails due to same node count, which is expected) + with open(hostfile, 'r') as hf_in: + original_lines = hf_in.readlines() + with open(hostfile, 'w') as hf_out: + for line in original_lines: + if not line.startswith('#') and line.strip(): + hf_out.write('+' + line) + else: + hf_out.write(line) + cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" + f"{malleability_bin} mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) @@ -189,7 +224,7 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py new file mode 100644 index 000000000..dfd3e1dda --- /dev/null +++ b/tests/integration/malleability/test_malleability_performance.py @@ -0,0 +1,1589 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Standalone performance test for GekkoFS malleability (shrink/expand/mutate). + +MARKER-BASED WORKFLOW: +- LIBGKFS_HOSTS_FILE must contain ALL hosts with markers +- For shrink: '-' markers on removing nodes +- For expand: '+' markers on adding nodes +- The workspace contains ALL daemons (active + adding/removing) +""" + +import os +import signal +import random +import stat +import time +import math +import hashlib +import json +import shutil +import statistics +import subprocess +import tempfile +import shlex +from pathlib import Path +from collections import Counter, defaultdict +from dataclasses import dataclass +from time import perf_counter +import pytest +from harness.gkfs import FwdDaemonCreator +from harness.logger import logger + + +# ========= +# Helpers +# ========= + +def _malleability_supports_mutate(binary): + """Return True if gkfs_malleability binary has the marker-based mutate CLI.""" + try: + completed = subprocess.run( + [binary, "--help"], + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired): + return False + + return "mutate" in completed.stdout or "mutate" in completed.stderr + + +def _get_malleability_bin(gkfs_shell): + """Find a gkfs_malleability binary compatible with these tests.""" + search_path = ":".join(str(p) for p in gkfs_shell._search_paths) + candidates = [] + found = shutil.which("gkfs_malleability", path=search_path) + if found is not None: + candidates.append(found) + + # The debug-local install prefix can be stale. Prefer the real install prefix + # used by CI/local deps, then the build-tree tool. + candidates.extend([ + "/home/rnou/iodeps/bin/gkfs_malleability", + "/home/rnou/gekkofs/builds/debug-local/tools/gkfs_malleability", + ]) + + checked = [] + for candidate in candidates: + if candidate in checked or not os.path.exists(candidate): + continue + checked.append(candidate) + if _malleability_supports_mutate(candidate): + logger.info(f"Using gkfs_malleability: {candidate}") + return candidate + + raise AssertionError( + "No compatible gkfs_malleability found. Checked: " + f"{', '.join(checked) or ''}. Use --bin-dir=/home/rnou/iodeps/bin " + "or rebuild/install so the binary supports 'mutate'." + ) + + +def _shutdown_daemons(daemons, timeout=10): + """Shutdown daemons without letting perf tests hang forever.""" + for daemon in daemons: + try: + proc = getattr(daemon, "_proc", None) + if proc is None or proc.poll() is not None: + continue + + logger.debug(f"terminating daemon pid {proc.pid}") + proc.terminate() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning(f"daemon pid {proc.pid} did not stop after SIGTERM; killing") + proc.kill() + proc.wait(timeout=timeout) + except Exception as exc: + logger.warning(f"daemon cleanup failed: {exc}") + + shutil.rmtree(daemon.rootdir.as_posix(), ignore_errors=True) + + # Give the backend a moment to release the mount and RPC state before the + # next performance run starts. Without this pause, the next run may hit + # transient EBUSY failures on file creation. + time.sleep(2) + + +def _snapshot_chunk_files(rootdir): + """Snapshot chunk files under a daemon rootdir. + + The relative chunk path is the primary chunk identity. The digest is kept as + a guard against stale files and as a fallback for path-renamed chunks. Do not + aggregate by digest alone: performance tests create deterministic equal-size + files, so many distinct chunks can have identical content. + """ + rootdir = Path(rootdir) + chunkdir = rootdir / "chunks" + snapshot = {} + + if not chunkdir.exists(): + return snapshot + + for path in chunkdir.rglob("*"): + if not path.is_file(): + continue + try: + data = path.read_bytes() + except Exception: + continue + relpath = path.relative_to(rootdir).as_posix() + snapshot[relpath] = (path.stat().st_size, hashlib.md5(data).hexdigest()) + + return snapshot + + +def _count_moved_chunk_bytes(before_snapshots, after_snapshots): + """Count bytes for chunks that changed daemon ownership. + + Robust rules: + - Primary match is `(relative chunk path, size, digest)`. This keeps chunks + with duplicate content separate. + - Count with multiplicity, not sets. If N equal chunks move, count N chunks. + - Count new materializations on different daemons. Some implementations copy + chunks first and delete old copies later, so an unchanged source copy must + not hide the redistributed destination copy. + - Lost chunks are caught by MD5 verification, not reported as redistributed + bytes. + - Fallback to `(size, digest)` only for unmatched chunks, in case a backend + changes the relative chunk path during redistribution. + """ + + def build_locations(snapshots, key_fn): + locations = defaultdict(Counter) + sizes = {} + for daemon, snapshot in snapshots.items(): + for relpath, (size, digest) in snapshot.items(): + key = key_fn(relpath, size, digest) + locations[key][daemon] += 1 + sizes[key] = size + return locations, sizes + + def count_moved(before_locations, after_locations, sizes): + moved = 0 + matched_keys = set() + for key, before_daemons in before_locations.items(): + after_daemons = after_locations.get(key) + if not after_daemons: + continue + after_total = sum(after_daemons.values()) + unchanged = sum( + min(before_daemons[daemon], after_daemons.get(daemon, 0)) + for daemon in before_daemons + ) + moved_instances = max(0, after_total - unchanged) + if moved_instances: + moved += moved_instances * sizes[key] + matched_keys.add(key) + return moved, matched_keys + + before_by_identity, identity_sizes = build_locations( + before_snapshots, lambda relpath, size, digest: (relpath, size, digest)) + after_by_identity, _ = build_locations( + after_snapshots, lambda relpath, size, digest: (relpath, size, digest)) + + moved, matched_identity_keys = count_moved( + before_by_identity, after_by_identity, identity_sizes) + + # Fallback for path-renamed chunks only. Exclude identities already matched + # exactly so duplicate-content chunks are not double counted or collapsed. + unmatched_before = defaultdict(dict) + unmatched_after = defaultdict(dict) + for daemon, snapshot in before_snapshots.items(): + for relpath, value in snapshot.items(): + size, digest = value + if (relpath, size, digest) not in matched_identity_keys: + unmatched_before[daemon][relpath] = value + for daemon, snapshot in after_snapshots.items(): + for relpath, value in snapshot.items(): + size, digest = value + if (relpath, size, digest) not in matched_identity_keys: + unmatched_after[daemon][relpath] = value + + before_by_digest, digest_sizes = build_locations( + unmatched_before, lambda relpath, size, digest: (size, digest)) + after_by_digest, _ = build_locations( + unmatched_after, lambda relpath, size, digest: (size, digest)) + fallback_moved, _ = count_moved(before_by_digest, after_by_digest, digest_sizes) + + return moved + fallback_moved + + +def _wait_for_unmounted(mountdir, timeout=10): + mountdir = Path(mountdir) + deadline = time.time() + timeout + while time.time() < deadline: + if not mountdir.exists(): + return + try: + if os.path.ismount(mountdir): + time.sleep(0.5) + continue + # If the path still exists but is no longer a mount, make sure it is + # actually writable before we continue. This catches stale fuse state + # that can still keep the next iteration busy. + probe = mountdir / ".gkfs_unmount_probe" + try: + probe.touch(exist_ok=True) + probe.unlink(missing_ok=True) + return + except Exception: + time.sleep(0.5) + continue + except Exception: + time.sleep(0.5) + + +def _cleanup_daemon_storage(daemon): + shutil.rmtree(daemon.rootdir.as_posix(), ignore_errors=True) + shutil.rmtree(daemon.metadir.as_posix(), ignore_errors=True) + #if daemon.logdir.exists(): + # shutil.rmtree(daemon.logdir.as_posix(), ignore_errors=True) + + +class _IterationWorkspaceAdapter: + """Per-iteration workspace with fresh mount/root/meta/log dirs.""" + + def __init__(self, real_workspace, iteration_root): + self._ws = real_workspace + self._twd = Path(iteration_root) + self._rootdir = self._twd / "root" + self._metadir = self._twd / "meta" + self._mountdir = self._twd / "mnt" + self._logdir = self._twd / "logs" + self._tmpdir = self._twd / "tmp" + self._rootdir.mkdir(parents=True, exist_ok=True) + self._metadir.mkdir(parents=True, exist_ok=True) + self._mountdir.mkdir(parents=True, exist_ok=True) + self._logdir.mkdir(parents=True, exist_ok=True) + self._tmpdir.mkdir(parents=True, exist_ok=True) + + @property + def twd(self): + return self._twd + + @property + def bindirs(self): + return self._ws.bindirs + + @property + def libdirs(self): + return self._ws.libdirs + + @property + def logdir(self): + return self._logdir + + @property + def rootdir(self): + return self._rootdir + + @property + def metadir(self): + return self._metadir + + @property + def mountdir(self): + return self._mountdir + + @property + def tmpdir(self): + return self._tmpdir + + +def _make_iteration_workspace(base_workspace, iteration_name): + iter_root = Path(tempfile.mkdtemp(prefix=f"{iteration_name}_", dir=str(base_workspace.twd))) + return _IterationWorkspaceAdapter(base_workspace, iter_root) + +def _set_client_hostfile(client, hostfile): + """Point harness client commands at the desired hosts file.""" + value = str(hostfile) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = value + + +def _wait_for_client_mount_ready(client, mountdir, timeout=45): + """Wait until the client can create and remove a file in the mount. + + Daemon processes can exist before the FUSE mount and RPC state are fully + ready. A fixed sleep is not enough on slower random-slicing/cutshift runs and + may make the first test file creation fail with EBUSY. + """ + mountdir = Path(mountdir) + deadline = time.time() + timeout + probe = mountdir / f".gkfs_ready_probe_{os.getpid()}_{int(time.time() * 1000)}" + last_error = None + + while time.time() < deadline: + try: + ret = client.open(probe, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if ret.retval != -1: + try: + client.unlink(probe) + except Exception: + pass + return + last_error = getattr(ret, 'errno', None) + except Exception as exc: + last_error = exc + time.sleep(1) + + pytest.fail(f"GekkoFS mount {mountdir} not ready after {timeout}s; last_error={last_error}") + + +def create_deterministic_file(client, mountdir, filename, size): + """Create a random reference file in GekkoFS and return its expected MD5. + + Data comes from /dev/urandom into a local reference file. Its MD5 is computed + once outside GekkoFS, then copied into the mounted file with one intercepted + dd process. This keeps random integrity coverage without reading generated + GekkoFS files back through Python before the malleability operation starts. + """ + fpath = mountdir / filename + ref_path = mountdir.parent / f".{filename}.ref" + # Make repeated perf runs idempotent: if a previous run left the file in + # place, remove it before recreating it. + try: + client.unlink(fpath) + except Exception: + pass + + last_ret = None + for attempt in range(30): + ret = client.open(fpath, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + last_ret = ret + if ret.retval != -1: + break + last_errno = getattr(ret, 'errno', None) + if last_errno != 16: + break + logger.warning(f"open busy for {fpath} (attempt {attempt + 1}/30), retrying") + logger.debug(f" mountdir exists={mountdir.exists()} ismount={os.path.ismount(mountdir)}") + time.sleep(1) + + assert last_ret is not None + assert last_ret.retval != -1, \ + f"open failed for {fpath}, errno={getattr(last_ret, 'errno', None)}" + + timeout = max(60, int(size / (1024 * 1024)) * 2) + completed = subprocess.run( + [ + "dd", + "if=/dev/urandom", + f"of={str(ref_path)}", + "bs=1M", + f"count={size}", + "iflag=count_bytes", + "conv=notrunc", + "status=none", + ], + env=os.environ, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + assert completed.returncode == 0, ( + f"reference random-write failed for {ref_path}: {completed.stderr.decode()[:300]}" + ) + + digest = hashlib.md5() + with ref_path.open("rb") as ref_file: + for block in iter(lambda: ref_file.read(1024 * 1024), b""): + digest.update(block) + expected_md5 = digest.hexdigest() + + completed = subprocess.run( + [ + "dd", + f"if={str(ref_path)}", + f"of={str(fpath)}", + "bs=1M", + "conv=notrunc", + "status=none", + ], + env=getattr(client, "_env", os.environ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + try: + ref_path.unlink() + except OSError: + pass + assert completed.returncode == 0, ( + f"copy into GekkoFS failed for {fpath}: " + f"returncode={describe_returncode(completed.returncode)}, " + f"stdout={completed.stdout.decode(errors='replace')[:300]}, " + f"stderr={completed.stderr.decode(errors='replace')[:300]}" + ) + + return fpath, expected_md5 + + +def describe_returncode(returncode): + """Return a readable subprocess return code description.""" + if returncode is None or returncode >= 0: + return str(returncode) + signum = -returncode + try: + signame = signal.Signals(signum).name + return f"{returncode} ({signame})" + except ValueError: + return str(returncode) + + +def count_accessible_files(file_paths, client, timeout=60): + """Count accessible files with one intercepted shell process.""" + if not file_paths: + return 0 + quoted = " ".join(shlex.quote(str(p)) for p in file_paths) + cmd = f"ok=0; for f in {quoted}; do [ -f \"$f\" ] && ok=$((ok+1)); done; echo $ok" + completed = subprocess.run( + ["bash", "-c", cmd], + env=getattr(client, "_env", os.environ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout, + ) + if completed.returncode != 0: + logger.warning(f"fast accessibility check failed: {completed.stderr.decode()[:300]}") + return sum(1 for fpath in file_paths if client.stat(fpath, timeout=15).retval == 0) + try: + return int(completed.stdout.decode().strip() or "0") + except ValueError: + return 0 + + +def verify_files_with_md5(file_md5_map, client, mountdir, max_read_checks=None): + """Read back all files and verify their MD5 checksums.""" + results = {'passed': 0, 'failed': 0, 'details': []} + items = list(file_md5_map.items()) + if max_read_checks is not None: + items = items[:max_read_checks] + + if items: + try: + paths = [path for path, _ in items] + completed = subprocess.run( + ["md5sum"] + paths, + env=getattr(client, "_env", os.environ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=max(30, len(paths) * 5), + ) + if completed.returncode == 0: + actual = {} + for line in completed.stdout.decode().splitlines(): + parts = line.split(None, 1) + if len(parts) == 2: + actual[parts[1].lstrip('*')] = parts[0] + for fpath_str, expected_md5 in items: + actual_md5 = actual.get(fpath_str) + if actual_md5 == expected_md5: + results['passed'] += 1 + else: + results['failed'] += 1 + results['details'].append({'file': fpath_str, + 'status': 'md5_mismatch', + 'expected': expected_md5, + 'actual': actual_md5}) + skipped = len(file_md5_map) - len(items) + results['passed'] += skipped + return results + logger.warning(f"fast md5sum failed: {completed.stderr.decode()[:300]}") + except Exception as exc: + logger.warning(f"fast md5sum exception: {exc}") + + for idx, (fpath_str, expected_md5) in enumerate(file_md5_map.items()): + fpath = Path(fpath_str) + try: + ret = client.open(fpath, os.O_RDONLY, 0, timeout=15) + if ret.retval == -1: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'open_failed'}) + continue + + stat_ret = client.stat(fpath, timeout=15) + if stat_ret.retval != 0: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'stat_failed'}) + continue + + if max_read_checks is not None and idx >= max_read_checks: + results['passed'] += 1 + continue + + file_size = stat_ret.statbuf.st_size + if file_size > 0: + ret = client.read(fpath, file_size, timeout=15) + if ret.retval == -1 or ret.buf is None: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'read_failed'}) + continue + actual_md5 = hashlib.md5(ret.buf).hexdigest() + else: + actual_md5 = hashlib.md5(b'').hexdigest() + + if actual_md5 == expected_md5: + results['passed'] += 1 + else: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'md5_mismatch', + 'expected': expected_md5, 'actual': actual_md5}) + except Exception as e: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'exception', 'error': str(e)}) + + return results + + +def _extract_port(address): + """Extract the port number from a daemon address (e.g., 'lo:54321' -> '54321').""" + return address.rsplit(':', 1)[-1] + + +def _get_entry_from_hostfile(hostfile_path, port): + """Get the full entry line for a daemon port from a hostfile. + + Match the RPC URI port exactly. Do not use substring matching: random ports + such as 1234 and 31234 can otherwise select the wrong daemon entry and build + a corrupt marker workspace for mutate start. + """ + if not os.path.exists(hostfile_path): + return None + with open(hostfile_path) as f: + for line in f: + stripped = line.strip() + if not stripped or stripped.startswith('#'): + continue + + clean = stripped[1:] if stripped and stripped[0] in '+-' else stripped + fields = clean.split() + if len(fields) < 2: + continue + uri_port = fields[1].rsplit(':', 1)[-1] + if uri_port == port: + return clean if clean.endswith('\n') else clean + '\n' + return None + + +def _build_workspace_with_markers(workspace_path, daemons_to_write, marker_map=None): + """Build workspace hostfile with markers for malleability operations. + + Args: + workspace_path: Path to write + daemons_to_write: List of daemon objects + marker_map: dict of daemon -> marker ('+', '-', or '') + """ + marker_map = marker_map or {} + with open(workspace_path, 'w') as f: + for d in daemons_to_write: + marker = marker_map.get(d.address, '') + entry = _get_entry_from_hostfile(d.hostfile, _extract_port(d.address)) + if entry: + f.write(marker + entry) + else: + f.write(f"{marker}lo {d.address} 10000 10000 1 1 lo {d.address} 10000 10000\n") + + +def _build_workspace_for_shrink(workspace_path, active_daemons, removing_daemons): + """Build workspace for shrink: active (no marker) + removing ('-' marker).""" + marker_map = {} + for d in active_daemons: + marker_map[d.address] = '' + for d in removing_daemons: + marker_map[d.address] = '-' + + all_daemons = list(active_daemons) + list(removing_daemons) + _build_workspace_with_markers(workspace_path, all_daemons, marker_map) + + +def _build_workspace_for_expand(workspace_path, active_daemons, adding_daemons): + """Build workspace for expand: active (no marker) + adding ('+' marker).""" + marker_map = {} + for d in active_daemons: + marker_map[d.address] = '' + for d in adding_daemons: + marker_map[d.address] = '+' + + all_daemons = list(active_daemons) + list(adding_daemons) + _build_workspace_with_markers(workspace_path, all_daemons, marker_map) + + +# ========= +# Stats +# ========= + +@dataclass +class OperationResult: + run_index: int + elapsed_seconds: float + wall_time_seconds: float + data_redistributed_bytes: int + md5_passed: int + md5_failed: int + files_accessible: int + files_total: int + + +def compute_statistics(results): + """Compute statistical summary over multiple runs.""" + if not results: + return {"num_runs": 0} + + times = [r.elapsed_seconds for r in results] + wall_times = [r.wall_time_seconds for r in results] + + mean_elapsed = statistics.mean(times) + std_elapsed = statistics.stdev(times) if len(times) > 1 else 0.0 + mean_wall = statistics.mean(wall_times) + + if len(times) > 1: + se = std_elapsed / math.sqrt(len(times)) + t_values = {1: 12.71, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571} + t_val = t_values.get(len(times), 1.96) + ci_low = mean_elapsed - t_val * se + ci_high = mean_elapsed + t_val * se + else: + ci_low = ci_high = mean_elapsed + + cv = (std_elapsed / mean_elapsed * 100) if mean_elapsed > 0 else 0.0 + + moved_bytes = [r.data_redistributed_bytes for r in results] + moved_mb = [value / (1024**2) for value in moved_bytes] + moved_mean_mb = statistics.mean(moved_mb) + moved_stddev_mb = statistics.stdev(moved_mb) if len(moved_mb) > 1 else 0.0 + moved_cv = (moved_stddev_mb / moved_mean_mb * 100) if moved_mean_mb > 0 else 0.0 + + if len(moved_mb) > 1: + moved_p95_bytes = statistics.quantiles(moved_bytes, n=100, method='inclusive')[94] + moved_p95_mb = statistics.quantiles(moved_mb, n=100, method='inclusive')[94] + else: + moved_p95_bytes = moved_bytes[0] + moved_p95_mb = moved_mb[0] + + return { + "num_runs": len(results), + "mean_elapsed_seconds": round(mean_elapsed, 4), + "stddev_elapsed_seconds": round(std_elapsed, 4), + "min_elapsed_seconds": round(min(times), 4), + "max_elapsed_seconds": round(max(times), 4), + "ci_95_lower": round(ci_low, 4), + "ci_95_upper": round(ci_high, 4), + "coefficient_of_variation_pct": round(cv, 2), + "mean_wall_time_seconds": round(mean_wall, 4), + "total_data_redistributed_bytes": sum(moved_bytes), + "mean_data_redistributed_bytes": round(statistics.mean(moved_bytes), 2), + "stddev_data_redistributed_bytes": round(statistics.stdev(moved_bytes), 2) if len(moved_bytes) > 1 else 0.0, + "min_data_redistributed_bytes": min(moved_bytes), + "max_data_redistributed_bytes": max(moved_bytes), + "median_data_redistributed_bytes": round(statistics.median(moved_bytes), 2), + "p95_data_redistributed_bytes": round(moved_p95_bytes, 2), + "coefficient_of_variation_data_redistributed_pct": round(moved_cv, 2), + "total_data_redistributed_mb": round(sum(moved_mb), 4), + "mean_data_redistributed_mb": round(moved_mean_mb, 4), + "stddev_data_redistributed_mb": round(moved_stddev_mb, 4), + "min_data_redistributed_mb": round(min(moved_mb), 4), + "max_data_redistributed_mb": round(max(moved_mb), 4), + "median_data_redistributed_mb": round(statistics.median(moved_mb), 4), + "p95_data_redistributed_mb": round(moved_p95_mb, 4), + "all_md5_passed": all(r.md5_failed == 0 for r in results), + "all_files_accessible": all(r.files_accessible == r.files_total for r in results), + } + + +def compute_numeric_statistics(values, unit_suffix=""): + """Compute complete statistics for a numeric series.""" + if not values: + return {f"num_samples{unit_suffix}": 0} + + mean_value = statistics.mean(values) + stddev_value = statistics.stdev(values) if len(values) > 1 else 0.0 + cv = (stddev_value / mean_value * 100) if mean_value > 0 else 0.0 + p95_value = (statistics.quantiles(values, n=100, method='inclusive')[94] + if len(values) > 1 else values[0]) + + return { + f"num_samples{unit_suffix}": len(values), + f"total{unit_suffix}": round(sum(values), 4), + f"mean{unit_suffix}": round(mean_value, 4), + f"stddev{unit_suffix}": round(stddev_value, 4), + f"min{unit_suffix}": round(min(values), 4), + f"max{unit_suffix}": round(max(values), 4), + f"median{unit_suffix}": round(statistics.median(values), 4), + f"p95{unit_suffix}": round(p95_value, 4), + f"coefficient_of_variation_pct{unit_suffix}": round(cv, 2), + } + + +DEFAULT_REPETITIONS = int(os.environ.get("GKFS_PERF_REPETITIONS", "3")) +DEFAULT_NUM_FILES = int(os.environ.get("GKFS_PERF_NUM_FILES", "32")) +DEFAULT_FILE_SIZE = int(os.environ.get("GKFS_PERF_FILE_SIZE", str(8 * 1024))) +DEFAULT_OLD_NODES = int(os.environ.get("GKFS_MALLEABILITY_OLD_NODES", "4")) +DEFAULT_NEW_NODES = int(os.environ.get("GKFS_MALLEABILITY_NEW_NODES", "2")) +DEFAULT_TIMEOUT = 340 +CI_FAST = os.environ.get("GKFS_MALLEABILITY_CI_FAST", "").lower() in ("1", "on", "true", "yes") +KEEP_ITERATION_WORKSPACES = os.environ.get( + "GKFS_MALLEABILITY_KEEP_WORKSPACES", "").lower() in ("1", "on", "true", "yes") +EXPAND_ON_DEMAND = os.environ.get( + "GKFS_EXPAND_ON_DEMAND", "OFF").lower() in ("1", "on", "true", "yes") +EXPAND_ON_DEMAND_MD5_READ_CHECKS = int(os.environ.get( + "GKFS_EXPAND_ON_DEMAND_MD5_READ_CHECKS", "1")) + + +def _cleanup_iteration_workspace(iter_workspace): + if KEEP_ITERATION_WORKSPACES: + logger.info(f" Preserving iteration workspace: {iter_workspace.twd}") + return + shutil.rmtree(iter_workspace.twd.as_posix(), ignore_errors=True) + + +def _dump_mutate_failure_context(label, cmd, workspace_file): + logger.error(f"{label} failed with exit code {cmd.exit_code}") + logger.error(f"stdout:\n{cmd.stdout.decode(errors='replace')[:4000]}") + logger.error(f"stderr:\n{cmd.stderr.decode(errors='replace')[:4000]}") + workspace_file = Path(workspace_file) + if workspace_file.exists(): + logger.error(f"workspace {workspace_file}:\n{workspace_file.read_text(errors='replace')[:4000]}") + else: + logger.error(f"workspace {workspace_file} does not exist") + + log_root = workspace_file.parent / "logs" + if log_root.exists(): + for daemon_log in sorted(log_root.glob("daemon_*/gkfs_daemon.log")): + try: + logger.error(f"tail {daemon_log}:\n{daemon_log.read_text(errors='replace')[-4000:]}") + except OSError as exc: + logger.error(f"failed to read {daemon_log}: {exc}") + + +# ============ +# Test: shrink +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_shrink_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple shrink operations and collect statistical performance data. + + MARKER-BASED WORKFLOW: + - Workspace contains ALL daemons (active + removing) + - Removing daemons get '-' markers + - CLI computes old_nodes = active + removing, new_nodes = active + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = DEFAULT_OLD_NODES + num_end = DEFAULT_NEW_NODES + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + + for run_idx in range(num_reps): + time.sleep(10) # Give the backend time to release resources from previous run + logger.info("=" * 70) + logger.info(f"SHRINK PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"shrink_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Create daemons + daemons = [] + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + time.sleep(5) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"shrink_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + logger.info(f" Generated {num_files} files in {gen_time:.3f}s") + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Pick survivors randomly + survivors = random.sample(daemons, num_end) + removing_daemons = [d for d in daemons if d not in survivors] + + # Build workspace with markers: all active + removing ('-' marker) + workspace_file = hostfile.parent / f"shrink_workspace_run{run_idx}.txt" + _build_workspace_for_shrink(workspace_file, survivors, removing_daemons) + + libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] + malleability_bin = _get_malleability_bin(gkfs_shell) + + env_file = hostfile.parent / f"test_env_shrink_run{run_idx}.sh" + env_file.write_text( + f'export LD_LIBRARY_PATH="{libdirs}"\n' + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + ) + + logger.info(f" Shrink: {len(survivors)} keep, {len(removing_daemons)} remove") + + # Execute shrink + t0 = perf_counter() + t_wall = time.time() + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) + assert cmd.exit_code == 0, f"Shrink start failed: {cmd.stderr.decode()[:300]}" + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail(f"Shrink did not complete within 120s (run {run_idx + 1})") + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + time.sleep(2) + + # Verify + accessible = count_accessible_files(created_files, client) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + if post_verify['failed']: + logger.warning(f" Shrink post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Shrink redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: shrink took {elapsed:.3f}s, files: {accessible}/{num_files}") + + # Clean up: shutdown daemons and wipe backend storage for fresh iteration + _shutdown_daemons(daemons) + _wait_for_unmounted(run_mountdir) + for d in daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + + stats = compute_statistics(all_results) + logger.info("=" * 70) + logger.info("SHRINK PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("SHRINK PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + _shutdown_daemons(all_daemons) + for d in all_daemons: + _cleanup_daemon_storage(d) + + + +# ============ +# Test: expand +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_expand_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple expand operations and collect statistical performance data. + + MARKER-BASED WORKFLOW: + - Workspace contains ALL daemons (active + adding) + - Adding daemons get '+' markers + - CLI computes old_nodes = active, new_nodes = active + adding + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = 1 if CI_FAST else 2 + num_end = 2 if CI_FAST else 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + expand_on_demand_value = "ON" if EXPAND_ON_DEMAND else "OFF" + + for run_idx in range(num_reps): + time.sleep(10) # Give the backend time to release resources from previous run + logger.info("=" * 70) + logger.info(f"EXPAND PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info(f" GKFS_EXPAND_ON_DEMAND={expand_on_demand_value}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"expand_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + client._patched_env["GKFS_EXPAND_ON_DEMAND"] = expand_on_demand_value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + client._env["GKFS_EXPAND_ON_DEMAND"] = expand_on_demand_value + + # Create initial daemons + daemons = [] + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + time.sleep(5) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"expand_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Expand: create new daemons, then build a marker workspace with active + # daemons and '+' adding daemons. The daemon auto-write path is tested + # elsewhere; this perf test uses an explicit workspace for repeatability. + new_daemons = [] + for i in range(num_end - num_start): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + time.sleep(2) # Wait for daemon to write '+' to workspace + all_daemons.extend(new_daemons) + logger.info(f" Expand: {len(daemons)} active, {len(new_daemons)} adding") + + workspace_file = hostfile.parent / f"expand_workspace_run{run_idx}.txt" + _build_workspace_for_expand(workspace_file, daemons, new_daemons) + + # Verify '+' entries appeared + with open(workspace_file) as f: + content = f.read() + adding_entries = [l for l in content.split('\n') if l.startswith('+')] + logger.info(f" Expand: {len(adding_entries)} '+' entries in marker workspace") + assert len(adding_entries) == len(new_daemons), \ + f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" + + libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] + malleability_bin = _get_malleability_bin(gkfs_shell) + + env_file = hostfile.parent / f"test_env_expand_run{run_idx}.sh" + env_file.write_text( + f'export LD_LIBRARY_PATH="{libdirs}"\n' + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + f'export GKFS_EXPAND_ON_DEMAND="{expand_on_demand_value}"\n' + ) + + logger.info(f" Expand: {len(daemons)} active, {len(new_daemons)} adding") + + # Execute expand + t0 = perf_counter() + t_wall = time.time() + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Expand mutate start", cmd, workspace_file) + assert cmd.exit_code == 0, f"Expand start failed: {cmd.stderr.decode()[:300]}" + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail(f"Expand did not complete within 120s (run {run_idx + 1})") + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + time.sleep(2) + + # Verify + accessible = count_accessible_files(created_files, client) + max_read_checks = (EXPAND_ON_DEMAND_MD5_READ_CHECKS + if EXPAND_ON_DEMAND else None) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=max_read_checks) + if post_verify['failed']: + logger.warning(f" Expand post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons + new_daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Expand redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: expand took {elapsed:.3f}s, files: {accessible}/{num_files}") + + # Clean up + _shutdown_daemons(daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + + stats = compute_statistics(all_results) + logger.info("=" * 70) + logger.info("EXPAND PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("EXPAND PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + +# ============ +# Test: mutate +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_mutate_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple mutate (4→4 with different hosts) operations. + + For same-count topology change: + - Old daemons get '-' markers (will be removed) + - New daemons get '+' markers (will be added) + - Workspace contains ALL daemons + - CLI computes old_nodes = removing, new_nodes = adding + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = 2 if CI_FAST else 4 + num_end = 2 if CI_FAST else 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + + for run_idx in range(num_reps): + time.sleep(10) # Give the backend time to release resources from previous run + logger.info("=" * 70) + logger.info(f"MUTATE PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"mutate_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Create initial daemons + old_daemons = [] + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + old_daemons.append(d) + time.sleep(5) + all_daemons.extend(old_daemons) + + hostfile = Path(old_daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = old_daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"mutate_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in old_daemons} + + # Create replacement daemons + new_daemons = [] + for i in range(num_end): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + time.sleep(1) + all_daemons.extend(new_daemons) + + # For same-count mutate: all old get '-' markers, all new get '+' markers + workspace_file = hostfile.parent / f"mutate_workspace_run{run_idx}.txt" + marker_map = {} + for d in old_daemons: + marker_map[d.address] = '-' + for d in new_daemons: + marker_map[d.address] = '+' + all_daemons_to_write = list(old_daemons) + list(new_daemons) + _build_workspace_with_markers(workspace_file, all_daemons_to_write, marker_map) + + libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] + malleability_bin = _get_malleability_bin(gkfs_shell) + + env_file = hostfile.parent / f"test_env_mutate_run{run_idx}.sh" + env_file.write_text( + f'export LD_LIBRARY_PATH="{libdirs}"\n' + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + ) + + logger.info(f" Mutate: {len(old_daemons)} removing, {len(new_daemons)} adding") + + # Execute mutate + t0 = perf_counter() + t_wall = time.time() + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Mutate start", cmd, workspace_file) + result = OperationResult( + run_index=run_idx, elapsed_seconds=0, wall_time_seconds=0, + data_redistributed_bytes=0, md5_passed=0, md5_failed=0, + files_accessible=0, files_total=num_files, + ) + all_results.append(result) + _shutdown_daemons(old_daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in old_daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + continue + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + time.sleep(2) + + # Verify + accessible = count_accessible_files(created_files, client) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + if post_verify['failed']: + logger.warning(f" Mutate post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in old_daemons + new_daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Mutate redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: mutate took {elapsed:.3f}s, files: {accessible}/{num_files}") + + # Clean up + _shutdown_daemons(old_daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in old_daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + + # Use all results (mutate can complete in <1s) + completed = all_results + stats = compute_statistics(completed) + logger.info("=" * 70) + logger.info("MUTATE PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("MUTATE PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + _shutdown_daemons(all_daemons) + for d in all_daemons: + _cleanup_daemon_storage(d) + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + + +# ============ +# Test: cycle +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_comprehensive_cycle_performance(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple shrink→expand→mutate cycles with full metrics.""" + if CI_FAST: + pytest.skip("skip heavy cycle performance test in CI fast mode") + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_initial = 4 + num_shrink_to = 2 + num_expand_to = 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_cycle_results = [] + all_daemons = [] + + for rep_idx in range(num_reps): + time.sleep(10) # Give the backend time to release resources from previous run + logger.info("=" * 70) + logger.info(f"COMPREHENSIVE CYCLE TEST - REPLICATION {rep_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"cycle_iter_{rep_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Step 1: Start daemons and generate data + daemons = [] + for i in range(num_initial): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + time.sleep(5) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"cycle_{rep_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + logger.info(f" Step 1: Generated {num_files} files in {gen_time:.3f}s") + before_shrink_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Step 2: Shrink + survivors = random.sample(daemons, num_shrink_to) + removing_daemons = [d for d in daemons if d not in survivors] + + workspace_shrink = hostfile.parent / f"cycle_shrink_workspace_{rep_idx}.txt" + _build_workspace_for_shrink(workspace_shrink, survivors, removing_daemons) + + libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] + malleability_bin = _get_malleability_bin(gkfs_shell) + + env_file = hostfile.parent / f"test_env_cycle{rep_idx}.sh" + env_file.write_text( + f'export LD_LIBRARY_PATH="{libdirs}"\n' + f'export LIBGKFS_HOSTS_FILE="{workspace_shrink}"\n' + ) + + shrink_t0 = perf_counter() + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Cycle shrink mutate start", cmd, workspace_shrink) + assert cmd.exit_code == 0, f"Cycle shrink start failed: {cmd.stderr.decode()[:300]}" + + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + + shrink_elapsed = perf_counter() - shrink_t0 + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_shrink) + time.sleep(2) + + post_shrink_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + logger.info(f" Step 2: Shrink took {shrink_elapsed:.3f}s") + if post_shrink_verify['failed']: + logger.warning(f" Cycle shrink post-verify failures: {post_shrink_verify['details'][:5]}") + after_shrink_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + shrink_moved_bytes = _count_moved_chunk_bytes(before_shrink_snapshots, + after_shrink_snapshots) + logger.info(f" Cycle shrink redistributed {shrink_moved_bytes / (1024 ** 2):.4f} MB") + + # Clean up removed daemons + _shutdown_daemons(removing_daemons) + for d in removing_daemons: + _cleanup_daemon_storage(d) + + surviving_daemons = survivors + before_expand_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in surviving_daemons} + + # Step 3: Expand with marker workspace + logger.info(f" Step 3: Expand {len(surviving_daemons)} -> {num_expand_to}") + + new_daemons = [] + for i in range(num_expand_to - num_shrink_to): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + time.sleep(2) # Wait for daemon to write '+' to workspace + all_daemons.extend(new_daemons) + + workspace_expand = hostfile.parent / f"cycle_expand_workspace_{rep_idx}.txt" + _build_workspace_for_expand(workspace_expand, surviving_daemons, + new_daemons) + + # Verify '+' entries appeared + with open(workspace_expand) as f: + content = f.read() + adding_entries = [l for l in content.split('\n') if l.startswith('+')] + logger.info(f" Expand: {len(adding_entries)} '+' entries in marker workspace") + assert len(adding_entries) == len(new_daemons), \ + f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" + + env_file.write_text( + f'export LD_LIBRARY_PATH="{libdirs}"\n' + f'export LIBGKFS_HOSTS_FILE="{workspace_expand}"\n' + ) + + expand_t0 = perf_counter() + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Cycle expand mutate start", cmd, workspace_expand) + assert cmd.exit_code == 0, ( + "Cycle expand start failed:\n" + f"cmd: {cmd_str}\n" + f"stdout: {cmd.stdout.decode(errors='replace')[:1000]}\n" + f"stderr: {cmd.stderr.decode(errors='replace')[:1000]}\n" + f"workspace:\n{workspace_expand.read_text(errors='replace')[:2000]}" + ) + + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + + expand_elapsed = perf_counter() - expand_t0 + cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_expand) + time.sleep(2) + + post_expand_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + logger.info(f" Step 3: Expand took {expand_elapsed:.3f}s") + if post_expand_verify['failed']: + logger.warning(f" Cycle expand post-verify failures: {post_expand_verify['details'][:5]}") + after_expand_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in surviving_daemons + new_daemons} + expand_moved_bytes = _count_moved_chunk_bytes(before_expand_snapshots, + after_expand_snapshots) + logger.info(f" Cycle expand redistributed {expand_moved_bytes / (1024 ** 2):.4f} MB") + + # Clean up + _shutdown_daemons(daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + all_cycle_results.append({ + "replication": rep_idx, + "shrink_elapsed": round(shrink_elapsed, 4), + "expand_elapsed": round(expand_elapsed, 4), + "shrink_data_redistributed_mb": round(shrink_moved_bytes / (1024 ** 2), 4), + "expand_data_redistributed_mb": round(expand_moved_bytes / (1024 ** 2), 4), + "pre_verify_passed": pre_verify['passed'], + "post_shrink_md5_passed": post_shrink_verify['passed'], + "post_expand_md5_passed": post_expand_verify['passed'], + }) + + logger.info("=" * 70) + logger.info("COMPREHENSIVE CYCLE TEST - SUMMARY") + logger.info("=" * 70) + for r in all_cycle_results: + logger.info(json.dumps(r, indent=2)) + cycle_stats = { + "num_runs": len(all_cycle_results), + "shrink_data_redistributed_mb_stats": compute_numeric_statistics( + [r["shrink_data_redistributed_mb"] for r in all_cycle_results], "_mb"), + "expand_data_redistributed_mb_stats": compute_numeric_statistics( + [r["expand_data_redistributed_mb"] for r in all_cycle_results], "_mb"), + "total_data_redistributed_mb_stats": compute_numeric_statistics( + [r["shrink_data_redistributed_mb"] + r["expand_data_redistributed_mb"] + for r in all_cycle_results], "_mb"), + } + logger.info("COMPREHENSIVE CYCLE DATA MOVEMENT STATS") + logger.info(json.dumps(cycle_stats, indent=2)) + print("=" * 70) + print("COMPREHENSIVE CYCLE DATA MOVEMENT STATS") + print("=" * 70) + print(json.dumps(cycle_stats, indent=2)) + + assert len(all_cycle_results) == num_reps + assert all(r["post_shrink_md5_passed"] == num_files for r in all_cycle_results) + assert all(r["post_expand_md5_passed"] == num_files for r in all_cycle_results) + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--tb=long"]) \ No newline at end of file diff --git a/tests/integration/malleability/test_malleability_tool.py b/tests/integration/malleability/test_malleability_tool.py index 8f3a3296e..3cd667a51 100644 --- a/tests/integration/malleability/test_malleability_tool.py +++ b/tests/integration/malleability/test_malleability_tool.py @@ -29,175 +29,209 @@ import harness from pathlib import Path import shutil -import errno import stat import os -import ctypes -import sh -import sys import pytest from harness.logger import logger -nonexisting = "nonexisting" def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): import time + + # Start with 1 daemon (before expand) d00 = gkfwd_daemon_factory.create() - # Add "#FS_INSTANCE_END" in the file with name d00.hostfile - time.sleep(5) - with open(d00.hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - - # loop 10 times, and create a file in each iteration + hostfile = Path(d00.hostfile) + # Create some files before expand for i in range(4): - file = d00.mountdir / f"file{i}" - # create a file in gekkofs - ret = gkfs_client.open(file, + fpath = d00.mountdir / f"file{i}" + ret = gkfs_client.open(fpath, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1 - - ret = gkfs_client.write_validate(file, 1024 * 1024) + ret = gkfs_client.write_validate(fpath, 1024 * 1024) assert ret.retval == 0 - # Create content + # Create new daemon to add (after expand) + d01 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) - d01 = gkfwd_daemon_factory.create() libdirs = gkfs_shell._patched_env['LD_LIBRARY_PATH'] search_path = ':'.join(str(p) for p in gkfs_shell._search_paths) malleability_bin = shutil.which('gkfs_malleability', path=search_path) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand status" + # Marker-based API: LIBGKFS_HOSTS_FILE = workspace hostfile with markers + # - d00 stays (no prefix) + # - d01 added with '+' prefix + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + + # Write d00's entry (no prefix = active) and d01's entry with '+' prefix (to add) + with open(hostfile, "w") as hf_out: + # Write d00's entry as-is (active) + for line in original_lines: + if not line.startswith("#") and line.strip(): + hf_out.write(line) + # Add d01's entry with '+' prefix (to-be-added daemon) + with open(d01.hostfile, "r") as d01_hf: + for line in d01_hf: + if not line.startswith("#"): + hf_out.write("+" + line) + + cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"Command '{cmd_str}' failed with {cmd.exit_code}: {cmd.stderr.decode()}" - assert "No expansion running/finished.\n" in cmd.stderr.decode() - - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand start" + assert cmd.exit_code == 0, f"Command '{cmd_str}' failed: {cmd.stderr.decode()}" + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) - + assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" + assert "Mutate process" in cmd.stderr.decode() + time.sleep(10) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand finalize" + cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate finalize" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - + assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" + d00.shutdown() d01.shutdown() def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): import time + d00 = gkfwd_daemon_factory.create() - # Add "#FS_INSTANCE_END" in the file with name d00.hostfile time.sleep(5) - cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + + search_path = ':'.join(str(p) for p in gkfs_shell._search_paths) + malleability_bin = shutil.which('gkfs_malleability', path=search_path) + + hostfile = Path(d00.hostfile) + + # Test error when old == new node count (no markers, same hosts) + cmd = gkfs_shell.gkfs_malleability('mutate', 'start', timeout=340) assert cmd.exit_code != 0 - assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" - with open(d00.hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") + # Test error when hostfile has markers for same daemon (removing all) + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + + # Mark all entries with '-' (would remove all daemons) + with open(hostfile, "w") as hf_out: + for line in original_lines: + if not line.startswith("#") and line.strip(): + hf_out.write("-" + line) + else: + hf_out.write(line) - cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate start" + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) assert cmd.exit_code != 0 - assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" + assert "All hosts would be removed" in cmd.stderr.decode() or "Error" in cmd.stderr.decode() + d00.shutdown() + def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): - """Test that shrinking from 2 nodes to 1 redistributes data correctly.""" + """Test that shrinking from 2 nodes to 1 redistributes data correctly. + + Uses the unified 'mutate' subcommand (shrinking = fewer nodes after). + """ import time + d00 = gkfwd_daemon_factory.create() d01 = gkfwd_daemon_factory.create() time.sleep(5) - # Create several files across the 2-node cluster so that chunks land on both nodes + # Create several files across the 2-node cluster for i in range(8): - f = d00.mountdir / f"shrink_file_{i}" - ret = gkfs_client.open(f, os.O_CREAT | os.O_WRONLY, + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.open(fpath, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1, f"open failed for {f}" - ret = gkfs_client.write_validate(f, 1024 * 1024) - assert ret.retval == 0, f"write_validate failed for {f}" - - # Build gkfs_hosts_new.txt containing only d00 (remove d01). - # d00.address is "iface:PORT" (e.g. "lo:15680"); extract the port and - # match it against the URI written in the hosts file - # ("ofi+sockets://127.0.0.1:PORT ..."). Each daemon gets a unique - # ephemeral port so this is a reliable discriminator. + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + old_hostfile = Path(d00.hostfile) - new_hostfile = old_hostfile.parent / "gkfs_hosts_new.txt" + + # Marker-based API: mark d01 with '-' prefix (to remove) d00_port = d00.address.split(":")[-1] - with open(old_hostfile) as hf_in, open(new_hostfile, "w") as hf_out: - for line in hf_in: - # Keep lines that belong to d00; skip d01 and comment lines - if not line.startswith("#") and f":{d00_port}" in line: + d01_port = d01.address.split(":")[-1] + + with open(old_hostfile, 'r') as hf_in: + original_lines = hf_in.readlines() + + # Write d00's entry (active, no prefix) and d01's entry with '-' prefix (to remove) + with open(old_hostfile, 'w') as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): hf_out.write(line) - assert new_hostfile.exists(), "New hosts file was not created" - assert new_hostfile.stat().st_size > 0, \ - f"New hosts file is empty; d00 port '{d00_port}' not found in {old_hostfile}" + continue + # Check if this line belongs to d01 + if f":{d01_port}" in stripped: + hf_out.write("-" + line) # Mark for removal + else: + hf_out.write(line) # Keep as active libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] search_path = ":".join(str(p) for p in gkfs_shell._search_paths) malleability_bin = shutil.which("gkfs_malleability", path=search_path) - # Check status before starting (should report nothing running) + # Check status before starting (no --new-hosts-file needed) cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - assert "No shrink running/finished." in cmd.stderr.decode() + assert "No mutate running/finished." in cmd.stderr.decode() - # Start shrink; node counts are auto-detected from both hostfiles + # Start mutate (shrink = fewer nodes) cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} start" + f"{malleability_bin} mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) - assert cmd.exit_code == 0, f"shrink start failed: {cmd.stderr.decode()}" - assert "Shrink process from 2 nodes to 1 nodes launched" in cmd.stderr.decode() + assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" + assert "Mutate process" in cmd.stderr.decode() - # Wait for redistribution to complete (poll with timeout) + # Wait for redistribution (no --new-hosts-file needed) deadline = time.time() + 120 while time.time() < deadline: cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} status" + f"{malleability_bin} mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - if "No shrink running/finished." in cmd.stderr.decode(): + if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: - pytest.fail("Shrink redistribution did not finish within 120 s") + pytest.fail("Mutate redistribution did not finish within 120 s") - # Finalize: gkfs_hosts_new.txt is renamed to gkfs_hosts.txt (old_hostfile) + # Finalize (no --new-hosts-file needed) cmd_str = ( f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} finalize" + f"{malleability_bin} mutate finalize" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"shrink finalize failed: {cmd.stderr.decode()}" - assert "Hosts file updated" in cmd.stderr.decode() - - # Postcondition: new file is gone, old path now holds the new config - assert not new_hostfile.exists(), "gkfs_hosts_new.txt should have been renamed" - assert old_hostfile.exists(), "hosts file should still exist with updated content" + assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" - # Verify all data is still accessible on the surviving node d00 + # Verify all data accessible for i in range(8): - f = d00.mountdir / f"shrink_file_{i}" - ret = gkfs_client.stat(f) - assert ret.retval == 0, f"stat failed for {f} after shrink" + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after mutate" d00.shutdown() d01.shutdown() diff --git a/tests/integration/malleability/test_malleability_tool_simple.py b/tests/integration/malleability/test_malleability_tool_simple.py new file mode 100644 index 000000000..9a817fa35 --- /dev/null +++ b/tests/integration/malleability/test_malleability_tool_simple.py @@ -0,0 +1,306 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Simple, easy-to-verify malleability tool tests. + +These tests cover three scenarios: + 1. ADD - Expand: add 1 node to a 1-node cluster + 2. REMOVE - Shrink: remove 1 node from a 2-node cluster + 3. CHANGE - Mutate: swap 1 node for another (remove + add in one operation) + +Each test is standalone and can be run by the user independently: + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_expand + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_shrink + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_mutate_swap +""" + +import os +import stat +import time +import shutil +from pathlib import Path +import pytest + + +def _get_malleability_bin(gkfs_shell): + """Return the path to the gkfs_malleability binary.""" + libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") + search_path = ":".join(str(p) for p in gkfs_shell._search_paths) + malleability_bin = shutil.which("gkfs_malleability", path=search_path) + assert malleability_bin is not None, "gkfs_malleability not found in PATH" + return malleability_bin + + +def _run_cmd(bin_path, hosts_file, args, gkfs_shell, timeout=None): + """Run a gkfs_malleability command and assert success.""" + libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") + cmd_str = ( + f"LD_LIBRARY_PATH={libdirs} " + f"LIBGKFS_HOSTS_FILE={hosts_file} " + f"{bin_path} {args}" + ) + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + assert cmd.exit_code == 0, ( + f"gkfs_malleability command failed: {cmd_str}\n" + f"stderr: {cmd.stderr.decode()}" + ) + return cmd + + +def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: ADD (expand) + Start with 1 daemon, add a 2nd, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + time.sleep(5) + hostfile = Path(d00.hostfile) + + for i in range(4): + fpath = d00.mountdir / f"expand_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + d01 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) + malleability_bin = _get_malleability_bin(gkfs_shell) + + d01_port = d01.address.split(":")[-1] + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + with open(hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(malleability_bin, hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd( + malleability_bin, hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(malleability_bin, hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(malleability_bin, hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + + for i in range(4): + fpath = d00.mountdir / f"expand_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after expand" + + with open(hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + + d00.shutdown() + d01.shutdown() + + +def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: REMOVE (shrink) + Start with 2 daemons, remove 1, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + time.sleep(5) + + for i in range(8): + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + old_hostfile = Path(d00.hostfile) + malleability_bin = _get_malleability_bin(gkfs_shell) + + d01_port = d01.address.split(":")[-1] + with open(old_hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + with open(old_hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("-" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd( + malleability_bin, old_hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + + for i in range(8): + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after shrink" + + with open(old_hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + assert f":{d01_port}" not in content, f"Removed daemon {d01_port} still in hostfile" + + d00.shutdown() + d01.shutdown() + d00.shutdown() + d01.shutdown() + + +def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: CHANGE (mutate swap) + Start with 2 daemons, remove 1 and add 1, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + time.sleep(5) + + for i in range(4): + fpath = d00.mountdir / f"mutate_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + old_hostfile = Path(d00.hostfile) + + d02 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) + + malleability_bin = _get_malleability_bin(gkfs_shell) + + d01_port = d01.address.split(":")[-1] + d02_port = d02.address.split(":")[-1] + with open(old_hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + with open(old_hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("-" + line) + elif f":{d02_port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd( + malleability_bin, old_hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(malleability_bin, old_hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + + for i in range(4): + fpath = d00.mountdir / f"mutate_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after mutate swap" + + with open(old_hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + assert f":{d01_port}" not in content, f"Removed daemon {d01_port} still in hostfile" + + d00.shutdown() + d01.shutdown() + d02.shutdown() + diff --git a/tests/integration/malleability/test_mutate_distributors_integrity.py b/tests/integration/malleability/test_mutate_distributors_integrity.py new file mode 100644 index 000000000..084ccdc14 --- /dev/null +++ b/tests/integration/malleability/test_mutate_distributors_integrity.py @@ -0,0 +1,263 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Mutate integration tests for distributor correctness. + +The matrix below checks that mutate can add, remove, and replace nodes for both +the default/simple-hash distributor and the random-slicing distributor. Each +case verifies: + +* data written before mutate is still readable and byte-identical; +* new data can be written/read after finalize; +* the hostfile markers are consumed by finalize; +* removed daemons are absent from the finalized hostfile; +* chunks exist after redistribution and are not left only on a removed daemon. +""" + +import hashlib +import os +import shutil +import stat +import time +from pathlib import Path + +import pytest + + +FILE_COUNT = 6 +FILE_SIZE = 64 * 1024 + + +def _get_malleability_bin(gkfs_shell): + search_path = ":".join(str(p) for p in gkfs_shell._search_paths) + malleability_bin = shutil.which("gkfs_malleability", path=search_path) + assert malleability_bin is not None, "gkfs_malleability not found in PATH" + return malleability_bin + + +def _run_mutate_cmd(gkfs_shell, bin_path, hosts_file, args, timeout=120, logdir=None): + libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") + cmd_str = ( + f'LD_LIBRARY_PATH="{libdirs}" ' + f'LIBGKFS_HOSTS_FILE="{hosts_file}" ' + f'{bin_path} {args}' + ) + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + diagnostics = "" + if logdir is not None: + daemon_log = Path(logdir) / "gkfs_daemon.log" + client_log = Path(logdir) / "gkfs_client.log" + diagnostics = ( + f"\nhostfile:\n{Path(hosts_file).read_text() if Path(hosts_file).exists() else ''}" + f"\ndaemon log tail:\n{daemon_log.read_text()[-8000:] if daemon_log.exists() else ''}" + f"\nclient log tail:\n{client_log.read_text()[-8000:] if client_log.exists() else ''}" + ) + assert cmd.exit_code == 0, ( + f"gkfs_malleability {args} failed\n" + f"stdout: {cmd.stdout.decode()}\n" + f"stderr: {cmd.stderr.decode()}" + f"{diagnostics}" + ) + return cmd + + +def _extract_port(daemon): + return daemon.address.rsplit(":", 1)[-1] + + +def _hostfile_entry_for(hostfile, daemon): + port = _extract_port(daemon) + with open(hostfile) as hf: + for line in hf: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + clean = stripped[1:] if stripped[0] in "+-" else stripped + if f":{port}" in clean: + return clean + raise AssertionError(f"No hostfile entry found for daemon {daemon.address}") + + +def _write_workspace(hostfile, active, removing=None, adding=None): + removing = removing or [] + adding = adding or [] + marker = {d: "" for d in active} + marker.update({d: "-" for d in removing}) + marker.update({d: "+" for d in adding}) + + entries = [ + marker[daemon] + _hostfile_entry_for(daemon.hostfile, daemon) + "\n" + for daemon in list(active) + list(removing) + list(adding) + ] + + with open(hostfile, "w") as hf: + hf.writelines(entries) + + +def _chunk_files(rootdir): + root = Path(rootdir) + if not root.exists(): + return set() + return {p.relative_to(root) for p in root.rglob("*") if p.is_file() and p.name.isdigit()} + + +def _write_file(client, path, size): + ret = client.open(path, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1, f"open failed for {path}" + ret = client.write_validate(path, size) + assert ret.retval == 0, f"write_validate failed for {path}" + + payload = bytes((ord("0") + (i % 10)) for i in range(size)) + return hashlib.md5(payload).hexdigest() + + +def _read_md5(client, path, logdir=None, hostfile=None): + ret = client.stat(path) + diagnostics = "" + if logdir is not None: + daemon_log = Path(logdir) / "gkfs_daemon.log" + client_log = Path(logdir) / "gkfs_client.log" + diagnostics = ( + f"\nhostfile:\n{Path(hostfile).read_text() if hostfile and Path(hostfile).exists() else ''}" + f"\ndaemon log tail:\n{daemon_log.read_text()[-12000:] if daemon_log.exists() else ''}" + f"\nclient log tail:\n{client_log.read_text()[-12000:] if client_log.exists() else ''}" + ) + assert ret.retval == 0, f"stat failed for {path}{diagnostics}" + size = ret.statbuf.st_size + + ret = client.open(path, os.O_RDONLY, 0) + assert ret.retval != -1, f"open for read failed for {path}" + ret = client.read(path, size) + assert ret.retval == size, f"read failed for {path}: {ret.retval}/{size}" + assert ret.buf is not None, f"read returned no buffer for {path}" + return hashlib.md5(ret.buf).hexdigest() + + +def _verify_md5_map(client, md5_map, logdir=None, hostfile=None): + for path, expected in md5_map.items(): + assert _read_md5(client, path, logdir, hostfile) == expected, ( + f"MD5 mismatch for {path}" + ) + + +def _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile): + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + if "No mutate running/finished." in cmd.stderr.decode(): + return + time.sleep(2) + pytest.fail("Mutate redistribution did not finish within 120 s") + + +def _assert_final_hostfile(hostfile, removed_daemons): + content = Path(hostfile).read_text() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + for daemon in removed_daemons: + assert f":{_extract_port(daemon)}" not in content, ( + f"Removed daemon {daemon.address} still in finalized hostfile" + ) + + +@pytest.mark.parametrize("strategy", ["simple_hash", "random_slicing"]) +@pytest.mark.parametrize("action", ["add", "remove", "swap"]) +def test_mutate_add_remove_swap_keeps_data_and_chunks_distributed( + strategy, + action, + monkeypatch, + gkfwd_daemon_factory, + gkfs_shell, + request, +): + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", strategy) + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + client = request.getfixturevalue("gkfs_client") + + daemons = [] + try: + if action == "add": + old_daemons = [gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [] + active_daemons = old_daemons + elif action == "remove": + old_daemons = [gkfwd_daemon_factory.create(), gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [old_daemons[1]] + active_daemons = [old_daemons[0]] + else: + old_daemons = [gkfwd_daemon_factory.create(), gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [old_daemons[1]] + active_daemons = [old_daemons[0]] + + daemons = list(old_daemons) + time.sleep(3) + + md5_map = {} + for i in range(FILE_COUNT): + path = old_daemons[0].mountdir / f"{strategy}_{action}_{i:03d}.dat" + md5_map[path] = _write_file(client, path, FILE_SIZE) + + _verify_md5_map(client, md5_map) + + rootdir = old_daemons[0].rootdir + pre_chunks = _chunk_files(rootdir) + assert pre_chunks, "No chunks found before mutate" + + if action == "add": + added_daemons = [gkfwd_daemon_factory.create(expand_mode=True)] + elif action == "swap": + added_daemons = [gkfwd_daemon_factory.create(expand_mode=True)] + daemons.extend(added_daemons) + time.sleep(2) + + hostfile = Path(old_daemons[0].hostfile) + _write_workspace(hostfile, active_daemons, removing=removed_daemons, adding=added_daemons) + + malleability_bin = _get_malleability_bin(gkfs_shell) + status = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + assert "No mutate running/finished." in status.stderr.decode() + + start = _run_mutate_cmd( + gkfs_shell, + malleability_bin, + hostfile, + "mutate start", + timeout=340, + logdir=old_daemons[0].logdir, + ) + assert "Mutate process" in start.stderr.decode() + _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile) + _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate finalize") + + _assert_final_hostfile(hostfile, removed_daemons) + for daemon in removed_daemons: + daemon._proc.wait(timeout=10) + assert daemon._proc.poll() is not None, ( + f"Removed daemon {daemon.address} was not gracefully shut down" + ) + _verify_md5_map(client, md5_map, old_daemons[0].logdir, hostfile) + + post_chunks = _chunk_files(rootdir) + assert post_chunks, "No chunks found after mutate" + assert pre_chunks <= post_chunks, "Some pre-mutate chunks disappeared from backend storage" + + post_md5_map = {} + for i in range(2): + path = old_daemons[0].mountdir / f"{strategy}_{action}_post_{i:03d}.dat" + post_md5_map[path] = _write_file(client, path, FILE_SIZE) + _verify_md5_map(client, post_md5_map) + + finally: + for daemon in daemons: + daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/startup/test_hosts_file_lifecycle.py b/tests/integration/startup/test_hosts_file_lifecycle.py index 21da0826e..a46e24665 100644 --- a/tests/integration/startup/test_hosts_file_lifecycle.py +++ b/tests/integration/startup/test_hosts_file_lifecycle.py @@ -30,7 +30,7 @@ Integration tests for hosts file lifecycle management. Tests that the shared hosts file is correctly destroyed by default on daemon shutdown -and preserved when the --keep-hosts flag or GKFS_KEEP_HOSTS_FILE environment variable is set. +and preserved when the --keep-hosts flag or GKFS_DAEMON_KEEP_HOSTS_FILE environment variable is set. """ import os @@ -62,9 +62,9 @@ def test_hosts_file_destroyed_on_normal_shutdown(gkfwd_daemon_factory, gkfs_shel def test_hosts_file_preserved_with_keep_env(test_workspace, request): - """Test that hosts file is preserved with GKFS_KEEP_HOSTS_FILE=ON.""" + """Test that hosts file is preserved with GKFS_DAEMON_KEEP_HOSTS_FILE=ON.""" d00 = Daemon(request.config.getoption('--interface'), "rocksdb", - test_workspace, env={"GKFS_KEEP_HOSTS_FILE": "ON"}) + test_workspace, env={"GKFS_DAEMON_KEEP_HOSTS_FILE": "ON"}) d00.run() time.sleep(5) @@ -76,7 +76,7 @@ def test_hosts_file_preserved_with_keep_env(test_workspace, request): time.sleep(2) assert hostfile.exists(), \ - "Hosts file should be preserved when GKFS_KEEP_HOSTS_FILE=ON" + "Hosts file should be preserved when GKFS_DAEMON_KEEP_HOSTS_FILE=ON" # Clean up hostfile.unlink() diff --git a/tests/integration/syscalls/test_config_env.py b/tests/integration/syscalls/test_config_env.py index 35c6ebb01..6e670bd1d 100644 --- a/tests/integration/syscalls/test_config_env.py +++ b/tests/integration/syscalls/test_config_env.py @@ -25,7 +25,7 @@ def test_inline_data(test_workspace, request, use_inline): # We use same value for Daemon to be safe (though strictly Client decides to send inline) # 1. Start Daemon with env var # We use same value for Daemon to be safe (though strictly Client decides to send inline) - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": use_inline} + daemon_env = {"GKFS_USE_INLINE_DATA": use_inline} interface = request.config.getoption('--interface') backend = "rocksdb" @@ -36,7 +36,7 @@ def test_inline_data(test_workspace, request, use_inline): try: # 2. Start Client with env var # 2. Start Client with env var - client_env = {"LIBGKFS_USE_INLINE_DATA": use_inline} + client_env = {"GKFS_USE_INLINE_DATA": use_inline} # We need a shell client or similar to execute commands # We can use gkfs.io via Client class which wraps it, or just use shell @@ -82,7 +82,7 @@ def test_dirents_compression(test_workspace, request, use_compression): but we can verify that the system runs and respects the flag in logs. """ - daemon_env = {"GKFS_DAEMON_USE_DIRENTS_COMPRESSION": use_compression} + daemon_env = {"GKFS_USE_DIRENTS_COMPRESSION": use_compression} # specific log level to see configuration output daemon_env["GKFS_DAEMON_LOG_LEVEL"] = "info" @@ -112,7 +112,7 @@ def test_dirents_compression(test_workspace, request, use_compression): # Client side verification # Client side verification - client_env = {"LIBGKFS_USE_DIRENTS_COMPRESSION": use_compression} + client_env = {"GKFS_USE_DIRENTS_COMPRESSION": use_compression} # Just run a simple ls to trigger dirents client = Client(test_workspace) diff --git a/tests/integration/syscalls/test_env_features.py b/tests/integration/syscalls/test_env_features.py index 9f7515dce..e81559b88 100644 --- a/tests/integration/syscalls/test_env_features.py +++ b/tests/integration/syscalls/test_env_features.py @@ -4,14 +4,14 @@ import os from harness.gkfs import Daemon, Client, ShellClient @pytest.mark.parametrize("opt_env", [ - {"LIBGKFS_CREATE_WRITE_OPTIMIZATION": "ON", "LIBGKFS_USE_INLINE_DATA": "ON"} + {"LIBGKFS_CREATE_WRITE_OPTIMIZATION": "ON", "GKFS_USE_INLINE_DATA": "ON"} ]) def test_create_write_optimization(test_workspace, request, opt_env): """ Test CREATE_WRITE_OPTIMIZATION. Should trigger 'forward_create_write_inline' in forward_metadata.cpp. """ - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": "ON"} + daemon_env = {"GKFS_USE_INLINE_DATA": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() @@ -40,14 +40,14 @@ def test_create_write_optimization(test_workspace, request, opt_env): daemon.shutdown() @pytest.mark.parametrize("prefetch_env", [ - {"LIBGKFS_READ_INLINE_PREFETCH": "ON", "LIBGKFS_USE_INLINE_DATA": "ON"} + {"LIBGKFS_READ_INLINE_PREFETCH": "ON", "GKFS_USE_INLINE_DATA": "ON"} ]) def test_read_inline_prefetch(test_workspace, request, prefetch_env): """ Test READ_INLINE_PREFETCH. Should trigger 'forward_stat' with include_inline=true in forward_metadata.cpp. """ - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": "ON"} + daemon_env = {"GKFS_USE_INLINE_DATA": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() @@ -56,7 +56,7 @@ def test_read_inline_prefetch(test_workspace, request, prefetch_env): file_path = test_workspace.mountdir / "prefetch_file" # 1. Create file with inline data (using standard write, ensuring inline is used) - create_env = {"LIBGKFS_USE_INLINE_DATA": "ON"} + create_env = {"GKFS_USE_INLINE_DATA": "ON"} ret = client.run("write_sequential", "--pathname", str(file_path), "--count", "1", @@ -73,14 +73,14 @@ def test_read_inline_prefetch(test_workspace, request, prefetch_env): daemon.shutdown() @pytest.mark.parametrize("compress_env", [ - {"LIBGKFS_USE_DIRENTS_COMPRESSION": "ON"} + {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} ]) def test_dirents_compression_large(test_workspace, request, compress_env): """ Test DIRENTS_COMPRESSION with enough entries to trigger compression logic. Should trigger 'decompress_and_parse_entries_standard' with compression path. """ - daemon_env = {"GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "ON"} + daemon_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index c30f2c127..fe7d7284a 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -61,7 +61,8 @@ target_sources(unit_tests ${CMAKE_CURRENT_LIST_DIR}/test_helpers.cpp ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_distributor.cpp ${CMAKE_CURRENT_LIST_DIR}/test_distributor_factory_and_migrator.cpp - ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_pipeline.cpp) + ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_pipeline.cpp + ${CMAKE_SOURCE_DIR}/src/common/malleability_markers.cpp) if (GKFS_TESTS_GUIDED_DISTRIBUTION) target_sources(unit_tests PRIVATE ${CMAKE_CURRENT_LIST_DIR}/test_guided_distributor.cpp) diff --git a/tests/unit/test_distributor.cpp b/tests/unit/test_distributor.cpp index f7e69150c..ee07002ae 100644 --- a/tests/unit/test_distributor.cpp +++ b/tests/unit/test_distributor.cpp @@ -51,6 +51,22 @@ TEST_CASE("SimpleHashDistributor", "[common][distributor]") { REQUIRE(d.locate_data("/foo", 0, 0) == c1); } +TEST_CASE("SimpleHashDistributor hosts_size updates directory targets", + "[common][distributor][malleability]") { + auto d = gkfs::rpc::SimpleHashDistributor(0, 4); + REQUIRE(d.locate_directory_metadata().size() == 4); + + d.hosts_size(2); + + auto targets = d.locate_directory_metadata(); + REQUIRE(targets.size() == 2); + REQUIRE(targets[0] == 0); + REQUIRE(targets[1] == 1); + REQUIRE(d.hosts_size() == 2); + REQUIRE(d.locate_file_metadata("/foo", 0) < 2); + REQUIRE(d.locate_data("/foo", 0, 0) < 2); +} + TEST_CASE("LocalOnlyDistributor", "[common][distributor]") { auto d = gkfs::rpc::LocalOnlyDistributor(5); REQUIRE(d.localhost() == 5); diff --git a/tests/unit/test_random_slicing_distributor.cpp b/tests/unit/test_random_slicing_distributor.cpp index 544ee2706..e002cb561 100644 --- a/tests/unit/test_random_slicing_distributor.cpp +++ b/tests/unit/test_random_slicing_distributor.cpp @@ -24,7 +24,9 @@ */ #include +#include #include +#include #include #include @@ -157,7 +159,7 @@ TEST_CASE("RandomSlicingDistributor reconfigure rebuilds from all_hosts", "[comm } // ===== Interval table persistence tests ===== -// ponytail: persistence is intentionally disabled — intervals are rebuilt from +// persistence is intentionally disabled — intervals are rebuilt from // the live host list on each daemon/proxy startup. These tests verify the stubs. TEST_CASE("RandomSlicingDistributor save_interval_table is stubbed", "[common][distributor][random_slicing]") { @@ -172,6 +174,73 @@ TEST_CASE("RandomSlicingDistributor save_interval_table is stubbed", "[common][d REQUIRE(!d2.load_interval_table(path)); } +TEST_CASE("RandomSlicingDistributor installs explicit interval table", + "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 3); + + std::vector intervals = {{0.0f, 0.25f, 2}, + {0.25f, 0.75f, 0}, + {0.75f, 1.0f, 1}}; + REQUIRE(d.set_intervals(intervals)); + auto got = d.get_intervals(); + REQUIRE(got.size() == intervals.size()); + REQUIRE(got[0].host_id == 2); + REQUIRE(got[0].start == Catch::Approx(0.0f)); + REQUIRE(got[0].end == Catch::Approx(0.25f)); + REQUIRE(got[1].host_id == 0); + REQUIRE(got[1].start == Catch::Approx(0.25f)); + REQUIRE(got[1].end == Catch::Approx(0.75f)); + REQUIRE(got[2].host_id == 1); + REQUIRE(got[2].start == Catch::Approx(0.75f)); + REQUIRE(got[2].end == Catch::Approx(1.0f)); + + REQUIRE_FALSE(d.set_intervals({{0.0f, 0.4f, 0}, {0.5f, 1.0f, 1}})); + REQUIRE_FALSE(d.set_intervals({{0.0f, 1.0f, 3}})); +} + +TEST_CASE("RandomSlicing interval comments round-trip through hostfile", + "[common][malleability][random_slicing]") { + const auto path = (std::filesystem::temp_directory_path() / + "gkfs_rs_interval_comments_test.hosts") + .string(); + { + std::ofstream out(path, std::ios::trunc); + REQUIRE(out.is_open()); + out << "node0 uri0 extra\n"; + out << "+ node1 uri1 extra\n"; + out << "# regular comment stays\n"; + out << "# GKFS_RS_INTERVAL host=99 start=0.0 end=1.0\n"; + } + + std::vector intervals = { + {0, 0.0, 0.5}, {1, 0.5, 1.0}}; + gkfs::malleable::write_rs_interval_comments(path, intervals); + + auto parsed = gkfs::malleable::parse_rs_interval_comments(path); + REQUIRE(parsed.size() == 2); + REQUIRE(parsed[0].host_id == 0); + REQUIRE(parsed[0].start == Catch::Approx(0.0)); + REQUIRE(parsed[0].end == Catch::Approx(0.5)); + REQUIRE(parsed[1].host_id == 1); + REQUIRE(parsed[1].start == Catch::Approx(0.5)); + REQUIRE(parsed[1].end == Catch::Approx(1.0)); + + auto markers = gkfs::malleable::parse_hostfile_markers(path); + REQUIRE(markers.active.size() == 1); + REQUIRE(markers.adding.size() == 1); + gkfs::malleable::write_clean_hostfile(path, markers, parsed); + + auto after_clean_markers = gkfs::malleable::parse_hostfile_markers(path); + REQUIRE(after_clean_markers.active.size() == 2); + REQUIRE(after_clean_markers.adding.empty()); + auto after_clean_intervals = gkfs::malleable::parse_rs_interval_comments(path); + REQUIRE(after_clean_intervals.size() == 2); + REQUIRE(after_clean_intervals[0].host_id == 0); + REQUIRE(after_clean_intervals[1].host_id == 1); + + std::filesystem::remove(path); +} + // ===== Compare with SimpleHash: deterministic across same config ===== TEST_CASE("RandomSlicingDistributor produces different mapping than SimpleHash", "[common][distributor][random_slicing]") { diff --git a/tests/unit/test_random_slicing_pipeline.cpp b/tests/unit/test_random_slicing_pipeline.cpp index 8375b57d6..063dd74d6 100644 --- a/tests/unit/test_random_slicing_pipeline.cpp +++ b/tests/unit/test_random_slicing_pipeline.cpp @@ -26,8 +26,10 @@ #include #include #include +#include #include #include +#include using namespace gkfs::rpc; @@ -49,7 +51,7 @@ TEST_CASE("DistributionConfig set_strategy from string", "[config][distribution] } TEST_CASE("DistributionConfig read_strategy_from_env returns default when unset", "[config][distribution]") { - // ponytail: Don't actually modify env - test that default is returned + // Don't actually modify env - test that default is returned auto strategy = read_strategy_from_env(); REQUIRE(strategy == DistributionStrategy::SimpleHash); } @@ -58,6 +60,8 @@ TEST_CASE("DistributionConfig read_strategy_from_env returns default when unset" TEST_CASE("Pipeline: factory creates RandomSlicing, add_nodes triggers migration estimation", "[pipeline][random_slicing]") { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + auto rs = create_distributor_from_string("random_slicing", 0, 3); REQUIRE(rs != nullptr); REQUIRE(rs->localhost() == 0); @@ -76,10 +80,46 @@ TEST_CASE("Pipeline: factory creates RandomSlicing, add_nodes triggers migration REQUIRE(new_partitions.size() == 6); } +TEST_CASE("Pipeline: RandomSlicing add_nodes uses CutShift when env is enabled", + "[pipeline][random_slicing][cutshift]") { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + auto default_rs = create_distributor_from_string("random_slicing", 0, 3); + auto* default_dist = + dynamic_cast(default_rs.get()); + REQUIRE(default_dist != nullptr); + default_dist->add_nodes({3}); + auto default_partitions = default_dist->get_partitions_copy(); + REQUIRE(default_partitions.size() == 4); + REQUIRE(default_partitions[1].intervals.size() == 1); + REQUIRE(default_partitions[1].intervals[0].start == + Catch::Approx(0.25f)); + + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + auto cutshift_rs = create_distributor_from_string("random_slicing", 0, 3); + auto* cutshift_dist = + dynamic_cast(cutshift_rs.get()); + REQUIRE(cutshift_dist != nullptr); + cutshift_dist->add_nodes({3}); + auto cutshift_partitions = cutshift_dist->get_partitions_copy(); + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + + REQUIRE(cutshift_partitions.size() == 4); + REQUIRE(cutshift_partitions[1].intervals.size() == 1); + REQUIRE(cutshift_partitions[1].intervals[0].start == + Catch::Approx(1.0f / 3.0f)); + REQUIRE(cutshift_partitions[1].intervals[0].end == + Catch::Approx(7.0f / 12.0f)); + REQUIRE(cutshift_partitions[3].host_id == 3); + REQUIRE(!cutshift_partitions[3].intervals.empty()); + for(const auto& interval : cutshift_partitions[3].intervals) { + REQUIRE(interval.host_id == 3); + } +} + TEST_CASE("Pipeline: CutShift+Sorted produces valid partition updates", "[pipeline][cutshift][random_slicing]") { - auto* dist = dynamic_cast( - create_distributor_from_string("random_slicing", 0, 3).get()); + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); REQUIRE(dist != nullptr); auto old_partitions = dist->get_partitions_copy(); @@ -98,13 +138,51 @@ TEST_CASE("Pipeline: CutShift+Sorted produces valid partition updates", for (size_t i = 3; i < 5; ++i) { REQUIRE(new_partitions[i].total_capacity > 0); REQUIRE(new_partitions[i].host_id == new_hosts[i - 3]); + REQUIRE(!new_partitions[i].intervals.empty()); + for (const auto& interval : new_partitions[i].intervals) { + REQUIRE(interval.host_id == new_partitions[i].host_id); + REQUIRE(interval.start < interval.end); + } + } +} + +TEST_CASE("Pipeline: CutShift+Sorted preserves full coverage without overlap", + "[pipeline][cutshift][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 4); + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + std::vector new_hosts = {4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + std::vector intervals; + for (const auto& partition : new_partitions) { + for (const auto& interval : partition.intervals) { + REQUIRE(interval.start >= 0.0f); + REQUIRE(interval.end <= 1.0f); + REQUIRE(interval.start < interval.end); + REQUIRE(interval.host_id == partition.host_id); + intervals.push_back(interval); + } + } + + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + + REQUIRE(!intervals.empty()); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for (size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i - 1].end <= intervals[i].start + 0.0001f); } } TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", "[pipeline][migrator][random_slicing]") { - auto* dist = dynamic_cast( - create_distributor_from_string("random_slicing", 0, 3).get()); + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); REQUIRE(dist != nullptr); auto old_partitions = dist->get_partitions_copy(); @@ -121,8 +199,8 @@ TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", TEST_CASE("Pipeline: full expansion cycle", "[pipeline][random_slicing]") { - auto* dist = dynamic_cast( - create_distributor_from_string("random_slicing", 0, 3).get()); + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); REQUIRE(dist != nullptr); auto old_partitions = dist->get_partitions_copy(); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 5283f37e2..e137179d8 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -41,5 +41,13 @@ add_executable(gkfs_malleability malleability.cpp) target_link_libraries(gkfs_malleability PUBLIC gkfs_user_lib + gkfs_common CLI11::CLI11) -install(TARGETS gkfs_malleability RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) \ No newline at end of file +install(TARGETS gkfs_malleability RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +add_executable(gkfs_malleability_sim malleability_simulator.cpp) +target_link_libraries(gkfs_malleability_sim + PUBLIC + distributor + CLI11::CLI11) +install(TARGETS gkfs_malleability_sim RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) \ No newline at end of file diff --git a/tools/malleability.cpp b/tools/malleability.cpp index aaccf55b3..3aab1d2eb 100644 --- a/tools/malleability.cpp +++ b/tools/malleability.cpp @@ -1,13 +1,70 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This software was partially supported by the + the European Union's Horizon 2020 JTI-EuroHPC research and + innovation programme, by the project ADMIRE (Project ID: 956748, + admire-eurohpc.eu) + + This project was partially promoted by the Ministry for Digital Transformation + and the Civil Service, within the framework of the Recovery, + Transformation and Resilience Plan - Funded by the European Union + -NextGenerationEU. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/** + * @brief Malleability CLI tool for GekkoFS cluster topology changes. + * + * This tool uses a marker-based single-hostfile approach: + * - Normal lines (no prefix) = active daemons + * - Lines with '-' prefix = to-be-removed daemons (user writes manually) + * - Lines with '+' prefix = to-be-added daemons (auto-written by daemons with GKFS_DAEMON_EXPAND=ON) + * + * The malleability CLI auto-discovers before/after state from markers in LIBGKFS_HOSTS_FILE. + * + * Usage: + * gkfs_malleability mutate start # Auto-discovers from LIBGKFS_HOSTS_FILE markers + * gkfs_malleability mutate status # Shows current mutate status + * gkfs_malleability mutate finalize # Rewrites hostfile clean, stops removed daemons + */ + #include #include #include #include +#include #include +#include +#include #include #include #include - +#include using namespace std; namespace fs = std::filesystem; @@ -16,55 +73,15 @@ struct cli_options { bool verbose = false; bool machine_readable = false; string action; - string subcommand; + bool show_status = false; }; -std::pair -get_expansion_host_num() { - // get hosts file and read how much should be expanded - auto hosts_file_path = std::getenv("LIBGKFS_HOSTS_FILE"); - if(!hosts_file_path) { - std::cerr - << "Error: LIBGKFS_HOSTS_FILE environment variable not set.\n"; - return {-1, -1}; - } - std::ifstream file(hosts_file_path); - if(!file) { - std::cerr << "Error: Unable to open file at " << hosts_file_path - << ".\n"; - return {-1, -1}; // Indicate an error - } - auto initialHostCount = 0; - auto finalHostCount = 0; - auto foundSeparator = false; - std::string line; - - while(std::getline(file, line)) { - if(line == "#FS_INSTANCE_END") { - if(foundSeparator) { - cerr << "marker was found twice. this is not allowed.\n"; - return {-1, -1}; - } - foundSeparator = true; - initialHostCount = finalHostCount; - continue; - } - if(!line.empty()) { - finalHostCount++; - } - } - if(!foundSeparator) { - initialHostCount = finalHostCount; - } - return {initialHostCount, finalHostCount}; -} - /** - * Count non-empty, non-comment lines in a hosts file. + * Count non-empty, non-comment, non-marker lines in a hosts file. * Returns -1 on error. */ int -count_hosts_in_file(const std::string& path) { +count_active_hosts_in_file(const std::string& path) { std::ifstream f(path); if(!f) { cerr << "Error: Unable to open file at " << path << ".\n"; @@ -73,15 +90,41 @@ count_hosts_in_file(const std::string& path) { int count = 0; std::string line; while(std::getline(f, line)) { - if(!line.empty() && line[0] != '#') - count++; + if(line.empty()) continue; + if(line[0] == '#') continue; + if(gkfs::malleable::is_marker_line(line)) continue; + count++; } return count; } +/** + * Extract the hostname (first field) from each non-comment line of a hosts file. + * Returns a set of hostnames for comparison. + */ +std::set +extract_hosts_in_file(const std::string& path) { + std::set hosts; + std::ifstream f(path); + if(!f) { + return hosts; + } + std::string line; + while(std::getline(f, line)) { + if(line.empty()) continue; + if(line[0] == '#') continue; + std::istringstream iss(line); + std::string host; + if(std::getline(iss, host, ' ')) { + hosts.insert(host); + } + } + return hosts; +} + int main(int argc, const char* argv[]) { - CLI::App desc{"Allowed options"}; + CLI::App desc{"GekkoFS Malleability Tool - Cluster topology changes"}; cli_options opts; // Global verbose flag @@ -90,31 +133,21 @@ main(int argc, const char* argv[]) { "machine-readable output"); - auto expand_args = - desc.add_subcommand("expand", "Expansion-related actions"); - expand_args->add_option("action", opts.action, "Action to perform") - ->required() - ->check(CLI::IsMember({"start", "status", "finalize"})); - + // Shared variables for mutate subcommand int old_nodes = -1; int new_nodes = -1; - string new_hosts_file; - auto shrink_args = - desc.add_subcommand("shrink", "Shrink-related actions"); - shrink_args->add_option("action", opts.action, "Action to perform") + + auto mutate_args = + desc.add_subcommand("mutate", "Mutate cluster topology (migrate nodes)"); + mutate_args->add_option("action", opts.action, "Action to perform") ->required() ->check(CLI::IsMember({"start", "status", "finalize"})); - shrink_args - ->add_option("--new-hosts-file", new_hosts_file, - "Path to new hosts file listing only the surviving " - "nodes (e.g. gkfs_hosts_new.txt)") - ->envname("LIBGKFS_HOSTS_FILE_NEW"); - shrink_args->add_option("--old-nodes", old_nodes, + mutate_args->add_option("--old-nodes", old_nodes, "Old number of nodes (auto-detected from " "LIBGKFS_HOSTS_FILE if not set)"); - shrink_args->add_option("--new-nodes", new_nodes, + mutate_args->add_option("--new-nodes", new_nodes, "New number of nodes (auto-detected from " - "--new-hosts-file if not set)"); + "LIBGKFS_HOSTS_FILE markers if not set)"); try { desc.parse(argc, argv); @@ -128,120 +161,137 @@ main(int argc, const char* argv[]) { int res; gkfs_init(); - if(expand_args->parsed()) { + if(mutate_args->parsed()) { if(opts.action == "start") { - auto [current_instance, expanded_instance] = - get_expansion_host_num(); - if(current_instance == -1 || expanded_instance == -1) { + // Auto-detect old_nodes and new_nodes from LIBGKFS_HOSTS_FILE markers + auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); + if(!hf) { + cerr << "Error: LIBGKFS_HOSTS_FILE is not set.\n"; return 1; } - res = gkfs::malleable::expand_start(current_instance, - expanded_instance); - if(res) { - cout << "Expand start failed. Exiting...\n"; - gkfs_end(); - cout.flush(); - return -1; - } else { - cerr << "Expansion process from " << current_instance - << " nodes to " << expanded_instance - << " nodes launched...\n"; - } - } else if(opts.action == "status") { - res = gkfs::malleable::expand_status(); - if(res > 0) { - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "Expansion in progress: " << res - << " nodes not finished.\n"; - } - } else { - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "No expansion running/finished.\n"; - } - } - } else if(opts.action == "finalize") { - res = gkfs::malleable::expand_finalize(); - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "Expand finalize " << res << endl; - } - } - } else if(shrink_args->parsed()) { - if(opts.action == "start") { - // new_hosts_file is required for start - if(new_hosts_file.empty()) { - cerr << "Error: --new-hosts-file (or LIBGKFS_HOSTS_FILE_NEW) " - "is required for shrink start.\n"; - return 1; - } - // Auto-detect old_nodes from LIBGKFS_HOSTS_FILE if not given + string workspace_hostfile(hf); + if(old_nodes == -1) { - auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); - if(!hf) { - cerr << "Error: --old-nodes not set and LIBGKFS_HOSTS_FILE " - "is not available.\n"; + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + // old_nodes = active + removing (before state: all nodes before shrink) + old_nodes = static_cast(markers.active.size() + markers.removing.size()); + if(old_nodes <= 0) { + cerr << "Error: Could not determine old node count from '" + << workspace_hostfile << "'. At least one active node is required.\n"; return 1; } - old_nodes = count_hosts_in_file(hf); - if(old_nodes < 0) - return 1; } - // Auto-detect new_nodes from new_hosts_file if not given if(new_nodes == -1) { - new_nodes = count_hosts_in_file(new_hosts_file); - if(new_nodes < 0) + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + new_nodes = static_cast(markers.active.size() + markers.adding.size()); + if(new_nodes <= 0) { + cerr << "Error: Could not determine new node count from '" + << workspace_hostfile << "'.\n"; + return 1; + } + } + + if(old_nodes == new_nodes) { + // Same count: check if host identities actually differ + // For marker-based workflow, we need markers present + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + if(!markers.has_markers()) { + cerr << "Error: Same node count (" << old_nodes << ") with no markers.\n" + << "Add '-' markers for nodes to remove or '+' markers for nodes to add.\n"; return 1; + } + if(markers.active.empty() && !markers.removing.empty() && + !markers.adding.empty()) { + cerr << "Note: Same node count (" << old_nodes + << ") with full replacement via '-' and '+' markers — proceeding with mutate.\n"; + } else { + auto old_hosts = extract_hosts_in_file(workspace_hostfile); + auto removing_hosts = std::set(); + std::ifstream f(workspace_hostfile); + string line; + while(std::getline(f, line)) { + if(gkfs::malleable::is_marker_line(line) && + gkfs::malleable::get_marker(line) == + gkfs::malleable::MARKER_REMOVE) { + std::istringstream iss( + gkfs::malleable::strip_marker(line)); + std::string host; + if(std::getline(iss, host, ' ')) { + removing_hosts.insert(host); + } + } + } + // Remove removing from old to get remaining + for(const auto& h : removing_hosts) { + old_hosts.erase(h); + } + if(old_hosts.empty()) { + cerr << "Warning: All hosts would be removed — aborting.\n"; + return 1; + } + if(old_hosts.size() == + extract_hosts_in_file(workspace_hostfile).size()) { + cerr << "Warning: No effective change detected — proceeding with mutate.\n"; + } else { + cerr << "Note: Same node count (" << old_nodes + << ") with different hosts via markers — proceeding with mutate.\n"; + } + } } - res = gkfs::malleable::shrink_start(old_nodes, new_nodes, - new_hosts_file); + + // For mutate, we use the workspace hostfile directly (no --new-hosts-file needed) + // The daemon parses markers to determine before/after state + res = gkfs::malleable::mutate_start(old_nodes, new_nodes, + workspace_hostfile); if(res) { - cout << "Shrink start failed. Exiting...\n"; + cout << "Mutate start failed. Exiting...\n"; gkfs_end(); cout.flush(); - return -1; + return 1; } else { - cerr << "Shrink process from " << old_nodes << " nodes to " - << new_nodes << " nodes launched...\n"; + cerr << "Mutate process from " << old_nodes + << " nodes to " << new_nodes + << " nodes launched (via LIBGKFS_HOSTS_FILE markers)...\n"; } } else if(opts.action == "status") { - res = gkfs::malleable::shrink_status(); + res = gkfs::malleable::mutate_status(); if(res > 0) { if(opts.machine_readable) { cerr << res; } else { - cerr << "Shrink in progress: " << res + cerr << "Mutate in progress: " << res << " nodes not finished.\n"; } } else { if(opts.machine_readable) { cerr << res; } else { - cerr << "No shrink running/finished.\n"; + cerr << "No mutate running/finished.\n"; } } } else if(opts.action == "finalize") { - res = gkfs::malleable::shrink_finalize(); + res = gkfs::malleable::mutate_finalize(); if(opts.machine_readable) { cerr << res; } else { - cerr << "Shrink finalize " << res << endl; + cerr << "Mutate finalize " << res << endl; } - // On success: move gkfs_hosts_new.txt -> gkfs_hosts.txt - if(res == 0 && !new_hosts_file.empty()) { + // On success: rewrite the workspace hostfile clean + if(res == 0) { auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); if(hf) { + // Parse markers to determine after state + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + auto rs_intervals = + gkfs::malleable::parse_rs_interval_comments(hf); + // Write clean hostfile (promote adding, remove removing) try { - fs::rename(new_hosts_file, hf); - cerr << "Hosts file updated: " << new_hosts_file - << " -> " << hf << "\n"; - } catch(const fs::filesystem_error& e) { - cerr << "Warning: Failed to rename hosts file: " - << e.what() << "\n"; + gkfs::malleable::write_clean_hostfile(hf, markers, + rs_intervals); + cerr << "Hosts file cleaned: " << hf + << " (removed markers, promoted adding to active)\n"; + } catch(const std::exception& e) { + cerr << "Warning: Failed to clean hosts file: " << e.what() << "\n"; } } } diff --git a/tools/malleability_simulator.cpp b/tools/malleability_simulator.cpp new file mode 100644 index 000000000..b71b184d9 --- /dev/null +++ b/tools/malleability_simulator.cpp @@ -0,0 +1,277 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using gkfs::rpc::chunkid_t; +using gkfs::rpc::Distributor; +using gkfs::rpc::host_t; +using gkfs::rpc::RandomSlicingDistributor; +using gkfs::rpc::SimpleHashDistributor; + +struct simulator_config { + unsigned int old_nodes = 4; + unsigned int new_nodes = 2; + std::string strategy = "simple_hash"; + bool cutshift = false; + uint64_t file_size = 1024 * 1024; + uint64_t chunk_size = gkfs::config::rpc::chunksize; + uint64_t files = 1; +}; + +struct host_stats { + uint64_t chunks = 0; + uint64_t bytes = 0; +}; + +struct simulation_result { + std::string strategy; + bool cutshift = false; + unsigned int old_nodes = 0; + unsigned int new_nodes = 0; + uint64_t files = 0; + uint64_t file_size = 0; + uint64_t chunk_size = 0; + uint64_t total_chunks = 0; + uint64_t total_bytes = 0; + uint64_t moved_chunks = 0; + uint64_t moved_bytes = 0; + double movement_ratio = 0.0; + double elapsed_ms = 0.0; + std::vector moved_from; + std::vector received_by; +}; + +std::unique_ptr +make_distributor(const std::string& strategy, unsigned int hosts) { + if(strategy == "simple_hash") { + return std::make_unique(0, hosts); + } + if(strategy == "random_slicing") { + return std::make_unique(0, hosts); + } + throw std::runtime_error("unsupported strategy: " + strategy); +} + +std::unique_ptr +make_final_distributor(const simulator_config& cfg) { + if(cfg.strategy == "simple_hash") { + return std::make_unique(0, cfg.new_nodes); + } + + if(cfg.strategy == "random_slicing") { + if(cfg.cutshift) { + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + } else { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + } + + auto rs = std::make_unique(0, cfg.old_nodes); + if(cfg.new_nodes > cfg.old_nodes) { + std::vector added; + for(unsigned int h = cfg.old_nodes; h < cfg.new_nodes; ++h) { + added.push_back(h); + } + rs->add_nodes(std::move(added)); + } else if(cfg.new_nodes < cfg.old_nodes) { + std::vector removed; + for(unsigned int h = cfg.new_nodes; h < cfg.old_nodes; ++h) { + removed.push_back(h); + } + rs->remove_nodes(std::move(removed)); + } + return rs; + } + + throw std::runtime_error("unsupported strategy: " + cfg.strategy); +} + +uint64_t +chunk_bytes(uint64_t file_size, uint64_t chunk_size, uint64_t chunk_id) { + const auto offset = chunk_id * chunk_size; + const auto remaining = file_size - offset; + return std::min(chunk_size, remaining); +} + +void +print_host_stats(const std::string& title, + const std::vector& stats) { + std::cout << title << "\n"; + for(size_t i = 0; i < stats.size(); ++i) { + std::cout << " host " << i << ": chunks=" << stats[i].chunks + << " bytes=" << stats[i].bytes << "\n"; + } +} + +simulation_result +simulate(simulator_config cfg) { + const auto old_dist = make_distributor(cfg.strategy, cfg.old_nodes); + const auto final_dist = make_final_distributor(cfg); + const auto chunks_per_file = + (cfg.file_size + cfg.chunk_size - 1) / cfg.chunk_size; + + simulation_result result; + result.strategy = cfg.strategy; + result.cutshift = cfg.cutshift; + result.old_nodes = cfg.old_nodes; + result.new_nodes = cfg.new_nodes; + result.files = cfg.files; + result.file_size = cfg.file_size; + result.chunk_size = cfg.chunk_size; + result.total_chunks = chunks_per_file * cfg.files; + result.total_bytes = cfg.file_size * cfg.files; + result.moved_from.resize(cfg.old_nodes); + result.received_by.resize(cfg.new_nodes); + + const auto t0 = std::chrono::steady_clock::now(); + for(uint64_t file = 0; file < cfg.files; ++file) { + const auto path = "/malleability-sim/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < chunks_per_file; ++chunk) { + const auto chunk_id = static_cast(chunk); + const auto old_host = old_dist->locate_data(path, chunk_id, 0); + const auto new_host = final_dist->locate_data(path, chunk_id, 0); + if(old_host == new_host) { + continue; + } + + const auto bytes = + chunk_bytes(cfg.file_size, cfg.chunk_size, chunk); + ++result.moved_chunks; + result.moved_bytes += bytes; + result.moved_from[old_host].chunks++; + result.moved_from[old_host].bytes += bytes; + result.received_by[new_host].chunks++; + result.received_by[new_host].bytes += bytes; + } + } + const auto t1 = std::chrono::steady_clock::now(); + result.elapsed_ms = + std::chrono::duration(t1 - t0).count(); + result.movement_ratio = + result.total_bytes == 0 + ? 0.0 + : static_cast(result.moved_bytes) / + static_cast(result.total_bytes); + return result; +} + +void +print_single_result(const simulation_result& result) { + std::cout << "strategy=" << result.strategy << "\n"; + std::cout << "cutshift=" << (result.cutshift ? "ON" : "OFF") << "\n"; + std::cout << "old_nodes=" << result.old_nodes << "\n"; + std::cout << "new_nodes=" << result.new_nodes << "\n"; + std::cout << "files=" << result.files << "\n"; + std::cout << "file_size=" << result.file_size << "\n"; + std::cout << "chunk_size=" << result.chunk_size << "\n"; + std::cout << "total_chunks=" << result.total_chunks << "\n"; + std::cout << "total_bytes=" << result.total_bytes << "\n"; + std::cout << "moved_chunks=" << result.moved_chunks << "\n"; + std::cout << "moved_bytes=" << result.moved_bytes << "\n"; + std::cout << std::fixed << std::setprecision(6) << "movement_ratio=" + << result.movement_ratio << "\n"; + std::cout << "planning_time_ms=" << result.elapsed_ms << "\n"; + print_host_stats("per_source_moved", result.moved_from); + print_host_stats("per_target_received", result.received_by); +} + +void +print_matrix(const std::vector& results) { + std::cout << "strategy comparison\n"; + std::cout << "old_nodes=" << results.front().old_nodes + << " new_nodes=" << results.front().new_nodes + << " files=" << results.front().files + << " file_size=" << results.front().file_size + << " chunk_size=" << results.front().chunk_size << "\n\n"; + + std::cout << std::left << std::setw(18) << "strategy" << std::setw(10) + << "cutshift" << std::right << std::setw(14) << "moved_bytes" + << std::setw(14) << "moved_mib" << std::setw(14) + << "moved_chunks" << std::setw(12) << "ratio_%" + << std::setw(12) << "time_ms" << "\n"; + std::cout << std::string(94, '-') << "\n"; + + for(const auto& result : results) { + const auto moved_mib = static_cast(result.moved_bytes) / + (1024.0 * 1024.0); + std::cout << std::left << std::setw(18) << result.strategy + << std::setw(10) << (result.cutshift ? "ON" : "OFF") + << std::right << std::setw(14) << result.moved_bytes + << std::setw(14) << std::fixed << std::setprecision(3) + << moved_mib << std::setw(14) << result.moved_chunks + << std::setw(12) << std::fixed << std::setprecision(3) + << (result.movement_ratio * 100.0) << std::setw(12) + << std::fixed << std::setprecision(3) << result.elapsed_ms + << "\n"; + } +} + +} // namespace + +int +main(int argc, char* argv[]) { + simulator_config cfg; + + CLI::App app{"GekkoFS theoretical malleability movement simulator"}; + app.add_option("--old-nodes", cfg.old_nodes, "node count before mutation") + ->check(CLI::PositiveNumber); + app.add_option("--new-nodes", cfg.new_nodes, "node count after mutation") + ->check(CLI::PositiveNumber); + auto strategy_opt = app.add_option( + "--strategy", cfg.strategy, + "distribution strategy: simple_hash|random_slicing. If omitted, run comparison matrix") + ->check(CLI::IsMember({"simple_hash", "random_slicing"})); + app.add_flag("--cutshift", cfg.cutshift, + "use Random Slicing CutShift for expand simulations"); + app.add_option("--file-size", cfg.file_size, "bytes per file") + ->check(CLI::PositiveNumber); + app.add_option("--chunk-size", cfg.chunk_size, "bytes per chunk") + ->check(CLI::PositiveNumber); + app.add_option("--files", cfg.files, "number of synthetic files") + ->check(CLI::PositiveNumber); + + CLI11_PARSE(app, argc, argv); + + if(strategy_opt->count() == 0) { + std::vector results; + cfg.strategy = "simple_hash"; + cfg.cutshift = false; + results.push_back(simulate(cfg)); + + cfg.strategy = "random_slicing"; + cfg.cutshift = false; + results.push_back(simulate(cfg)); + + cfg.cutshift = true; + results.push_back(simulate(cfg)); + + print_matrix(results); + return 0; + } + + print_single_result(simulate(cfg)); + + return 0; +} -- GitLab From 012106dabe2eb4d1c9bc49bb3f7951ca31a4dd86 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 27 Aug 2026 18:05:22 +0200 Subject: [PATCH 06/21] Use gkfs.io MD5 helpers in malleability perf tests --- tests/integration/harness/CMakeLists.txt | 3 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + .../harness/gkfs.io/random_md5.cpp | 263 ++++++++++++++++++ tests/integration/harness/io.py | 14 + .../test_malleability_performance.py | 140 ++-------- 6 files changed, 305 insertions(+), 123 deletions(-) create mode 100644 tests/integration/harness/gkfs.io/random_md5.cpp diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 151e30f9c..d9843d85d 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -56,6 +56,7 @@ add_executable(gkfs.io gkfs.io/lseek.cpp gkfs.io/write_validate.cpp gkfs.io/write_random.cpp + gkfs.io/random_md5.cpp gkfs.io/truncate.cpp gkfs.io/util/file_compare.cpp gkfs.io/chdir.cpp @@ -74,6 +75,7 @@ add_executable(gkfs.io ) include(load_nlohmann_json) +find_package(OpenSSL REQUIRED) target_include_directories(gkfs.io PRIVATE ${BOOST_PREPROCESSOR_INCLUDE_DIRS} @@ -84,6 +86,7 @@ target_link_libraries(gkfs.io fmt::fmt CLI11::CLI11 std::filesystem + OpenSSL::Crypto rt ) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 7f0894c24..4eaf56348 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -103,6 +103,12 @@ directory_validate_init(CLI::App& app); void write_random_init(CLI::App& app); +void +write_random_and_md5_init(CLI::App& app); + +void +read_random_and_md5_init(CLI::App& app); + void truncate_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 854c96e61..552c24b3f 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -67,6 +67,8 @@ init_commands(CLI::App& app) { write_validate_init(app); directory_validate_init(app); write_random_init(app); + write_random_and_md5_init(app); + read_random_and_md5_init(app); truncate_init(app); access_init(app); statfs_init(app); diff --git a/tests/integration/harness/gkfs.io/random_md5.cpp b/tests/integration/harness/gkfs.io/random_md5.cpp new file mode 100644 index 000000000..2c307e8eb --- /dev/null +++ b/tests/integration/harness/gkfs.io/random_md5.cpp @@ -0,0 +1,263 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +namespace { + +constexpr std::size_t buffer_size = 1024 * 1024; + +constexpr std::size_t md5_digest_length = 16; + +std::string +md5_to_hex(const unsigned char* digest, unsigned int digest_len) { + std::string hex; + hex.reserve(digest_len * 2); + for(unsigned int i = 0; i < digest_len; ++i) { + hex += fmt::format("{:02x}", digest[i]); + } + return hex; +} + +class md5_stream { +public: + md5_stream() : ctx_(EVP_MD_CTX_new()) { + if(ctx_ == nullptr || EVP_DigestInit_ex(ctx_, EVP_md5(), nullptr) != 1) { + ok_ = false; + } + } + + ~md5_stream() { + if(ctx_ != nullptr) { + EVP_MD_CTX_free(ctx_); + } + } + + bool + update(const void* data, std::size_t size) { + if(!ok_) { + return false; + } + if(EVP_DigestUpdate(ctx_, data, size) != 1) { + ok_ = false; + } + return ok_; + } + + std::string + final() { + if(!ok_) { + return ""; + } + std::array digest{}; + unsigned int digest_len = 0; + if(EVP_DigestFinal_ex(ctx_, digest.data(), &digest_len) != 1) { + return ""; + } + if(digest_len != md5_digest_length) { + return ""; + } + return md5_to_hex(digest.data(), digest_len); + } + +private: + EVP_MD_CTX* ctx_; + bool ok_{true}; +}; + +void +fill_random(std::vector& buffer, std::mt19937_64& rng) { + std::size_t pos = 0; + while(pos < buffer.size()) { + auto value = rng(); + for(std::size_t i = 0; i < sizeof(value) && pos < buffer.size(); ++i) { + buffer[pos++] = static_cast((value >> (i * 8U)) & 0xffU); + } + } +} + +} // namespace + +struct random_md5_options { + bool verbose{}; + std::string pathname{}; + ::size_t count{}; + uint64_t seed{42}; + + REFL_DECL_STRUCT(random_md5_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(uint64_t, seed)); +}; + +struct random_md5_output { + ::ssize_t retval; + int errnum; + std::string md5; + + REFL_DECL_STRUCT(random_md5_output, REFL_DECL_MEMBER(::ssize_t, retval), + REFL_DECL_MEMBER(int, errnum), + REFL_DECL_MEMBER(std::string, md5)); +}; + +void +to_json(json& record, const random_md5_output& out) { + record = serialize(out); +} + +void +write_random_and_md5_exec(const random_md5_options& opts) { + auto fd = ::open(opts.pathname.c_str(), O_CREAT | O_WRONLY | O_TRUNC, + S_IRWXU | S_IRWXG | S_IRWXO); + if(fd == -1) { + json out = random_md5_output{-1, errno, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + md5_stream md5_ctx; + + std::mt19937_64 rng(opts.seed); + std::vector buffer(buffer_size); + std::size_t remaining = opts.count; + ::ssize_t written_total = 0; + int err = 0; + + while(remaining > 0) { + const auto step = std::min(buffer.size(), remaining); + if(buffer.size() != step) { + buffer.resize(step); + } + fill_random(buffer, rng); + if(!md5_ctx.update(buffer.data(), step)) { + ::close(fd); + json out = random_md5_output{-1, EIO, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + std::size_t done = 0; + while(done < step) { + auto rv = ::write(fd, buffer.data() + done, step - done); + if(rv < 0) { + err = errno; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + if(rv == 0) { + err = EIO; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + done += static_cast(rv); + written_total += rv; + } + remaining -= step; + } + + if(::close(fd) != 0 && err == 0) { + err = errno; + } + + json out = random_md5_output{err == 0 ? written_total : -1, err, + md5_ctx.final()}; + fmt::print("{}\n", out.dump(2)); +} + +void +read_random_and_md5_exec(const random_md5_options& opts) { + auto fd = ::open(opts.pathname.c_str(), O_RDONLY); + if(fd == -1) { + json out = random_md5_output{-1, errno, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + md5_stream md5_ctx; + + std::vector buffer(buffer_size); + ::ssize_t read_total = 0; + int err = 0; + + for(;;) { + auto rv = ::read(fd, buffer.data(), buffer.size()); + if(rv < 0) { + err = errno; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + if(rv == 0) { + break; + } + if(!md5_ctx.update(buffer.data(), static_cast(rv))) { + ::close(fd); + json out = random_md5_output{-1, EIO, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + read_total += rv; + } + + if(::close(fd) != 0 && err == 0) { + err = errno; + } + + json out = random_md5_output{err == 0 ? read_total : -1, err, + md5_ctx.final()}; + fmt::print("{}\n", out.dump(2)); +} + +void +write_random_and_md5_init(CLI::App& app) { + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("write_random_and_md5", + "Create/truncate a file, write deterministic random data, and return MD5"); + cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output"); + cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name(""); + cmd->add_option("count", opts->count, "Number of bytes to write")->required()->type_name(""); + cmd->add_option("--seed", opts->seed, "Pseudo-random seed")->type_name(""); + cmd->callback([opts]() { write_random_and_md5_exec(*opts); }); +} + +void +read_random_and_md5_init(CLI::App& app) { + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("read_random_and_md5", + "Read a full file and return MD5"); + cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output"); + cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name(""); + cmd->callback([opts]() { read_random_and_md5_exec(*opts); }); +} \ No newline at end of file diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index b41990f9e..5d677e347 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -373,6 +373,18 @@ class WriteRandomOutputSchema(Schema): return namedtuple('WriteRandomReturn', ['retval', 'errno'])(**data) +class RandomMd5OutputSchema(Schema): + """Schema to deserialize random write/read with MD5 helper output""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + md5 = fields.String(required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('RandomMd5Return', ['retval', 'errno', 'md5'])(**data) + + class WriteSyncOutputSchema(Schema): """Schema to deserialize the results of a write_sync() execution""" @@ -516,6 +528,8 @@ class IOParser: 'statx' : StatxOutputSchema(), 'lseek' : LseekOutputSchema(), 'write_random': WriteRandomOutputSchema(), + 'write_random_and_md5': RandomMd5OutputSchema(), + 'read_random_and_md5': RandomMd5OutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'write_sequential' : WriteValidateOutputSchema(), diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py index dfd3e1dda..1b72ebc48 100644 --- a/tests/integration/malleability/test_malleability_performance.py +++ b/tests/integration/malleability/test_malleability_performance.py @@ -368,15 +368,8 @@ def _wait_for_client_mount_ready(client, mountdir, timeout=45): def create_deterministic_file(client, mountdir, filename, size): - """Create a random reference file in GekkoFS and return its expected MD5. - - Data comes from /dev/urandom into a local reference file. Its MD5 is computed - once outside GekkoFS, then copied into the mounted file with one intercepted - dd process. This keeps random integrity coverage without reading generated - GekkoFS files back through Python before the malleability operation starts. - """ + """Create deterministic random data in GekkoFS and return its MD5.""" fpath = mountdir / filename - ref_path = mountdir.parent / f".{filename}.ref" # Make repeated perf runs idempotent: if a previous run left the file in # place, remove it before recreating it. try: @@ -385,10 +378,9 @@ def create_deterministic_file(client, mountdir, filename, size): pass last_ret = None + seed = 0x474b4653 ^ sum(ord(ch) for ch in filename) ^ size for attempt in range(30): - ret = client.open(fpath, - os.O_CREAT | os.O_WRONLY | os.O_TRUNC, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + ret = client.write_random_and_md5(fpath, size, "--seed", seed, timeout=max(60, int(size / (1024 * 1024)) * 5)) last_ret = ret if ret.retval != -1: break @@ -401,61 +393,12 @@ def create_deterministic_file(client, mountdir, filename, size): assert last_ret is not None assert last_ret.retval != -1, \ - f"open failed for {fpath}, errno={getattr(last_ret, 'errno', None)}" - - timeout = max(60, int(size / (1024 * 1024)) * 2) - completed = subprocess.run( - [ - "dd", - "if=/dev/urandom", - f"of={str(ref_path)}", - "bs=1M", - f"count={size}", - "iflag=count_bytes", - "conv=notrunc", - "status=none", - ], - env=os.environ, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - ) - assert completed.returncode == 0, ( - f"reference random-write failed for {ref_path}: {completed.stderr.decode()[:300]}" - ) - - digest = hashlib.md5() - with ref_path.open("rb") as ref_file: - for block in iter(lambda: ref_file.read(1024 * 1024), b""): - digest.update(block) - expected_md5 = digest.hexdigest() + f"write_random_and_md5 failed for {fpath}, errno={getattr(last_ret, 'errno', None)}" + assert last_ret.retval == size, \ + f"short write for {fpath}: wrote {last_ret.retval}, expected {size}, errno={last_ret.errno}" + assert last_ret.md5, f"write_random_and_md5 returned empty md5 for {fpath}" - completed = subprocess.run( - [ - "dd", - f"if={str(ref_path)}", - f"of={str(fpath)}", - "bs=1M", - "conv=notrunc", - "status=none", - ], - env=getattr(client, "_env", os.environ), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - ) - try: - ref_path.unlink() - except OSError: - pass - assert completed.returncode == 0, ( - f"copy into GekkoFS failed for {fpath}: " - f"returncode={describe_returncode(completed.returncode)}, " - f"stdout={completed.stdout.decode(errors='replace')[:300]}, " - f"stderr={completed.stderr.decode(errors='replace')[:300]}" - ) - - return fpath, expected_md5 + return fpath, last_ret.md5 def describe_returncode(returncode): @@ -493,75 +436,26 @@ def count_accessible_files(file_paths, client, timeout=60): def verify_files_with_md5(file_md5_map, client, mountdir, max_read_checks=None): - """Read back all files and verify their MD5 checksums.""" + """Read back files through gkfs.io and verify their MD5 checksums.""" results = {'passed': 0, 'failed': 0, 'details': []} items = list(file_md5_map.items()) if max_read_checks is not None: items = items[:max_read_checks] - if items: - try: - paths = [path for path, _ in items] - completed = subprocess.run( - ["md5sum"] + paths, - env=getattr(client, "_env", os.environ), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=max(30, len(paths) * 5), - ) - if completed.returncode == 0: - actual = {} - for line in completed.stdout.decode().splitlines(): - parts = line.split(None, 1) - if len(parts) == 2: - actual[parts[1].lstrip('*')] = parts[0] - for fpath_str, expected_md5 in items: - actual_md5 = actual.get(fpath_str) - if actual_md5 == expected_md5: - results['passed'] += 1 - else: - results['failed'] += 1 - results['details'].append({'file': fpath_str, - 'status': 'md5_mismatch', - 'expected': expected_md5, - 'actual': actual_md5}) - skipped = len(file_md5_map) - len(items) - results['passed'] += skipped - return results - logger.warning(f"fast md5sum failed: {completed.stderr.decode()[:300]}") - except Exception as exc: - logger.warning(f"fast md5sum exception: {exc}") - - for idx, (fpath_str, expected_md5) in enumerate(file_md5_map.items()): + for idx, (fpath_str, expected_md5) in enumerate(items): fpath = Path(fpath_str) try: - ret = client.open(fpath, os.O_RDONLY, 0, timeout=15) - if ret.retval == -1: - results['failed'] += 1 - results['details'].append({'file': fpath_str, 'status': 'open_failed'}) - continue - - stat_ret = client.stat(fpath, timeout=15) - if stat_ret.retval != 0: - results['failed'] += 1 - results['details'].append({'file': fpath_str, 'status': 'stat_failed'}) - continue - if max_read_checks is not None and idx >= max_read_checks: results['passed'] += 1 continue - file_size = stat_ret.statbuf.st_size - if file_size > 0: - ret = client.read(fpath, file_size, timeout=15) - if ret.retval == -1 or ret.buf is None: - results['failed'] += 1 - results['details'].append({'file': fpath_str, 'status': 'read_failed'}) - continue - actual_md5 = hashlib.md5(ret.buf).hexdigest() - else: - actual_md5 = hashlib.md5(b'').hexdigest() - + ret = client.read_random_and_md5(fpath, timeout=30) + if ret.retval == -1: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'read_md5_failed', 'errno': ret.errno}) + continue + actual_md5 = ret.md5 + if actual_md5 == expected_md5: results['passed'] += 1 else: -- GitLab From ec8e43b8c5968631c5e4f9318f76e3e37ed8d0a7 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 27 Aug 2026 20:05:31 +0200 Subject: [PATCH 07/21] fix: preserve stdio fds and malleability host files Avoid closing kernel stdin, stdout, or stderr when removing GKFS virtual file descriptor mappings, preventing libc or logging descriptors from being invalidated during teardown. Update integration test forwarding daemon setup to preserve host files for malleability tests and simplify gkfs_malleability invocation paths. --- src/client/open_file_map.cpp | 10 ++ tests/integration/conftest.py | 4 +- tests/integration/conftest.template | 4 +- tests/integration/harness/gkfs.py | 17 ++- .../test_client_disconnect_during_rpc.py | 27 +---- .../malleability/test_expand_on_demand.py | 28 ++--- .../test_malleability_error_handling.py | 73 ++----------- .../test_malleability_performance.py | 103 ++++-------------- .../malleability/test_malleability_tool.py | 34 ++---- .../test_malleability_tool_simple.py | 51 +++------ .../test_mutate_distributors_integrity.py | 28 ++--- .../syscalls/test_client_ofi_interface.py | 52 ++------- 12 files changed, 103 insertions(+), 328 deletions(-) diff --git a/src/client/open_file_map.cpp b/src/client/open_file_map.cpp index 1e36b6b08..52b403ae7 100644 --- a/src/client/open_file_map.cpp +++ b/src/client/open_file_map.cpp @@ -260,6 +260,16 @@ OpenFileMap::remove(const int fd) { shard.files.erase(fd); total_files_--; + if(fd >= 0 && fd < 3) { + // A GKFS file descriptor may intentionally be mapped onto stdin, + // stdout, or stderr via dup2()/dup3() (GNU dd does this for its input + // and output files). In that case the numeric descriptor can also name + // a real kernel stdio fd. The GKFS close path must only remove the + // virtual mapping; closing the kernel fd here may invalidate + // libc/logging descriptors during process teardown. + return true; + } + if(!CTX->protect_fds()) { if(!CTX->range_fd()) { // We close the dev null fd diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2aed09fea..22831dc41 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -215,7 +215,9 @@ def gkfwd_daemon_factory(test_workspace, request): interface = request.config.getoption('--interface') - return FwdDaemonCreator(interface, test_workspace) + keep_hosts = "malleability" in request.node.nodeid.split("/") + + return FwdDaemonCreator(interface, test_workspace, keep_hosts=keep_hosts) @pytest.fixture def gkfwd_client_factory(test_workspace): diff --git a/tests/integration/conftest.template b/tests/integration/conftest.template index 2aed09fea..22831dc41 100644 --- a/tests/integration/conftest.template +++ b/tests/integration/conftest.template @@ -215,7 +215,9 @@ def gkfwd_daemon_factory(test_workspace, request): interface = request.config.getoption('--interface') - return FwdDaemonCreator(interface, test_workspace) + keep_hosts = "malleability" in request.node.nodeid.split("/") + + return FwdDaemonCreator(interface, test_workspace, keep_hosts=keep_hosts) @pytest.fixture def gkfwd_client_factory(test_workspace): diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index a3e608aee..757c0231d 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -239,11 +239,12 @@ class FwdDaemonCreator: Factory that allows tests to create forwarding daemons in a workspace. """ - def __init__(self, interface, workspace): + def __init__(self, interface, workspace, keep_hosts=False): self._interface = interface self._workspace = workspace + self._keep_hosts = keep_hosts - def create(self, expand_mode=False, enable_forwarding=True): + def create(self, expand_mode=False, enable_forwarding=True, keep_hosts=None): """ Create a forwarding daemon in the tests workspace. @@ -257,10 +258,14 @@ class FwdDaemonCreator: The `FwdDaemon` object to interact with the daemon. """ + if keep_hosts is None: + keep_hosts = self._keep_hosts + daemon = FwdDaemon(self._interface, self._workspace, expand_mode=expand_mode, - enable_forwarding=enable_forwarding) + enable_forwarding=enable_forwarding, + keep_hosts=keep_hosts) daemon.run() return daemon @@ -1402,7 +1407,7 @@ class ShellClientLibc: return self._workspace.twd class FwdDaemon: - def __init__(self, interface, workspace, expand_mode=False, enable_forwarding=True): + def __init__(self, interface, workspace, expand_mode=False, enable_forwarding=True, keep_hosts=False): self._address = get_ephemeral_address(interface) self._workspace = workspace @@ -1411,6 +1416,7 @@ class FwdDaemon: self._env = os.environ.copy() self._expand_mode = expand_mode self._enable_forwarding = enable_forwarding + self._keep_hosts = keep_hosts self._rootdir = self._workspace.rootdir self._metadir = self._workspace.metadir self._logdir = self._workspace.logdir @@ -1432,8 +1438,9 @@ class FwdDaemon: 'GKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'GKFS_DAEMON_LOG_PATH' : str(self.logdir / gkfwd_daemon_log_file), 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level, - 'GKFS_DAEMON_KEEP_HOSTS_FILE': 'ON' } + if keep_hosts: + self._patched_env['GKFS_DAEMON_KEEP_HOSTS_FILE'] = 'ON' if expand_mode: self._patched_env['GKFS_DAEMON_EXPAND'] = 'ON' diff --git a/tests/integration/malleability/test_client_disconnect_during_rpc.py b/tests/integration/malleability/test_client_disconnect_during_rpc.py index f8aec82af..5b3505cae 100644 --- a/tests/integration/malleability/test_client_disconnect_during_rpc.py +++ b/tests/integration/malleability/test_client_disconnect_during_rpc.py @@ -37,7 +37,6 @@ which contains NA_NOENTRY errors when clients vanish during RPC handling. import os import subprocess import time -import shutil from pathlib import Path import pytest @@ -48,22 +47,16 @@ def test_daemon_survives_client_abort_during_write(gkfwd_daemon_factory, gkfs_sh time.sleep(5) hostfile = Path(d00.hostfile) - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" # Write some data first to establish the file exists - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") ret = gkfs_shell.bash( - f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} " + f"LIBGKFS_HOSTS_FILE={hostfile} " f"dd if=/dev/zero of={d00.mountdir}/test_abort_file bs=1024 count=1 2>&1" ) # Verify daemon works before abort cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" + f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"Daemon not responding before client abort: {cmd.stderr.decode()}" @@ -82,13 +75,6 @@ def test_safe_respond_handles_vanished_client(gkfwd_daemon_factory, gkfs_shell): hostfile = Path(d00.hostfile) - # Verify safe_respond wrapper exists in the compiled source - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - daemon_bin = shutil.which("gkfs_daemon", path=search_path) - assert daemon_bin is not None, "gkfs_daemon not found in PATH" - - # Check objdump for safe/respond symbols - ret = gkfs_shell.bash(f"objdump -t {daemon_bin} 2>/dev/null | grep -i safe | wc -l || echo 0") # The test just needs to pass if the daemon is functional # Verify the daemon survives shutdown and restart d00.shutdown() @@ -110,7 +96,6 @@ def test_multiple_rapid_client_disconnections(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") # Simulate multiple rapid disconnections for i in range(5): @@ -118,7 +103,6 @@ def test_multiple_rapid_client_disconnections(gkfwd_daemon_factory, gkfs_shell): client_proc = subprocess.Popen( ["python3", "-c", f""" import os, time -os.environ['LD_LIBRARY_PATH'] = '{libdirs}' os.environ['LIBGKFS_HOSTS_FILE'] = '{hostfile}' fd = os.open('{d00.mountdir}/test_rapid_{i}', os.O_CREAT | os.O_WRONLY, 0o644) if fd >= 0: @@ -133,14 +117,9 @@ if fd >= 0: time.sleep(0.5) # Verify daemon survived all disconnections - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" + f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ diff --git a/tests/integration/malleability/test_expand_on_demand.py b/tests/integration/malleability/test_expand_on_demand.py index fffddfe39..9f781ae39 100644 --- a/tests/integration/malleability/test_expand_on_demand.py +++ b/tests/integration/malleability/test_expand_on_demand.py @@ -7,7 +7,6 @@ import hashlib import os -import shutil import stat import time from pathlib import Path @@ -21,20 +20,9 @@ PARTIAL_OVERWRITE_OFFSET = 128 PARTIAL_OVERWRITE = b"expand-on-demand-partial" -def _get_malleability_bin(gkfs_shell): - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - return malleability_bin - -def _run_mutate_cmd(gkfs_shell, bin_path, hosts_file, args, timeout=120): - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - cmd_str = ( - f'LD_LIBRARY_PATH="{libdirs}" ' - f'LIBGKFS_HOSTS_FILE="{hosts_file}" ' - f'{bin_path} {args}' - ) +def _run_mutate_cmd(gkfs_shell, hosts_file, args, timeout=120): + cmd_str = f'LIBGKFS_HOSTS_FILE="{hosts_file}" gkfs_malleability {args}' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) assert cmd.exit_code == 0, ( f"gkfs_malleability {args} failed\n" @@ -86,10 +74,10 @@ def _write_file(client, path): return _read_md5(client, path) -def _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile): +def _wait_for_mutate_done(gkfs_shell, hostfile): deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + cmd = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") if "No mutate running/finished." in cmd.stderr.decode(): return time.sleep(2) @@ -120,11 +108,9 @@ def test_expand_on_demand_skips_eager_data_and_keeps_reads_correct( hostfile = Path(d00.hostfile) _mark_added_daemon(hostfile, d01) - malleability_bin = _get_malleability_bin(gkfs_shell) - - _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate start", 340) - _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile) - _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate finalize") + _run_mutate_cmd(gkfs_shell, hostfile, "mutate start", 340) + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") daemon_log = Path(d00.logdir) / "gkfs_daemon.log" log_text = daemon_log.read_text() if daemon_log.exists() else "" diff --git a/tests/integration/malleability/test_malleability_error_handling.py b/tests/integration/malleability/test_malleability_error_handling.py index 5e97dd6bd..83dcf277a 100644 --- a/tests/integration/malleability/test_malleability_error_handling.py +++ b/tests/integration/malleability/test_malleability_error_handling.py @@ -35,7 +35,6 @@ RPC errors, and mid-operation failures without crashing the daemon. import os import time -import shutil from pathlib import Path import pytest @@ -47,17 +46,7 @@ def test_expand_status_with_no_running_expansion(gkfwd_daemon_factory, gkfs_shel hostfile = Path(d00.hostfile) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"mutate status failed: {cmd.stderr.decode()}" @@ -78,12 +67,6 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Use marker-based approach: no --new-hosts-file needed # Mark all entries with '+' to simulate adding same nodes (will fail gracefully) with open(hostfile, 'r') as hf_in: @@ -95,20 +78,12 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): else: hf_out.write(line) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) # Verify daemon is still running (didn't crash) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" @@ -130,12 +105,6 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Try mutate with markers (may fail due to same node count) with open(hostfile, 'r') as hf_in: original_lines = hf_in.readlines() @@ -146,20 +115,12 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): else: hf_out.write(line) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) # Verify mutate status works - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"shrink status after failed expand failed: {cmd.stderr.decode()}" @@ -188,18 +149,8 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s hostfile = Path(d00.hostfile) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Verify no running mutation - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 @@ -213,19 +164,11 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s else: hf_out.write(line) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) # Verify daemon is still running - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} mutate status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py index 1b72ebc48..3cfb1625e 100644 --- a/tests/integration/malleability/test_malleability_performance.py +++ b/tests/integration/malleability/test_malleability_performance.py @@ -62,54 +62,6 @@ from harness.logger import logger # Helpers # ========= -def _malleability_supports_mutate(binary): - """Return True if gkfs_malleability binary has the marker-based mutate CLI.""" - try: - completed = subprocess.run( - [binary, "--help"], - check=False, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=10, - ) - except (OSError, subprocess.TimeoutExpired): - return False - - return "mutate" in completed.stdout or "mutate" in completed.stderr - - -def _get_malleability_bin(gkfs_shell): - """Find a gkfs_malleability binary compatible with these tests.""" - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - candidates = [] - found = shutil.which("gkfs_malleability", path=search_path) - if found is not None: - candidates.append(found) - - # The debug-local install prefix can be stale. Prefer the real install prefix - # used by CI/local deps, then the build-tree tool. - candidates.extend([ - "/home/rnou/iodeps/bin/gkfs_malleability", - "/home/rnou/gekkofs/builds/debug-local/tools/gkfs_malleability", - ]) - - checked = [] - for candidate in candidates: - if candidate in checked or not os.path.exists(candidate): - continue - checked.append(candidate) - if _malleability_supports_mutate(candidate): - logger.info(f"Using gkfs_malleability: {candidate}") - return candidate - - raise AssertionError( - "No compatible gkfs_malleability found. Checked: " - f"{', '.join(checked) or ''}. Use --bin-dir=/home/rnou/iodeps/bin " - "or rebuild/install so the binary supports 'mutate'." - ) - - def _shutdown_daemons(daemons, timeout=10): """Shutdown daemons without letting perf tests hang forever.""" for daemon in daemons: @@ -724,7 +676,7 @@ def test_shrink_performance_multi_run(client_fixture, base_workspace = request.getfixturevalue("test_workspace") iter_workspace = _make_iteration_workspace(base_workspace, f"shrink_iter_{run_idx}") - iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) if hasattr(client, "_patched_env"): client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") if hasattr(client, "_env"): @@ -768,12 +720,8 @@ def test_shrink_performance_multi_run(client_fixture, workspace_file = hostfile.parent / f"shrink_workspace_run{run_idx}.txt" _build_workspace_for_shrink(workspace_file, survivors, removing_daemons) - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - malleability_bin = _get_malleability_bin(gkfs_shell) - env_file = hostfile.parent / f"test_env_shrink_run{run_idx}.sh" env_file.write_text( - f'export LD_LIBRARY_PATH="{libdirs}"\n' f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' ) @@ -782,7 +730,7 @@ def test_shrink_performance_multi_run(client_fixture, # Execute shrink t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) if cmd.exit_code != 0: _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) @@ -791,7 +739,7 @@ def test_shrink_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break @@ -803,7 +751,7 @@ def test_shrink_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -900,7 +848,7 @@ def test_expand_performance_multi_run(client_fixture, base_workspace = request.getfixturevalue("test_workspace") iter_workspace = _make_iteration_workspace(base_workspace, f"expand_iter_{run_idx}") - iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) if hasattr(client, "_patched_env"): client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") client._patched_env["GKFS_EXPAND_ON_DEMAND"] = expand_on_demand_value @@ -959,12 +907,8 @@ def test_expand_performance_multi_run(client_fixture, assert len(adding_entries) == len(new_daemons), \ f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - malleability_bin = _get_malleability_bin(gkfs_shell) - env_file = hostfile.parent / f"test_env_expand_run{run_idx}.sh" env_file.write_text( - f'export LD_LIBRARY_PATH="{libdirs}"\n' f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' f'export GKFS_EXPAND_ON_DEMAND="{expand_on_demand_value}"\n' ) @@ -974,7 +918,7 @@ def test_expand_performance_multi_run(client_fixture, # Execute expand t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) if cmd.exit_code != 0: _dump_mutate_failure_context("Expand mutate start", cmd, workspace_file) @@ -983,7 +927,7 @@ def test_expand_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break @@ -995,7 +939,7 @@ def test_expand_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -1088,7 +1032,7 @@ def test_mutate_performance_multi_run(client_fixture, base_workspace = request.getfixturevalue("test_workspace") iter_workspace = _make_iteration_workspace(base_workspace, f"mutate_iter_{run_idx}") - iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) if hasattr(client, "_patched_env"): client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") if hasattr(client, "_env"): @@ -1141,12 +1085,8 @@ def test_mutate_performance_multi_run(client_fixture, all_daemons_to_write = list(old_daemons) + list(new_daemons) _build_workspace_with_markers(workspace_file, all_daemons_to_write, marker_map) - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - malleability_bin = _get_malleability_bin(gkfs_shell) - env_file = hostfile.parent / f"test_env_mutate_run{run_idx}.sh" env_file.write_text( - f'export LD_LIBRARY_PATH="{libdirs}"\n' f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' ) @@ -1155,7 +1095,7 @@ def test_mutate_performance_multi_run(client_fixture, # Execute mutate t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) if cmd.exit_code != 0: _dump_mutate_failure_context("Mutate start", cmd, workspace_file) @@ -1175,7 +1115,7 @@ def test_mutate_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break @@ -1185,7 +1125,7 @@ def test_mutate_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -1278,7 +1218,7 @@ def test_comprehensive_cycle_performance(client_fixture, base_workspace = request.getfixturevalue("test_workspace") iter_workspace = _make_iteration_workspace(base_workspace, f"cycle_iter_{rep_idx}") - iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace) + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) if hasattr(client, "_patched_env"): client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") if hasattr(client, "_env"): @@ -1319,17 +1259,13 @@ def test_comprehensive_cycle_performance(client_fixture, workspace_shrink = hostfile.parent / f"cycle_shrink_workspace_{rep_idx}.txt" _build_workspace_for_shrink(workspace_shrink, survivors, removing_daemons) - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - malleability_bin = _get_malleability_bin(gkfs_shell) - env_file = hostfile.parent / f"test_env_cycle{rep_idx}.sh" env_file.write_text( - f'export LD_LIBRARY_PATH="{libdirs}"\n' f'export LIBGKFS_HOSTS_FILE="{workspace_shrink}"\n' ) shrink_t0 = perf_counter() - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) if cmd.exit_code != 0: _dump_mutate_failure_context("Cycle shrink mutate start", cmd, workspace_shrink) @@ -1337,14 +1273,14 @@ def test_comprehensive_cycle_performance(client_fixture, deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) shrink_elapsed = perf_counter() - shrink_t0 - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_shrink) @@ -1392,12 +1328,11 @@ def test_comprehensive_cycle_performance(client_fixture, f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" env_file.write_text( - f'export LD_LIBRARY_PATH="{libdirs}"\n' f'export LIBGKFS_HOSTS_FILE="{workspace_expand}"\n' ) expand_t0 = perf_counter() - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate start"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) if cmd.exit_code != 0: _dump_mutate_failure_context("Cycle expand mutate start", cmd, workspace_expand) @@ -1411,14 +1346,14 @@ def test_comprehensive_cycle_performance(client_fixture, deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate status"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) expand_elapsed = perf_counter() - expand_t0 - cmd_str = f'bash -c "source {env_file} && {malleability_bin} mutate finalize"' + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_expand) diff --git a/tests/integration/malleability/test_malleability_tool.py b/tests/integration/malleability/test_malleability_tool.py index 3cd667a51..0a208757b 100644 --- a/tests/integration/malleability/test_malleability_tool.py +++ b/tests/integration/malleability/test_malleability_tool.py @@ -28,7 +28,6 @@ import harness from pathlib import Path -import shutil import stat import os import pytest @@ -58,9 +57,6 @@ def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d01 = gkfwd_daemon_factory.create(expand_mode=True) time.sleep(2) - libdirs = gkfs_shell._patched_env['LD_LIBRARY_PATH'] - search_path = ':'.join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which('gkfs_malleability', path=search_path) # Marker-based API: LIBGKFS_HOSTS_FILE = workspace hostfile with markers # - d00 stays (no prefix) @@ -80,19 +76,19 @@ def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): if not line.startswith("#"): hf_out.write("+" + line) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate status" + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"Command '{cmd_str}' failed: {cmd.stderr.decode()}" assert "No mutate running/finished." in cmd.stderr.decode() - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate start" + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" assert "Mutate process" in cmd.stderr.decode() time.sleep(10) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate finalize" + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate finalize" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" @@ -107,8 +103,6 @@ def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): time.sleep(5) - search_path = ':'.join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which('gkfs_malleability', path=search_path) hostfile = Path(d00.hostfile) @@ -128,7 +122,7 @@ def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} {malleability_bin} mutate start" + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) assert cmd.exit_code != 0 assert "All hosts would be removed" in cmd.stderr.decode() or "Error" in cmd.stderr.decode() @@ -179,15 +173,9 @@ def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) # Keep as active - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - # Check status before starting (no --new-hosts-file needed) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} mutate status" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 @@ -195,9 +183,7 @@ def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): # Start mutate (shrink = fewer nodes) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} mutate start" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" @@ -207,9 +193,7 @@ def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): deadline = time.time() + 120 while time.time() < deadline: cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} mutate status" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): @@ -220,9 +204,7 @@ def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): # Finalize (no --new-hosts-file needed) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} mutate finalize" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate finalize" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" diff --git a/tests/integration/malleability/test_malleability_tool_simple.py b/tests/integration/malleability/test_malleability_tool_simple.py index 9a817fa35..3aaf22511 100644 --- a/tests/integration/malleability/test_malleability_tool_simple.py +++ b/tests/integration/malleability/test_malleability_tool_simple.py @@ -43,28 +43,14 @@ Each test is standalone and can be run by the user independently: import os import stat import time -import shutil from pathlib import Path import pytest -def _get_malleability_bin(gkfs_shell): - """Return the path to the gkfs_malleability binary.""" - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - return malleability_bin - -def _run_cmd(bin_path, hosts_file, args, gkfs_shell, timeout=None): +def _run_cmd(hosts_file, args, gkfs_shell, timeout=None): """Run a gkfs_malleability command and assert success.""" - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hosts_file} " - f"{bin_path} {args}" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hosts_file} gkfs_malleability {args}" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) assert cmd.exit_code == 0, ( f"gkfs_malleability command failed: {cmd_str}\n" @@ -94,8 +80,6 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d01 = gkfwd_daemon_factory.create(expand_mode=True) time.sleep(2) - malleability_bin = _get_malleability_bin(gkfs_shell) - d01_port = d01.address.split(":")[-1] with open(hostfile, "r") as hf_in: original_lines = hf_in.readlines() @@ -110,24 +94,23 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(malleability_bin, hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd( - malleability_bin, hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(malleability_bin, hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(malleability_bin, hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 for i in range(4): @@ -167,8 +150,6 @@ def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): assert ret.retval == 0, f"write_validate failed for {fpath}" old_hostfile = Path(d00.hostfile) - malleability_bin = _get_malleability_bin(gkfs_shell) - d01_port = d01.address.split(":")[-1] with open(old_hostfile, "r") as hf_in: original_lines = hf_in.readlines() @@ -183,24 +164,23 @@ def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd( - malleability_bin, old_hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(old_hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 for i in range(8): @@ -247,8 +227,6 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d02 = gkfwd_daemon_factory.create(expand_mode=True) time.sleep(2) - malleability_bin = _get_malleability_bin(gkfs_shell) - d01_port = d01.address.split(":")[-1] d02_port = d02.address.split(":")[-1] with open(old_hostfile, "r") as hf_in: @@ -266,24 +244,23 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd( - malleability_bin, old_hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(old_hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(malleability_bin, old_hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(old_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 for i in range(4): diff --git a/tests/integration/malleability/test_mutate_distributors_integrity.py b/tests/integration/malleability/test_mutate_distributors_integrity.py index 084ccdc14..e6543f447 100644 --- a/tests/integration/malleability/test_mutate_distributors_integrity.py +++ b/tests/integration/malleability/test_mutate_distributors_integrity.py @@ -21,7 +21,6 @@ case verifies: import hashlib import os -import shutil import stat import time from pathlib import Path @@ -33,20 +32,9 @@ FILE_COUNT = 6 FILE_SIZE = 64 * 1024 -def _get_malleability_bin(gkfs_shell): - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - return malleability_bin - -def _run_mutate_cmd(gkfs_shell, bin_path, hosts_file, args, timeout=120, logdir=None): - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - cmd_str = ( - f'LD_LIBRARY_PATH="{libdirs}" ' - f'LIBGKFS_HOSTS_FILE="{hosts_file}" ' - f'{bin_path} {args}' - ) +def _run_mutate_cmd(gkfs_shell, hosts_file, args, timeout=120, logdir=None): + cmd_str = f'LIBGKFS_HOSTS_FILE="{hosts_file}" gkfs_malleability {args}' cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) diagnostics = "" if logdir is not None: @@ -145,10 +133,10 @@ def _verify_md5_map(client, md5_map, logdir=None, hostfile=None): ) -def _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile): +def _wait_for_mutate_done(gkfs_shell, hostfile): deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + cmd = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") if "No mutate running/finished." in cmd.stderr.decode(): return time.sleep(2) @@ -224,21 +212,19 @@ def test_mutate_add_remove_swap_keeps_data_and_chunks_distributed( hostfile = Path(old_daemons[0].hostfile) _write_workspace(hostfile, active_daemons, removing=removed_daemons, adding=added_daemons) - malleability_bin = _get_malleability_bin(gkfs_shell) - status = _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate status") + status = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") assert "No mutate running/finished." in status.stderr.decode() start = _run_mutate_cmd( gkfs_shell, - malleability_bin, hostfile, "mutate start", timeout=340, logdir=old_daemons[0].logdir, ) assert "Mutate process" in start.stderr.decode() - _wait_for_mutate_done(gkfs_shell, malleability_bin, hostfile) - _run_mutate_cmd(gkfs_shell, malleability_bin, hostfile, "mutate finalize") + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") _assert_final_hostfile(hostfile, removed_daemons) for daemon in removed_daemons: diff --git a/tests/integration/syscalls/test_client_ofi_interface.py b/tests/integration/syscalls/test_client_ofi_interface.py index 9d8e03bd0..c85ef9260 100644 --- a/tests/integration/syscalls/test_client_ofi_interface.py +++ b/tests/integration/syscalls/test_client_ofi_interface.py @@ -36,7 +36,6 @@ and that LIBGKFS_OFI_INTERFACE takes precedence when both are set. import os import time -import shutil from pathlib import Path import pytest @@ -50,31 +49,19 @@ def test_ofi_interface_env_var_honored(gkfwd_daemon_factory, gkfs_shell): with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Without LIBGKFS_OFI_INTERFACE, status should work - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"expand status failed: {cmd.stderr.decode()}" + assert cmd.exit_code == 0, f"mutate status failed: {cmd.stderr.decode()}" # With LIBGKFS_OFI_INTERFACE=lo, should still work cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"LIBGKFS_OFI_INTERFACE=lo " - f"{malleability_bin} expand status" + f"LIBGKFS_OFI_INTERFACE=lo gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ - f"expand status with LIBGKFS_OFI_INTERFACE=lo failed: {cmd.stderr.decode()}" + f"mutate status with LIBGKFS_OFI_INTERFACE=lo failed: {cmd.stderr.decode()}" d00.shutdown() @@ -88,22 +75,15 @@ def test_fi_sockets_iface_env_var_honored(gkfwd_daemon_factory, gkfs_shell): with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # With FI_SOCKETS_IFACE set, should work cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"FI_SOCKETS_IFACE=lo " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ - f"expand status with FI_SOCKETS_IFACE=lo failed: {cmd.stderr.decode()}" + f"mutate status with FI_SOCKETS_IFACE=lo failed: {cmd.stderr.decode()}" d00.shutdown() @@ -117,19 +97,12 @@ def test_libgkfs_ofi_interface_takes_precedence(gkfwd_daemon_factory, gkfs_shell with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Both set - LIBGKFS_OFI_INTERFACE should take precedence cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"LIBGKFS_OFI_INTERFACE=mlx5_0 " f"FI_SOCKETS_IFACE=ib0 " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ @@ -147,22 +120,15 @@ def test_libgkfs_ofi_interface_with_multiple_iface_values(gkfwd_daemon_factory, with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - for iface in ["lo", "eth0", "ib0", "mlx5_0"]: cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"LIBGKFS_OFI_INTERFACE={iface} " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) # Status doesn't require the interface to exist, just to be set assert cmd.exit_code == 0, \ - f"expand status with LIBGKFS_OFI_INTERFACE={iface} failed: {cmd.stderr.decode()}" + f"mutate status with LIBGKFS_OFI_INTERFACE={iface} failed: {cmd.stderr.decode()}" d00.shutdown() \ No newline at end of file -- GitLab From 6d69072176b37bae7419573f716e433ba00ed687 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 27 Aug 2026 20:41:41 +0200 Subject: [PATCH 08/21] test: improve malleability integration test cleanup Propagate workspace PATH for shell clients while composing library paths consistently, and avoid leaking LD_PRELOAD when interception is disabled. Make malleability performance tests more robust by retrying transient storage cleanup and file creation failures, including EBUSY, EDQUOT, and ENOSPC, so successive runs are less likely to fail due to backend teardown races. --- tests/integration/harness/gkfs.py | 19 +- .../test_malleability_performance.py | 294 +++++++++--------- 2 files changed, 166 insertions(+), 147 deletions(-) diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index 757c0231d..c84144010 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -893,9 +893,16 @@ class ShellClient: self._env = os.environ.copy() self._proxy = proxy - libdirs = ':'.join( - filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + - [str(p) for p in self._workspace.libdirs])) + bindirs = os.pathsep.join(str(p) for p in self._workspace.bindirs) + if bindirs: + self._env['PATH'] = os.pathsep.join( + filter(None, [bindirs, os.environ.get('PATH', '')])) + + libdirs = _compose_ld_library_path( + preferred_dirs=[], + workspace_dirs=self._workspace.libdirs, + inherited_ld_path=os.environ.get('LD_LIBRARY_PATH', ''), + ) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, @@ -1008,8 +1015,12 @@ class ShellClient: logger.debug(f"patched env: {self._patched_env}") + script_env = self._env.copy() + if not intercept_shell: + script_env.pop('LD_PRELOAD', None) + proc = subprocess.Popen(['bash', '-c', code], - env = (self._env if intercept_shell else os.environ), + env = script_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py index 3cfb1625e..18186e84d 100644 --- a/tests/integration/malleability/test_malleability_performance.py +++ b/tests/integration/malleability/test_malleability_performance.py @@ -43,6 +43,7 @@ import stat import time import math import hashlib +import errno import json import shutil import statistics @@ -67,27 +68,19 @@ def _shutdown_daemons(daemons, timeout=10): for daemon in daemons: try: proc = getattr(daemon, "_proc", None) - if proc is None or proc.poll() is not None: - continue - - logger.debug(f"terminating daemon pid {proc.pid}") - proc.terminate() - try: - proc.wait(timeout=timeout) - except subprocess.TimeoutExpired: - logger.warning(f"daemon pid {proc.pid} did not stop after SIGTERM; killing") - proc.kill() - proc.wait(timeout=timeout) + if proc is not None and proc.poll() is None: + logger.debug(f"terminating daemon pid {proc.pid}") + proc.terminate() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning(f"daemon pid {proc.pid} did not stop after SIGTERM; killing") + proc.kill() + proc.wait(timeout=timeout) except Exception as exc: logger.warning(f"daemon cleanup failed: {exc}") - - shutil.rmtree(daemon.rootdir.as_posix(), ignore_errors=True) - - # Give the backend a moment to release the mount and RPC state before the - # next performance run starts. Without this pause, the next run may hit - # transient EBUSY failures on file creation. - time.sleep(2) - + finally: + _cleanup_daemon_storage(daemon) def _snapshot_chunk_files(rootdir): """Snapshot chunk files under a daemon rootdir. @@ -219,12 +212,31 @@ def _wait_for_unmounted(mountdir, timeout=10): def _cleanup_daemon_storage(daemon): - shutil.rmtree(daemon.rootdir.as_posix(), ignore_errors=True) - shutil.rmtree(daemon.metadir.as_posix(), ignore_errors=True) + for path in (daemon.rootdir, daemon.metadir): + path = Path(path) + for attempt in range(20): + shutil.rmtree(path.as_posix(), ignore_errors=True) + if not path.exists(): + break + time.sleep(0.25) + if path.exists(): + logger.warning(f"failed to remove daemon storage {path}") #if daemon.logdir.exists(): # shutil.rmtree(daemon.logdir.as_posix(), ignore_errors=True) +def _cleanup_iteration_workspace(iter_workspace): + for path in (iter_workspace.rootdir, iter_workspace.metadir, iter_workspace.mountdir): + path = Path(path) + for attempt in range(20): + shutil.rmtree(path.as_posix(), ignore_errors=True) + if not path.exists(): + break + time.sleep(0.25) + if path.exists(): + logger.warning(f"failed to remove iteration workspace path {path}") + + class _IterationWorkspaceAdapter: """Per-iteration workspace with fresh mount/root/meta/log dirs.""" @@ -331,16 +343,26 @@ def create_deterministic_file(client, mountdir, filename, size): last_ret = None seed = 0x474b4653 ^ sum(ord(ch) for ch in filename) ^ size + retry_errnos = {errno.EBUSY, errno.EDQUOT, errno.ENOSPC} for attempt in range(30): ret = client.write_random_and_md5(fpath, size, "--seed", seed, timeout=max(60, int(size / (1024 * 1024)) * 5)) last_ret = ret if ret.retval != -1: break last_errno = getattr(ret, 'errno', None) - if last_errno != 16: + if last_errno not in retry_errnos: break - logger.warning(f"open busy for {fpath} (attempt {attempt + 1}/30), retrying") - logger.debug(f" mountdir exists={mountdir.exists()} ismount={os.path.ismount(mountdir)}") + try: + usage = shutil.disk_usage(mountdir) + free = usage.free + except OSError: + free = "unknown" + logger.warning( + f"transient write failure for {fpath}, errno={last_errno} " + f"(attempt {attempt + 1}/30), retrying") + logger.debug( + f" mountdir exists={mountdir.exists()} " + f"ismount={os.path.ismount(mountdir)} free={free}") time.sleep(1) assert last_ret is not None @@ -606,6 +628,7 @@ DEFAULT_FILE_SIZE = int(os.environ.get("GKFS_PERF_FILE_SIZE", str(8 * 1024))) DEFAULT_OLD_NODES = int(os.environ.get("GKFS_MALLEABILITY_OLD_NODES", "4")) DEFAULT_NEW_NODES = int(os.environ.get("GKFS_MALLEABILITY_NEW_NODES", "2")) DEFAULT_TIMEOUT = 340 +MUTATE_STATUS_POLL_INTERVAL = 0.25 CI_FAST = os.environ.get("GKFS_MALLEABILITY_CI_FAST", "").lower() in ("1", "on", "true", "yes") KEEP_ITERATION_WORKSPACES = os.environ.get( "GKFS_MALLEABILITY_KEEP_WORKSPACES", "").lower() in ("1", "on", "true", "yes") @@ -669,7 +692,6 @@ def test_shrink_performance_multi_run(client_fixture, all_daemons = [] for run_idx in range(num_reps): - time.sleep(10) # Give the backend time to release resources from previous run logger.info("=" * 70) logger.info(f"SHRINK PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") logger.info("=" * 70) @@ -677,117 +699,116 @@ def test_shrink_performance_multi_run(client_fixture, base_workspace = request.getfixturevalue("test_workspace") iter_workspace = _make_iteration_workspace(base_workspace, f"shrink_iter_{run_idx}") iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) - if hasattr(client, "_patched_env"): - client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") - if hasattr(client, "_env"): - client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") - - # Create daemons daemons = [] - for i in range(num_start): - d = iter_daemon_factory.create(enable_forwarding=False) - daemons.append(d) - time.sleep(5) - all_daemons.extend(daemons) - - hostfile = Path(daemons[0].hostfile) - _set_client_hostfile(client, hostfile) - run_mountdir = daemons[0].mountdir - _wait_for_client_mount_ready(client, run_mountdir) - - # Generate data - file_md5_map = {} - created_files = [] - t0 = perf_counter() - for i in range(num_files): - fname = f"shrink_perf_{run_idx}_{i:03d}" - fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) - file_md5_map[str(fpath)] = md5 - created_files.append(fpath) - gen_time = perf_counter() - t0 - logger.info(f" Generated {num_files} files in {gen_time:.3f}s") - - pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) - assert pre_verify['failed'] == 0 - - before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} - - # Pick survivors randomly - survivors = random.sample(daemons, num_end) - removing_daemons = [d for d in daemons if d not in survivors] - - # Build workspace with markers: all active + removing ('-' marker) - workspace_file = hostfile.parent / f"shrink_workspace_run{run_idx}.txt" - _build_workspace_for_shrink(workspace_file, survivors, removing_daemons) - - env_file = hostfile.parent / f"test_env_shrink_run{run_idx}.sh" - env_file.write_text( - f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' - ) + run_mountdir = None + try: + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Create daemons + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"shrink_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + logger.info(f" Generated {num_files} files in {gen_time:.3f}s") + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Pick survivors randomly + survivors = random.sample(daemons, num_end) + removing_daemons = [d for d in daemons if d not in survivors] + + # Build workspace with markers: all active + removing ('-' marker) + workspace_file = hostfile.parent / f"shrink_workspace_run{run_idx}.txt" + _build_workspace_for_shrink(workspace_file, survivors, removing_daemons) + + env_file = hostfile.parent / f"test_env_shrink_run{run_idx}.sh" + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + ) - logger.info(f" Shrink: {len(survivors)} keep, {len(removing_daemons)} remove") + logger.info(f" Shrink: {len(survivors)} keep, {len(removing_daemons)} remove") + + # Execute shrink + t0 = perf_counter() + t_wall = time.time() + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) + assert cmd.exit_code == 0, f"Shrink start failed: {cmd.stderr.decode()[:300]}" + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + else: + pytest.fail(f"Shrink did not complete within 120s (run {run_idx + 1})") - # Execute shrink - t0 = perf_counter() - t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) - if cmd.exit_code != 0: - _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) - assert cmd.exit_code == 0, f"Shrink start failed: {cmd.stderr.decode()[:300]}" + elapsed = perf_counter() - t0 + wall = time.time() - t_wall - # Wait for completion - deadline = time.time() + 120 - while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' + # Finalize + cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - if "No mutate running/finished." in cmd.stderr.decode(): - break - time.sleep(2) - else: - pytest.fail(f"Shrink did not complete within 120s (run {run_idx + 1})") - - elapsed = perf_counter() - t0 - wall = time.time() - t_wall - - # Finalize - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0 - _set_client_hostfile(client, workspace_file) - time.sleep(2) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) - # Verify - accessible = count_accessible_files(created_files, client) - post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, - max_read_checks=1) - if post_verify['failed']: - logger.warning(f" Shrink post-verify failures: {post_verify['details'][:5]}") + # Verify + accessible = count_accessible_files(created_files, client) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + if post_verify['failed']: + logger.warning(f" Shrink post-verify failures: {post_verify['details'][:5]}") - after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} - moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) - logger.info(f" Shrink redistributed {moved_bytes / (1024 ** 2):.4f} MB") + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Shrink redistributed {moved_bytes / (1024 ** 2):.4f} MB") - result = OperationResult( - run_index=run_idx, - elapsed_seconds=elapsed, - wall_time_seconds=wall, - data_redistributed_bytes=moved_bytes, - md5_passed=post_verify['passed'], - md5_failed=post_verify['failed'], - files_accessible=accessible, - files_total=num_files, - ) - all_results.append(result) + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) - logger.info(f" Run {run_idx + 1}: shrink took {elapsed:.3f}s, files: {accessible}/{num_files}") + logger.info(f" Run {run_idx + 1}: shrink took {elapsed:.3f}s, files: {accessible}/{num_files}") - # Clean up: shutdown daemons and wipe backend storage for fresh iteration - _shutdown_daemons(daemons) - _wait_for_unmounted(run_mountdir) - for d in daemons: - _cleanup_daemon_storage(d) - _cleanup_iteration_workspace(iter_workspace) + finally: + _shutdown_daemons(daemons) + if run_mountdir is not None: + _wait_for_unmounted(run_mountdir) + _cleanup_iteration_workspace(iter_workspace) stats = compute_statistics(all_results) @@ -840,7 +861,6 @@ def test_expand_performance_multi_run(client_fixture, expand_on_demand_value = "ON" if EXPAND_ON_DEMAND else "OFF" for run_idx in range(num_reps): - time.sleep(10) # Give the backend time to release resources from previous run logger.info("=" * 70) logger.info(f"EXPAND PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") logger.info(f" GKFS_EXPAND_ON_DEMAND={expand_on_demand_value}") @@ -861,7 +881,6 @@ def test_expand_performance_multi_run(client_fixture, for i in range(num_start): d = iter_daemon_factory.create(enable_forwarding=False) daemons.append(d) - time.sleep(5) all_daemons.extend(daemons) hostfile = Path(daemons[0].hostfile) @@ -892,7 +911,6 @@ def test_expand_performance_multi_run(client_fixture, d = iter_daemon_factory.create(expand_mode=True, enable_forwarding=False) new_daemons.append(d) - time.sleep(2) # Wait for daemon to write '+' to workspace all_daemons.extend(new_daemons) logger.info(f" Expand: {len(daemons)} active, {len(new_daemons)} adding") @@ -931,7 +949,7 @@ def test_expand_performance_multi_run(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break - time.sleep(2) + time.sleep(MUTATE_STATUS_POLL_INTERVAL) else: pytest.fail(f"Expand did not complete within 120s (run {run_idx + 1})") @@ -943,7 +961,6 @@ def test_expand_performance_multi_run(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) - time.sleep(2) # Verify accessible = count_accessible_files(created_files, client) @@ -1025,7 +1042,6 @@ def test_mutate_performance_multi_run(client_fixture, all_daemons = [] for run_idx in range(num_reps): - time.sleep(10) # Give the backend time to release resources from previous run logger.info("=" * 70) logger.info(f"MUTATE PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") logger.info("=" * 70) @@ -1043,7 +1059,6 @@ def test_mutate_performance_multi_run(client_fixture, for i in range(num_start): d = iter_daemon_factory.create(enable_forwarding=False) old_daemons.append(d) - time.sleep(5) all_daemons.extend(old_daemons) hostfile = Path(old_daemons[0].hostfile) @@ -1072,7 +1087,6 @@ def test_mutate_performance_multi_run(client_fixture, d = iter_daemon_factory.create(expand_mode=True, enable_forwarding=False) new_daemons.append(d) - time.sleep(1) all_daemons.extend(new_daemons) # For same-count mutate: all old get '-' markers, all new get '+' markers @@ -1119,7 +1133,7 @@ def test_mutate_performance_multi_run(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break - time.sleep(2) + time.sleep(MUTATE_STATUS_POLL_INTERVAL) elapsed = perf_counter() - t0 wall = time.time() - t_wall @@ -1129,7 +1143,6 @@ def test_mutate_performance_multi_run(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) - time.sleep(2) # Verify accessible = count_accessible_files(created_files, client) @@ -1211,7 +1224,6 @@ def test_comprehensive_cycle_performance(client_fixture, all_daemons = [] for rep_idx in range(num_reps): - time.sleep(10) # Give the backend time to release resources from previous run logger.info("=" * 70) logger.info(f"COMPREHENSIVE CYCLE TEST - REPLICATION {rep_idx + 1}/{num_reps}") logger.info("=" * 70) @@ -1229,7 +1241,6 @@ def test_comprehensive_cycle_performance(client_fixture, for i in range(num_initial): d = iter_daemon_factory.create(enable_forwarding=False) daemons.append(d) - time.sleep(5) all_daemons.extend(daemons) hostfile = Path(daemons[0].hostfile) @@ -1277,14 +1288,13 @@ def test_comprehensive_cycle_performance(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break - time.sleep(2) + time.sleep(MUTATE_STATUS_POLL_INTERVAL) shrink_elapsed = perf_counter() - shrink_t0 cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_shrink) - time.sleep(2) post_shrink_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, max_read_checks=1) @@ -1312,7 +1322,6 @@ def test_comprehensive_cycle_performance(client_fixture, d = iter_daemon_factory.create(expand_mode=True, enable_forwarding=False) new_daemons.append(d) - time.sleep(2) # Wait for daemon to write '+' to workspace all_daemons.extend(new_daemons) workspace_expand = hostfile.parent / f"cycle_expand_workspace_{rep_idx}.txt" @@ -1350,14 +1359,13 @@ def test_comprehensive_cycle_performance(client_fixture, cmd = gkfs_shell.script(cmd_str, intercept_shell=False) if "No mutate running/finished." in cmd.stderr.decode(): break - time.sleep(2) + time.sleep(MUTATE_STATUS_POLL_INTERVAL) expand_elapsed = perf_counter() - expand_t0 cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_expand) - time.sleep(2) post_expand_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, max_read_checks=1) -- GitLab From 86b1a8021a68fa4c131890f91c884e657afb6394 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 27 Aug 2026 20:58:20 +0200 Subject: [PATCH 09/21] test(malleability): isolate mutate hostfiles in tests Use separate mutate hostfiles for expand workflows instead of rewriting the active hostfile, and remove redundant marker rewrites from error-handling tests. This keeps daemon discovery state intact while still exercising malleability commands with the intended hostfile mutations. --- .../test_malleability_error_handling.py | 31 ------------------- .../test_malleability_tool_simple.py | 26 +++++++++------- 2 files changed, 14 insertions(+), 43 deletions(-) diff --git a/tests/integration/malleability/test_malleability_error_handling.py b/tests/integration/malleability/test_malleability_error_handling.py index 83dcf277a..78f410eff 100644 --- a/tests/integration/malleability/test_malleability_error_handling.py +++ b/tests/integration/malleability/test_malleability_error_handling.py @@ -67,17 +67,6 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - # Use marker-based approach: no --new-hosts-file needed - # Mark all entries with '+' to simulate adding same nodes (will fail gracefully) - with open(hostfile, 'r') as hf_in: - original_lines = hf_in.readlines() - with open(hostfile, 'w') as hf_out: - for line in original_lines: - if not line.startswith('#') and line.strip(): - hf_out.write('+' + line) - else: - hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) @@ -105,16 +94,6 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - # Try mutate with markers (may fail due to same node count) - with open(hostfile, 'r') as hf_in: - original_lines = hf_in.readlines() - with open(hostfile, 'w') as hf_out: - for line in original_lines: - if not line.startswith('#') and line.strip(): - hf_out.write('+' + line) - else: - hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) @@ -154,16 +133,6 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - # Start mutation with markers (fails due to same node count, which is expected) - with open(hostfile, 'r') as hf_in: - original_lines = hf_in.readlines() - with open(hostfile, 'w') as hf_out: - for line in original_lines: - if not line.startswith('#') and line.strip(): - hf_out.write('+' + line) - else: - hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) diff --git a/tests/integration/malleability/test_malleability_tool_simple.py b/tests/integration/malleability/test_malleability_tool_simple.py index 3aaf22511..37f7ea1c9 100644 --- a/tests/integration/malleability/test_malleability_tool_simple.py +++ b/tests/integration/malleability/test_malleability_tool_simple.py @@ -81,9 +81,10 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d01 = gkfwd_daemon_factory.create(expand_mode=True) time.sleep(2) d01_port = d01.address.split(":")[-1] + mutate_hostfile = hostfile.parent / "gkfs_hosts_expand_mutate.txt" with open(hostfile, "r") as hf_in: original_lines = hf_in.readlines() - with open(hostfile, "w") as hf_out: + with open(mutate_hostfile, "w") as hf_out: for line in original_lines: stripped = line.strip() if not stripped or stripped.startswith("#"): @@ -94,23 +95,23 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd(hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 for i in range(4): @@ -118,7 +119,7 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): ret = gkfs_client.stat(fpath) assert ret.retval == 0, f"stat failed for {fpath} after expand" - with open(hostfile, "r") as hf: + with open(mutate_hostfile, "r") as hf: content = hf.read() for line in content.splitlines(): stripped = line.strip() @@ -229,9 +230,10 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d01_port = d01.address.split(":")[-1] d02_port = d02.address.split(":")[-1] + mutate_hostfile = old_hostfile.parent / "gkfs_hosts_mutate_swap.txt" with open(old_hostfile, "r") as hf_in: original_lines = hf_in.readlines() - with open(old_hostfile, "w") as hf_out: + with open(mutate_hostfile, "w") as hf_out: for line in original_lines: stripped = line.strip() if not stripped or stripped.startswith("#"): @@ -244,23 +246,23 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd(old_hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(old_hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 for i in range(4): @@ -268,7 +270,7 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): ret = gkfs_client.stat(fpath) assert ret.retval == 0, f"stat failed for {fpath} after mutate swap" - with open(old_hostfile, "r") as hf: + with open(mutate_hostfile, "r") as hf: content = hf.read() for line in content.splitlines(): stripped = line.strip() -- GitLab From b45d460a9ac1bb098fd186b972d371199c00f44c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 07:49:22 +0200 Subject: [PATCH 10/21] test: use clean hostfiles in malleability tests Update malleability integration tests to run commands and clients against the intended active topology hostfiles. Add helpers to derive clean hostfiles from marker files and update client hostfile environment after mutation finalization to avoid stale or marker-based topology state. --- .../test_malleability_error_handling.py | 17 ++++--- .../test_malleability_tool_simple.py | 49 ++++++++++++++++--- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/tests/integration/malleability/test_malleability_error_handling.py b/tests/integration/malleability/test_malleability_error_handling.py index 78f410eff..0c67bbac9 100644 --- a/tests/integration/malleability/test_malleability_error_handling.py +++ b/tests/integration/malleability/test_malleability_error_handling.py @@ -67,12 +67,12 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) # Verify daemon is still running (didn't crash) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" @@ -94,12 +94,12 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): if not line.startswith("#"): hf_out.write(line) - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) # Verify mutate status works - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"shrink status after failed expand failed: {cmd.stderr.decode()}" @@ -127,17 +127,20 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s assert ret.retval == 0, f"write_validate failed for {f}" hostfile = Path(d00.hostfile) + same_hostfile = hostfile.parent / "gkfs_hosts_expand_with_data_same.txt" + with open(hostfile, "r") as hf_in, open(same_hostfile, "w") as hf_out: + hf_out.writelines(hf_in.readlines()) # Verify no running mutation - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) # Verify daemon is still running - cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" diff --git a/tests/integration/malleability/test_malleability_tool_simple.py b/tests/integration/malleability/test_malleability_tool_simple.py index 37f7ea1c9..49a283337 100644 --- a/tests/integration/malleability/test_malleability_tool_simple.py +++ b/tests/integration/malleability/test_malleability_tool_simple.py @@ -59,6 +59,29 @@ def _run_cmd(hosts_file, args, gkfs_shell, timeout=None): return cmd +def _write_clean_hostfile(marker_hostfile, clean_hostfile): + """Write final active topology from a marker hostfile.""" + with open(marker_hostfile, "r") as hf_in, open(clean_hostfile, "w") as hf_out: + for line in hf_in: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + elif stripped.startswith("-"): + continue + elif stripped.startswith("+"): + hf_out.write(line[1:]) + else: + hf_out.write(line) + + +def _set_client_hostfile(client, hostfile): + value = str(hostfile) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = value + + def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): """ Scenario: ADD (expand) @@ -113,13 +136,16 @@ def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 + clean_hostfile = hostfile.parent / "gkfs_hosts_expand_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) for i in range(4): fpath = d00.mountdir / f"expand_file_{i}" ret = gkfs_client.stat(fpath) assert ret.retval == 0, f"stat failed for {fpath} after expand" - with open(mutate_hostfile, "r") as hf: + with open(clean_hostfile, "r") as hf: content = hf.read() for line in content.splitlines(): stripped = line.strip() @@ -154,7 +180,8 @@ def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): d01_port = d01.address.split(":")[-1] with open(old_hostfile, "r") as hf_in: original_lines = hf_in.readlines() - with open(old_hostfile, "w") as hf_out: + mutate_hostfile = old_hostfile.parent / "gkfs_hosts_shrink_mutate.txt" + with open(mutate_hostfile, "w") as hf_out: for line in original_lines: stripped = line.strip() if not stripped or stripped.startswith("#"): @@ -165,31 +192,34 @@ def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): else: hf_out.write(line) - cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) assert "No mutate running/finished." in cmd.stderr.decode() - cmd = _run_cmd(old_hostfile, "mutate start", gkfs_shell, timeout=340 + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 ) assert "Mutate process" in cmd.stderr.decode() deadline = time.time() + 120 while time.time() < deadline: - cmd = _run_cmd(old_hostfile, "mutate status", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: pytest.fail("Mutate redistribution did not finish within 120 s") - cmd = _run_cmd(old_hostfile, "mutate finalize", gkfs_shell) + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 + clean_hostfile = old_hostfile.parent / "gkfs_hosts_shrink_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) for i in range(8): fpath = d00.mountdir / f"shrink_file_{i}" ret = gkfs_client.stat(fpath) assert ret.retval == 0, f"stat failed for {fpath} after shrink" - with open(old_hostfile, "r") as hf: + with open(clean_hostfile, "r") as hf: content = hf.read() for line in content.splitlines(): stripped = line.strip() @@ -264,13 +294,16 @@ def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) assert cmd.exit_code == 0 + clean_hostfile = old_hostfile.parent / "gkfs_hosts_mutate_swap_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) for i in range(4): fpath = d00.mountdir / f"mutate_file_{i}" ret = gkfs_client.stat(fpath) assert ret.retval == 0, f"stat failed for {fpath} after mutate swap" - with open(mutate_hostfile, "r") as hf: + with open(clean_hostfile, "r") as hf: content = hf.read() for line in content.splitlines(): stripped = line.strip() -- GitLab From 86f627f1605ccad52dc138ce2994a004258808b9 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 09:45:31 +0200 Subject: [PATCH 11/21] test: avoid bash wrapper in sfind integration test Run sfind and ls directly with the required environment instead of wrapping commands in bash -c. This prevents double preloading of bash and sfind, which can cause shell teardown crashes after successful sfind runs. Also count matching ls output lines in Python and remove the unused shlex import. --- tests/integration/directories/test_sfind.py | 28 ++++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/tests/integration/directories/test_sfind.py b/tests/integration/directories/test_sfind.py index 033dc0bc9..a770f0e77 100644 --- a/tests/integration/directories/test_sfind.py +++ b/tests/integration/directories/test_sfind.py @@ -3,7 +3,6 @@ import pytest import logging from harness.gkfs import Daemon, ShellClient, Client, find_command import os -import shlex import time log = logging.getLogger(__name__) @@ -83,17 +82,19 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): "LIBGKFS_LOG": "info" } - test_env_str = "\n".join([f"export {k}={v}" for k,v in test_client_env.items()]) - sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" # --- sfind Check --- log.info(f"Running sfind...") # sfind -S 1 -M - sfind_cmd = f"{test_env_str}\n{sfind_bin} {test_dir} -S 1 -M {mount_dir}" - # Use run("bash", "-c", ...) - ret = client.run("bash", "-c", sfind_cmd, timeout=SHELL_CHECK_TIMEOUT) + # Run sfind directly instead of wrapping it in `bash -c`: ShellClient + # preloads the executed process, and preloading both bash and sfind can + # make the shell crash during teardown after sfind already printed a + # successful MATCHED line. + ret = client.run( + str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir), + timeout=SHELL_CHECK_TIMEOUT, env=test_client_env) sfind_stderr = ret.stderr.decode() if ret.stderr else "" sfind_stdout = ret.stdout.decode() if ret.stdout else "" @@ -106,21 +107,18 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): log.info(f"Running ls check...") # Avoid `ls -l` here: long format stats every entry and is the first # thing to time out on overloaded CI runners. - # Expected: 2000 - ls_cmd = ( - f"{test_env_str}\n" - f"ls -1 {shlex.quote(str(test_dir))} | grep '^file_' | wc -l" - ) - ret_ls = client.run("bash", "-c", ls_cmd, timeout=SHELL_CHECK_TIMEOUT) + ret_ls = client.run( + "ls", "-1", str(test_dir), timeout=SHELL_CHECK_TIMEOUT, + env=test_client_env) ls_stderr = ret_ls.stderr.decode() if ret_ls.stderr else "" ls_stdout = ret_ls.stdout.decode() if ret_ls.stdout else "" assert ret_ls.exit_code == 0, f"ls check failed with {ret_ls.exit_code}\nStderr: {ls_stderr}" - # parse count - count = ls_stdout.strip() - assert count == "2000", f"ls count expected 2000, got '{count}'" + count = sum(1 for line in ls_stdout.splitlines() + if line.startswith("file_")) + assert count == 2000, f"ls count expected 2000, got '{count}'" log.info("ls verification successful.") finally: -- GitLab From 1cedd02ab4863b9b2c75cc81dae0032f2c571c77 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 10:47:09 +0200 Subject: [PATCH 12/21] fix: validate dirent buffer sizes and add shrink cutshift Guard dirent decompression/parsing against server-reported payload sizes that exceed the exposed client buffer to avoid unsafe reads. Add cutshift shrink support for malleability, including survivor host id compaction and partition redistribution for removed hosts. --- include/client/rpc/utils.hpp | 12 +- include/common/rpc/cutshift_sorted.hpp | 14 ++ src/client/gkfs_metadata.cpp | 3 +- src/client/rpc/forward_metadata.cpp | 17 +- src/client/rpc/forward_metadata_proxy.cpp | 2 +- src/common/rpc/cutshift_sorted.cpp | 180 ++++++++++++++++++ src/common/rpc/random_slicing_distributor.cpp | 29 ++- src/daemon/handler/srv_metadata.cpp | 5 +- src/daemon/malleability/malleable_manager.cpp | 142 ++++++++++++-- tests/unit/test_random_slicing_pipeline.cpp | 83 ++++++++ tools/malleability_simulator.cpp | 68 ++++++- 11 files changed, 526 insertions(+), 29 deletions(-) diff --git a/include/client/rpc/utils.hpp b/include/client/rpc/utils.hpp index 80bd300a7..d2f31072a 100644 --- a/include/client/rpc/utils.hpp +++ b/include/client/rpc/utils.hpp @@ -37,6 +37,7 @@ #include #include #include +#include namespace gkfs::rpc { @@ -54,8 +55,9 @@ namespace gkfs::rpc { */ template std::vector> -decompress_and_parse_entries(const OutputOrErr& out, - const void* compressed_buffer) { +decompress_and_parse_entries( + const OutputOrErr& out, const void* compressed_buffer, + std::size_t buffer_size = std::numeric_limits::max()) { if(out.err != 0) { throw std::runtime_error("Server returned an error: " + std::to_string(out.err)); @@ -63,6 +65,12 @@ decompress_and_parse_entries(const OutputOrErr& out, if(out.dirents_size == 0) { return {}; } + if(out.dirents_size > buffer_size) { + throw std::runtime_error( + "Server returned dirents payload larger than exposed client buffer: " + + std::to_string(out.dirents_size) + " > " + + std::to_string(buffer_size)); + } const char* p = nullptr; const char* end = nullptr; diff --git a/include/common/rpc/cutshift_sorted.hpp b/include/common/rpc/cutshift_sorted.hpp index 82f3c02f1..0207aece0 100644 --- a/include/common/rpc/cutshift_sorted.hpp +++ b/include/common/rpc/cutshift_sorted.hpp @@ -59,6 +59,20 @@ expand_with_cutshift(std::vector current_partitions, const std::vector& new_hosts, float old_total_capacity, float new_total_capacity); +/// Shrink the cluster with minimum data movement. +/// +/// Existing survivor intervals stay owned by the same physical survivor. Output +/// host ids are compacted to the post-shrink rank order, i.e. sorted survivor +/// old ids map to 0..survivor_count-1. Only intervals owned by removed hosts +/// are redistributed to survivors to restore equal capacity. +/// @param current_partitions Current pre-shrink partition table using old host +/// ids +/// @param removed_hosts Old host ids that disappear after mutate finishes +/// @return Post-shrink compact partition table for surviving nodes +std::vector +shrink_with_cutshift(std::vector current_partitions, + const std::vector& removed_hosts); + } // namespace rpc } // namespace gkfs diff --git a/src/client/gkfs_metadata.cpp b/src/client/gkfs_metadata.cpp index c611fd09b..ba31c629d 100644 --- a/src/client/gkfs_metadata.cpp +++ b/src/client/gkfs_metadata.cpp @@ -1259,7 +1259,8 @@ gkfs_opendir(const std::string& path) { } auto entries = gkfs::rpc::decompress_and_parse_entries( - out, buffers[buffer_id].data()); + out, buffers[buffer_id].data(), + buffers[buffer_id].size()); consume_entries(entries); if(!entries.empty()) { const auto& last_key = std::get<0>(entries.back()); diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index 7fe5460ac..a18168474 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -334,6 +334,7 @@ forward_mk_symlink(const std::string& path, const std::string& target_path) { template std::pair decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, + std::size_t buffer_size, std::vector& decompressed_data) { if(out.err != 0) { throw std::runtime_error("Server returned an error: " + @@ -342,6 +343,12 @@ decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, if(out.dirents_size == 0) { return {nullptr, 0}; } + if(out.dirents_size > buffer_size) { + throw std::runtime_error( + "Server returned dirents payload larger than exposed client buffer: " + + std::to_string(out.dirents_size) + " > " + + std::to_string(buffer_size)); + } if(gkfs::config::rpc::use_dirents_compression) { const unsigned long long uncompressed_size = @@ -380,10 +387,10 @@ decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, inline std::vector> decompress_and_parse_entries_standard( const gkfs::rpc::rpc_get_dirents_out_t& out, - const void* compressed_buffer) { + const void* compressed_buffer, std::size_t buffer_size) { std::vector decompressed_data; auto [payload, payload_size] = decompress_dirents_payload( - out, compressed_buffer, decompressed_data); + out, compressed_buffer, buffer_size, decompressed_data); if(payload_size == 0) { return {}; } @@ -503,8 +510,8 @@ forward_get_dirents(const string& path) { // Decompress and parse entries // The decompress function expects rpc_get_dirents_out_t // which matches the Thallium RPC output. - auto entries = - decompress_and_parse_entries_standard(out, base_ptr); + auto entries = decompress_and_parse_entries_standard( + out, base_ptr, per_host_buff_size); for(auto& e : entries) { auto type = get<1>(e); gkfs::filemap::FileType ftype = @@ -835,7 +842,7 @@ forward_get_dirents_single(const string& path, int server, } auto current_entries = gkfs::rpc::decompress_and_parse_entries( - out, large_buffer.data()); + out, large_buffer.data(), buffer_size); if(current_entries.empty()) { return make_pair(0, std::move(all_entries)); diff --git a/src/client/rpc/forward_metadata_proxy.cpp b/src/client/rpc/forward_metadata_proxy.cpp index b4148db72..16900d0ea 100644 --- a/src/client/rpc/forward_metadata_proxy.cpp +++ b/src/client/rpc/forward_metadata_proxy.cpp @@ -225,7 +225,7 @@ forward_get_dirents_single_proxy_v2(const string& path, int server, try { // Here we still assume the buffer is populated by RMA auto entries_vector = gkfs::rpc::decompress_and_parse_entries( - out, large_buffer.data()); + out, large_buffer.data(), large_buffer.size()); if(entries_vector.empty()) { return make_pair(0, std::move(all_entries)); diff --git a/src/common/rpc/cutshift_sorted.cpp b/src/common/rpc/cutshift_sorted.cpp index e608f0019..65cfd2829 100644 --- a/src/common/rpc/cutshift_sorted.cpp +++ b/src/common/rpc/cutshift_sorted.cpp @@ -21,6 +21,7 @@ #include "common/rpc/cutshift_sorted.hpp" #include #include +#include namespace gkfs { namespace rpc { @@ -73,6 +74,38 @@ sort_gaps_largest_first(std::vector& gaps) { }); } +void +sort_intervals_by_start(std::vector& intervals) { + std::sort(intervals.begin(), intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); +} + +void +merge_adjacent_same_owner(std::vector& intervals) { + if(intervals.empty()) { + return; + } + + sort_intervals_by_start(intervals); + + std::vector merged; + merged.reserve(intervals.size()); + merged.push_back(intervals.front()); + + for(size_t i = 1; i < intervals.size(); ++i) { + auto& last = merged.back(); + const auto& current = intervals[i]; + if(last.host_id == current.host_id && + current.start <= last.end + epsilon) { + last.end = std::max(last.end, current.end); + } else if(interval_size(current) > epsilon) { + merged.push_back(current); + } + } + + intervals = std::move(merged); +} + std::vector collect_gaps_cutshift_in_place( std::vector& partitions, @@ -312,5 +345,152 @@ expand_with_cutshift(std::vector current_partitions, return current_partitions; } +std::vector +shrink_with_cutshift(std::vector current_partitions, + const std::vector& removed_hosts) { + + if(removed_hosts.empty()) { + return current_partitions; + } + + std::unordered_set removed(removed_hosts.begin(), + removed_hosts.end()); + + std::vector survivors_old_ids; + survivors_old_ids.reserve(current_partitions.size()); + std::vector evacuated; + float evacuated_capacity = 0.0f; + + for(const auto& partition : current_partitions) { + if(removed.find(partition.host_id) != removed.end()) { + for(auto interval : partition.intervals) { + if(interval_size(interval) <= epsilon) { + continue; + } + interval.host_id = 0; + evacuated_capacity += interval_size(interval); + evacuated.push_back(interval); + } + } else { + survivors_old_ids.push_back(partition.host_id); + } + } + + if(survivors_old_ids.empty()) { + return {}; + } + + std::sort(survivors_old_ids.begin(), survivors_old_ids.end()); + + std::unordered_map old_to_new; + old_to_new.reserve(survivors_old_ids.size()); + for(size_t i = 0; i < survivors_old_ids.size(); ++i) { + old_to_new.emplace(survivors_old_ids[i], static_cast(i)); + } + + std::vector survivors(survivors_old_ids.size()); + for(size_t i = 0; i < survivors_old_ids.size(); ++i) { + survivors[i].host_id = static_cast(i); + } + + for(const auto& partition : current_partitions) { + const auto map_it = old_to_new.find(partition.host_id); + if(map_it == old_to_new.end()) { + continue; + } + + auto& survivor = survivors[map_it->second]; + for(auto interval : partition.intervals) { + if(interval_size(interval) <= epsilon) { + continue; + } + interval.host_id = survivor.host_id; + survivor.intervals.push_back(interval); + } + survivor.total_capacity = survivor.coverage(); + } + + if(evacuated.empty()) { + for(auto& survivor : survivors) { + survivor.total_capacity = survivor.coverage(); + merge_adjacent_same_owner(survivor.intervals); + } + return survivors; + } + + merge_adjacent_gaps(evacuated); + sort_gaps_largest_first(evacuated); + + const auto target_capacity = 1.0f / static_cast(survivors.size()); + std::vector deficits(survivors.size(), 0.0f); + float total_deficit = 0.0f; + for(size_t i = 0; i < survivors.size(); ++i) { + deficits[i] = std::max(0.0f, target_capacity - survivors[i].coverage()); + total_deficit += deficits[i]; + } + + // Floating point/comments may not add up exactly. Scale demands to match + // evacuated capacity, so every removed interval is assigned before removed + // daemons disappear at mutate-finalize time. + if(total_deficit > epsilon && evacuated_capacity > epsilon) { + const auto scale = evacuated_capacity / total_deficit; + for(auto& deficit : deficits) { + deficit *= scale; + } + } + + size_t gap_idx = 0; + for(size_t survivor_idx = 0; + survivor_idx < survivors.size() && gap_idx < evacuated.size(); + ++survivor_idx) { + auto needed = deficits[survivor_idx]; + while(needed > epsilon && gap_idx < evacuated.size()) { + auto gap = evacuated[gap_idx]; + const auto gap_size = interval_size(gap); + if(gap_size <= epsilon) { + ++gap_idx; + continue; + } + + if(gap_size <= needed + epsilon) { + gap.host_id = survivors[survivor_idx].host_id; + survivors[survivor_idx].intervals.push_back(gap); + needed -= gap_size; + ++gap_idx; + } else { + Interval piece = gap; + piece.end = piece.start + needed; + piece.host_id = survivors[survivor_idx].host_id; + survivors[survivor_idx].intervals.push_back(piece); + evacuated[gap_idx].start = piece.end; + needed = 0.0f; + } + } + } + + // If rounding left bytes of evacuated ownership, append them to + // least-loaded survivors. This keeps full coverage and avoids dangling + // ownership on nodes that will disappear after mutate completes. + while(gap_idx < evacuated.size()) { + auto gap = evacuated[gap_idx++]; + if(interval_size(gap) <= epsilon) { + continue; + } + auto target = std::min_element(survivors.begin(), survivors.end(), + [](const auto& a, const auto& b) { + return a.coverage() < b.coverage(); + }); + gap.host_id = target->host_id; + target->intervals.push_back(gap); + } + + for(auto& survivor : survivors) { + merge_adjacent_same_owner(survivor.intervals); + survivor.total_capacity = survivor.coverage(); + } + + return survivors; +} + } // namespace rpc } // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/random_slicing_distributor.cpp b/src/common/rpc/random_slicing_distributor.cpp index e54d50b3b..bc2207f77 100644 --- a/src/common/rpc/random_slicing_distributor.cpp +++ b/src/common/rpc/random_slicing_distributor.cpp @@ -309,7 +309,29 @@ RandomSlicingDistributor::add_nodes(std::vector new_nodes) { void RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { - // Remove hosts from all_hosts_ + if(old_nodes.empty()) { + return; + } + + if(random_slicing_cutshift_enabled() && + old_nodes.size() < all_hosts_.size() && !partitions_.empty()) { + auto updated_partitions = shrink_with_cutshift(partitions_, old_nodes); + if(!updated_partitions.empty()) { + hosts_size_ = static_cast(updated_partitions.size()); + all_hosts_.clear(); + all_hosts_.reserve(hosts_size_); + for(host_t h = 0; h < hosts_size_; ++h) { + all_hosts_.push_back(h); + } + partitions_ = std::move(updated_partitions); + interval_idx_.build(partitions_); + return; + } + } + + // Remove hosts from all_hosts_ and compact ids: removed daemons disappear + // once mutate is finalized, so the post-shrink layout uses new ranks + // 0..N-1. for(auto h : old_nodes) { auto it = std::find(all_hosts_.begin(), all_hosts_.end(), h); if(it != all_hosts_.end()) { @@ -318,6 +340,11 @@ RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { } hosts_size_ = static_cast(all_hosts_.size()); + all_hosts_.clear(); + all_hosts_.reserve(hosts_size_); + for(host_t h = 0; h < hosts_size_; ++h) { + all_hosts_.push_back(h); + } // Recompute partitions init_partitions_from_hosts(); diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index e7522cc27..6e6891dca 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -747,6 +747,8 @@ rpc_srv_get_dirents(const std::shared_ptr& engine, GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' ", __func__, in.path); + const auto client_bulk_size = in.bulk_handle.size(); + std::vector> entries{}; try { entries = gkfs::metadata::get_dirents(in.path); @@ -761,7 +763,8 @@ rpc_srv_get_dirents(const std::shared_ptr& engine, "{}() path '{}' Read database with '{}' entries", __func__, in.path, entries.size()); - get_dirents_helper(engine, req, entries, 0, in.bulk_handle, out); + get_dirents_helper(engine, req, entries, client_bulk_size, in.bulk_handle, + out); if(GKFS_DATA->enable_stats()) { GKFS_DATA->stats()->add_value_iops( diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index f59d79d80..e7a32df41 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -130,6 +130,85 @@ make_equal_rs_partitions_for_before_hosts( return partitions; } +static vector +make_equal_rs_partitions_for_before_hosts( + const vector>& before_hosts) { + vector partitions; + partitions.reserve(before_hosts.size()); + const auto n = static_cast(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + const auto start = static_cast(old_id) / n; + const auto end = static_cast(old_id + 1) / n; + gkfs::rpc::Partition partition; + partition.host_id = static_cast(old_id); + partition.total_capacity = end - start; + partition.intervals.push_back( + {start, end, static_cast(old_id)}); + partitions.push_back(partition); + } + return partitions; +} + +static vector +rs_comments_to_old_id_intervals( + const vector& comments, + const vector>& before_hosts) { + vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + if(comment.host_id >= before_hosts.size()) { + return {}; + } + intervals.push_back({static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + return intervals; +} + +static vector +remap_compact_survivor_partitions_to_after_ids( + vector partitions, + const vector>& before_hosts, + const vector>& after_hosts, + const vector& removed_old_ids) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + unordered_map removed; + for(auto old_id : removed_old_ids) { + removed.emplace(old_id, true); + } + + vector survivor_old_ids; + survivor_old_ids.reserve(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + if(removed.find(static_cast(old_id)) == + removed.end()) { + survivor_old_ids.push_back(static_cast(old_id)); + } + } + + for(auto& partition : partitions) { + if(partition.host_id >= survivor_old_ids.size()) { + return {}; + } + const auto old_id = survivor_old_ids[partition.host_id]; + const auto after_it = after_ids.find(host_key(before_hosts[old_id])); + if(after_it == after_ids.end()) { + return {}; + } + partition.host_id = after_it->second; + for(auto& interval : partition.intervals) { + interval.host_id = partition.host_id; + } + } + return partitions; +} + static vector remap_existing_rs_comments( const vector& comments, @@ -717,6 +796,11 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, before_keys.emplace(host_key(host), true); } + unordered_map after_keys; + for(const auto& host : hosts) { + after_keys.emplace(host_key(host), true); + } + vector added_ids; for(size_t i = 0; i < hosts.size(); ++i) { if(before_keys.find(host_key(hosts[i])) == before_keys.end()) { @@ -724,28 +808,60 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, } } + vector removed_old_ids; + for(size_t i = 0; i < before_hosts.size(); ++i) { + if(after_keys.find(host_key(before_hosts[i])) == after_keys.end()) { + removed_old_ids.push_back(static_cast(i)); + } + } + auto* rs = dynamic_cast( distributor.get()); vector final_intervals; const bool cutshift = env_truthy(std::getenv(gkfs::env::RANDOM_SLICING_CUTSHIFT)); - if(rs && cutshift && markers.removing.empty() && !added_ids.empty() && + if(rs && cutshift && (!added_ids.empty() || !removed_old_ids.empty()) && !before_hosts.empty()) { auto comments = gkfs::malleable::parse_rs_interval_comments(new_hosts_file); - auto old_intervals = - comments.empty() ? vector{} - : remap_existing_rs_comments( - comments, before_hosts, hosts); - auto old_partitions = - old_intervals.empty() - ? make_equal_rs_partitions_for_before_hosts( - before_hosts, hosts) - : intervals_to_partitions(old_intervals); - - auto updated_partitions = gkfs::rpc::expand_with_cutshift( - old_partitions, added_ids, 1.0f, 1.0f); + vector updated_partitions; + + if(removed_old_ids.empty()) { + auto old_intervals = + comments.empty() + ? vector{} + : remap_existing_rs_comments( + comments, before_hosts, hosts); + updated_partitions = + old_intervals.empty() + ? make_equal_rs_partitions_for_before_hosts( + before_hosts, hosts) + : intervals_to_partitions(old_intervals); + } else { + auto old_intervals = comments.empty() + ? vector{} + : rs_comments_to_old_id_intervals( + comments, before_hosts); + auto old_partitions = + old_intervals.empty() + ? make_equal_rs_partitions_for_before_hosts( + before_hosts) + : intervals_to_partitions(old_intervals); + + updated_partitions = gkfs::rpc::shrink_with_cutshift( + old_partitions, removed_old_ids); + updated_partitions = + remap_compact_survivor_partitions_to_after_ids( + std::move(updated_partitions), before_hosts, + hosts, removed_old_ids); + } + + if(!added_ids.empty() && !updated_partitions.empty()) { + updated_partitions = gkfs::rpc::expand_with_cutshift( + updated_partitions, added_ids, 1.0f, 1.0f); + } + for(const auto& partition : updated_partitions) { for(const auto& interval : partition.intervals) { final_intervals.push_back(interval); diff --git a/tests/unit/test_random_slicing_pipeline.cpp b/tests/unit/test_random_slicing_pipeline.cpp index 063dd74d6..2f9f855d5 100644 --- a/tests/unit/test_random_slicing_pipeline.cpp +++ b/tests/unit/test_random_slicing_pipeline.cpp @@ -30,6 +30,7 @@ #include #include #include +#include using namespace gkfs::rpc; @@ -179,6 +180,88 @@ TEST_CASE("Pipeline: CutShift+Sorted preserves full coverage without overlap", } } +TEST_CASE("Pipeline: CutShift shrink preserves survivor ownership", + "[pipeline][cutshift][random_slicing][shrink]") { + std::vector old_partitions; + for(host_t host = 0; host < 4; ++host) { + const auto start = static_cast(host) / 4.0f; + const auto end = static_cast(host + 1) / 4.0f; + Partition partition; + partition.host_id = host; + partition.total_capacity = end - start; + partition.intervals.push_back({start, end, host}); + old_partitions.push_back(partition); + } + + auto new_partitions = shrink_with_cutshift(old_partitions, {3}); + + REQUIRE(new_partitions.size() == 3); + for(host_t host = 0; host < 3; ++host) { + REQUIRE(new_partitions[host].host_id == host); + REQUIRE(new_partitions[host].coverage() == Catch::Approx(1.0f / 3.0f)); + bool kept_original = false; + for(const auto& interval : new_partitions[host].intervals) { + REQUIRE(interval.host_id == host); + if(interval.start == Catch::Approx(static_cast(host) / 4.0f) && + interval.end == Catch::Approx(static_cast(host + 1) / 4.0f)) { + kept_original = true; + } + } + REQUIRE(kept_original); + } + + std::vector intervals; + for(const auto& partition : new_partitions) { + intervals.insert(intervals.end(), partition.intervals.begin(), + partition.intervals.end()); + } + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } +} + +TEST_CASE("Pipeline: RandomSlicing CutShift shrink only moves removed-node chunks", + "[pipeline][cutshift][random_slicing][shrink]") { + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + RandomSlicingDistributor old_dist(0, 4); + RandomSlicingDistributor new_dist(0, 4); + new_dist.remove_nodes({3}); + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + + REQUIRE(new_dist.hosts_size() == 3); + + constexpr uint64_t files = 128; + constexpr uint64_t chunks = 128; + uint64_t moved = 0; + uint64_t moved_from_survivors = 0; + std::unordered_set targets; + + for(uint64_t file = 0; file < files; ++file) { + const auto path = "/rs-shrink/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < chunks; ++chunk) { + const auto old_host = old_dist.locate_data(path, chunk, 0); + const auto new_host = new_dist.locate_data(path, chunk, 0); + REQUIRE(new_host < 3); + if(old_host != new_host) { + ++moved; + targets.insert(new_host); + if(old_host < 3) { + ++moved_from_survivors; + } + } + } + } + + REQUIRE(moved > 0); + REQUIRE(targets.size() == 3); + REQUIRE(moved_from_survivors == 0); +} + TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", "[pipeline][migrator][random_slicing]") { auto rs = create_distributor_from_string("random_slicing", 0, 3); diff --git a/tools/malleability_simulator.cpp b/tools/malleability_simulator.cpp index b71b184d9..9d044641b 100644 --- a/tools/malleability_simulator.cpp +++ b/tools/malleability_simulator.cpp @@ -58,6 +58,12 @@ struct simulation_result { uint64_t total_bytes = 0; uint64_t moved_chunks = 0; uint64_t moved_bytes = 0; + uint64_t moved_from_removed_chunks = 0; + uint64_t moved_from_removed_bytes = 0; + uint64_t moved_from_survivor_chunks = 0; + uint64_t moved_from_survivor_bytes = 0; + double shrink_lower_bound_ratio = 0.0; + double extra_survivor_movement_ratio = 0.0; double movement_ratio = 0.0; double elapsed_ms = 0.0; std::vector moved_from; @@ -160,6 +166,13 @@ simulate(simulator_config cfg) { chunk_bytes(cfg.file_size, cfg.chunk_size, chunk); ++result.moved_chunks; result.moved_bytes += bytes; + if(cfg.new_nodes < cfg.old_nodes && old_host >= cfg.new_nodes) { + ++result.moved_from_removed_chunks; + result.moved_from_removed_bytes += bytes; + } else { + ++result.moved_from_survivor_chunks; + result.moved_from_survivor_bytes += bytes; + } result.moved_from[old_host].chunks++; result.moved_from[old_host].bytes += bytes; result.received_by[new_host].chunks++; @@ -174,6 +187,14 @@ simulate(simulator_config cfg) { ? 0.0 : static_cast(result.moved_bytes) / static_cast(result.total_bytes); + if(cfg.new_nodes < cfg.old_nodes && result.total_bytes != 0) { + result.shrink_lower_bound_ratio = + static_cast(cfg.old_nodes - cfg.new_nodes) / + static_cast(cfg.old_nodes); + result.extra_survivor_movement_ratio = + static_cast(result.moved_from_survivor_bytes) / + static_cast(result.total_bytes); + } return result; } @@ -190,8 +211,26 @@ print_single_result(const simulation_result& result) { std::cout << "total_bytes=" << result.total_bytes << "\n"; std::cout << "moved_chunks=" << result.moved_chunks << "\n"; std::cout << "moved_bytes=" << result.moved_bytes << "\n"; + std::cout << "moved_from_removed_chunks=" + << result.moved_from_removed_chunks << "\n"; + std::cout << "moved_from_removed_bytes=" << result.moved_from_removed_bytes + << "\n"; + std::cout << "moved_from_survivor_chunks=" + << result.moved_from_survivor_chunks << "\n"; + std::cout << "moved_from_survivor_bytes=" + << result.moved_from_survivor_bytes << "\n"; std::cout << std::fixed << std::setprecision(6) << "movement_ratio=" << result.movement_ratio << "\n"; + if(result.new_nodes < result.old_nodes) { + std::cout << "shrink_lower_bound_ratio=" + << result.shrink_lower_bound_ratio << "\n"; + std::cout << "extra_survivor_movement_ratio=" + << result.extra_survivor_movement_ratio << "\n"; + std::cout + << "definition_moved_from_removed=chunks whose old owner is a removed daemon; unavoidable shrink movement\n"; + std::cout + << "definition_moved_from_survivor=chunks whose old owner is a surviving daemon; extra remap movement\n"; + } std::cout << "planning_time_ms=" << result.elapsed_ms << "\n"; print_host_stats("per_source_moved", result.moved_from); print_host_stats("per_target_received", result.received_by); @@ -206,21 +245,40 @@ print_matrix(const std::vector& results) { << " file_size=" << results.front().file_size << " chunk_size=" << results.front().chunk_size << "\n\n"; + const bool is_shrink = results.front().new_nodes < results.front().old_nodes; + if(is_shrink) { + std::cout + << "removed_mib: moved data whose old owner is a removed daemon (unavoidable shrink lower-bound component)\n"; + std::cout + << "survivor_mib: moved data whose old owner is a surviving daemon (extra remap/rebalance movement)\n\n"; + } + std::cout << std::left << std::setw(18) << "strategy" << std::setw(10) << "cutshift" << std::right << std::setw(14) << "moved_bytes" - << std::setw(14) << "moved_mib" << std::setw(14) - << "moved_chunks" << std::setw(12) << "ratio_%" + << std::setw(14) << (is_shrink ? "removed_mib" : "moved_mib") + << std::setw(14) << (is_shrink ? "survivor_mib" : "extra_mib") + << std::setw(14) << "moved_chunks" + << std::setw(12) << "ratio_%" << std::setw(12) << "time_ms" << "\n"; - std::cout << std::string(94, '-') << "\n"; + std::cout << std::string(108, '-') << "\n"; for(const auto& result : results) { const auto moved_mib = static_cast(result.moved_bytes) / (1024.0 * 1024.0); + const auto removed_mib = + static_cast(result.moved_from_removed_bytes) / + (1024.0 * 1024.0); + const auto survivor_mib = + static_cast(result.moved_from_survivor_bytes) / + (1024.0 * 1024.0); std::cout << std::left << std::setw(18) << result.strategy << std::setw(10) << (result.cutshift ? "ON" : "OFF") << std::right << std::setw(14) << result.moved_bytes << std::setw(14) << std::fixed << std::setprecision(3) - << moved_mib << std::setw(14) << result.moved_chunks + << (result.new_nodes < result.old_nodes ? removed_mib + : moved_mib) + << std::setw(14) << survivor_mib << std::setw(14) + << result.moved_chunks << std::setw(12) << std::fixed << std::setprecision(3) << (result.movement_ratio * 100.0) << std::setw(12) << std::fixed << std::setprecision(3) << result.elapsed_ms @@ -244,7 +302,7 @@ main(int argc, char* argv[]) { "distribution strategy: simple_hash|random_slicing. If omitted, run comparison matrix") ->check(CLI::IsMember({"simple_hash", "random_slicing"})); app.add_flag("--cutshift", cfg.cutshift, - "use Random Slicing CutShift for expand simulations"); + "use Random Slicing CutShift for expand/shrink simulations"); app.add_option("--file-size", cfg.file_size, "bytes per file") ->check(CLI::PositiveNumber); app.add_option("--chunk-size", cfg.chunk_size, "bytes per chunk") -- GitLab From b3c7e3fbca8c0d8bb44d6d19a15be62bcd694188 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 13:03:47 +0200 Subject: [PATCH 13/21] ci: split integration tests into separate pipeline job Move malleability, forwarding, concurrency, and resilience tests from gkfs:integration-2 into a new gkfs:integration-3 job with dedicated JUnit and coverage output. Update the coverage report dependencies to include the new integration job, reducing load and isolating slower test groups in CI. --- .gitlab-ci.yml | 62 ++++++-- README.md | 2 + tests/apps/lockfile.sh | 2 +- .../directories/test_packing_order.py | 77 ++++----- tests/integration/directories/test_sfind.py | 78 ++++------ .../directories/test_sfind_diagnostic.py | 147 ++++++++++++++++++ .../directories/test_sfind_filtered.py | 81 +++++----- .../integration/directories/test_symlinks.py | 14 +- tests/integration/harness/CMakeLists.txt | 1 + .../integration/harness/gkfs.io/commands.hpp | 3 + tests/integration/harness/gkfs.io/main.cpp | 1 + .../integration/harness/gkfs.io/readlink.cpp | 120 ++++++++++++++ tests/integration/harness/io.py | 13 ++ .../operations/test_client_cache.py | 46 +++--- tests/integration/syscalls/test_config_env.py | 143 ++++++----------- 15 files changed, 523 insertions(+), 267 deletions(-) create mode 100644 tests/integration/directories/test_sfind_diagnostic.py create mode 100644 tests/integration/harness/gkfs.io/readlink.cpp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c71f0439e..05e49f301 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,12 +172,6 @@ gkfs:integration-2: script: ## run tests - export PATH=${PATH}:/usr/local/bin - - export GKFS_MALLEABILITY_CI_FAST=ON - - export GKFS_PERF_REPETITIONS=1 - - export GKFS_PERF_NUM_FILES=1 - - export GKFS_PERF_FILE_SIZE=$((1024 * 1024)) - - export GKFS_MALLEABILITY_OLD_NODES=2 - - export GKFS_MALLEABILITY_NEW_NODES=1 - mkdir -p ${BUILD_PATH}/tests/run - cd ${BUILD_PATH}/tests/integration - ${PYTEST} -v -n $(nproc) @@ -187,10 +181,6 @@ gkfs:integration-2: ${INTEGRATION_TESTS_BIN_PATH}/rename ${INTEGRATION_TESTS_BIN_PATH}/position ${INTEGRATION_TESTS_BIN_PATH}/status - ${INTEGRATION_TESTS_BIN_PATH}/malleability - ${INTEGRATION_TESTS_BIN_PATH}/forwarding - ${INTEGRATION_TESTS_BIN_PATH}/concurrency - ${INTEGRATION_TESTS_BIN_PATH}/resilience --basetemp=${BUILD_PATH}/tests/run/ --junit-xml=report-2.xml @@ -218,6 +208,56 @@ gkfs:integration-2: reports: junit: ${BUILD_PATH}/tests/integration/report-2.xml + +gkfs:integration-3: + stage: test + image: ${TESTING} + interruptible: true + needs: ['gkfs'] + + script: + ## run tests + - export PATH=${PATH}:/usr/local/bin + - export GKFS_MALLEABILITY_CI_FAST=ON + - export GKFS_PERF_REPETITIONS=1 + - export GKFS_PERF_NUM_FILES=1 + - export GKFS_PERF_FILE_SIZE=$((1024 * 1024)) + - export GKFS_MALLEABILITY_OLD_NODES=2 + - export GKFS_MALLEABILITY_NEW_NODES=1 + - mkdir -p ${BUILD_PATH}/tests/run + - cd ${BUILD_PATH}/tests/integration + - ${PYTEST} -v -n $(nproc) + ${INTEGRATION_TESTS_BIN_PATH}/malleability + ${INTEGRATION_TESTS_BIN_PATH}/forwarding + ${INTEGRATION_TESTS_BIN_PATH}/concurrency + ${INTEGRATION_TESTS_BIN_PATH}/resilience + --basetemp=${BUILD_PATH}/tests/run/ + --junit-xml=report-3.xml + + ## capture coverage information + - cd ${CI_PROJECT_DIR} + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration3.info + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + after_script: + - perl -i.orig + -pe 's%file="(.*?)"%file="tests/integration/$1"%g;' + -pe 's%(../)+install/share/gkfs/%%g;' + ${BUILD_PATH}/tests/integration/report-3.xml + + artifacts: + expire_in: 1 day + when: always + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/integration/report-3.xml + ## == integration tests for gkfs =========== gkfs:integration: stage: test @@ -572,7 +612,7 @@ coverage: stage: report image: ${TESTING} #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] + needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:integration-3', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] script: # use ccache - ccache --zero-stats diff --git a/README.md b/README.md index f2ef4d5da..16f36ae9e 100644 --- a/README.md +++ b/README.md @@ -566,6 +566,8 @@ For `random_slicing`, `mutate start` writes the chosen interval table into the w CutShift is applied for expand-only Random Slicing operations (`+` entries present, no `-` entries). Other Random Slicing mutate cases currently fall back to the final equal RS layout. +Reference: Random Slicing is based on "Random slicing: Efficient and scalable data placement for large-scale storage systems" by Alberto Miranda, Sascha Effert, Yangwook Kang, Ethan L. Miller, Ivan Popov, Andre Brinkmann, Tom Friedetzky, and Toni Cortes, published in ACM Transactions on Storage 10(3), 2014. See the [Google Scholar entry](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=hK-ogNAAAAAJ&cstart=20&pagesize=80&sortby=pubdate&citation_for_view=hK-ogNAAAAAJ:RGFaLdJalmkC). + ### Manual `gkfs_malleability` workflow Use this when you manage daemon startup and hostfile markers yourself. diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index 7b4e83a74..0153f8c68 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -46,7 +46,7 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfi export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & GENERATOR=$! sleep 1 OUTPUTLS=`LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC ls $MNT/syscall/` diff --git a/tests/integration/directories/test_packing_order.py b/tests/integration/directories/test_packing_order.py index bd0d8d3f5..afe74122b 100644 --- a/tests/integration/directories/test_packing_order.py +++ b/tests/integration/directories/test_packing_order.py @@ -1,11 +1,12 @@ - -import pytest import logging +import re import time -from harness.gkfs import Daemon, ShellClient, Client, find_command + +from harness.gkfs import Client, Daemon, ShellClient, find_command log = logging.getLogger(__name__) + def test_packing_order_all_fields(test_workspace, request): """ Comprehensive test to verify the packing order of compressed directory entries. @@ -16,11 +17,13 @@ def test_packing_order_all_fields(test_workspace, request): "GKFS_DAEMON_LOG_LEVEL": "info", "GKFS_USE_DIRENTS_COMPRESSION": "ON" } - daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() try: - client = ShellClient(test_workspace) + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir client_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} @@ -29,91 +32,93 @@ def test_packing_order_all_fields(test_workspace, request): # file_b: size 200 # (wait) # file_c: size 100 - + file_a = mount_dir / "file_a" file_b = mount_dir / "file_b" file_c = mount_dir / "file_c" - - # Use dd to create files with specific sizes - client.run("dd", "if=/dev/zero", f"of={file_a}", "bs=100", "count=1", env=client_env) - client.run("dd", "if=/dev/zero", f"of={file_b}", "bs=200", "count=1", env=client_env) - + + ret = io_client.run("write_sequential", "--pathname", str(file_a), + "--count", "1", "--size", "100", env=client_env) + assert ret.retval == 0 + ret = io_client.run("write_sequential", "--pathname", str(file_b), + "--count", "1", "--size", "200", env=client_env) + assert ret.retval == 0 + log.info("Waiting 2 seconds to distinguish ctimes...") time.sleep(2) log.info("Resuming test after sleep.") - + # Create a timestamp file for 'newer' check timestamp_file = test_workspace.twd / "timestamp" timestamp_file.touch() - + time.sleep(1) - client.run("dd", "if=/dev/zero", f"of={file_c}", "bs=100", "count=1", env=client_env) - + ret = io_client.run("write_sequential", "--pathname", str(file_c), + "--count", "1", "--size", "100", env=client_env) + assert ret.retval == 0 + sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - + common_args = ["-M", str(mount_dir), "-S", "1", "--server-side"] - import re def check_match(ret, expected_matches, expected_total=3): output = ret.stdout.decode() log.info(f"SFIND OUTPUT:\n{output}") match = re.search(r"MATCHED (\d+)/(\d+)", output) if not match: assert False, f"Could not find MATCHED line in output:\n{output}" - + actual_matches = int(match.group(1)) total_checked = int(match.group(2)) - + log.info(f"Actual matches: {actual_matches}, Total checked: {total_checked}") - assert actual_matches == expected_matches, f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}" - assert total_checked == expected_total, f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}" + assert actual_matches == expected_matches, \ + f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}" + assert total_checked == expected_total, \ + f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}" # Scenario 1: Filter by Name log.info("Testing -name filter...") - ret = client.run(str(sfind_bin), str(mount_dir), "-name", "file_a", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "file_a", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 2: Filter by Size (100c) log.info("Testing -size filter...") # Should match file_a and file_c (both 100 bytes) - ret = client.run(str(sfind_bin), str(mount_dir), "-size", "100c", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "100c", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 2) # Scenario 3: Filter by Size (200c) log.info("Testing -size filter (200c)...") # Should match file_b - ret = client.run(str(sfind_bin), str(mount_dir), "-size", "200c", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "200c", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 4: Filter by Newer (ctime) log.info("Testing -newer filter...") # Should match file_c (created after timestamp_file) - ret = client.run(str(sfind_bin), str(mount_dir), "-newer", str(timestamp_file), *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-newer", + str(timestamp_file), *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 5: Combined Filter (Name *file* AND Size 100c AND Newer) log.info("Testing combined filter...") # Should match only file_c - ret = client.run(str(sfind_bin), str(mount_dir), "-name", "*file*", "-size", "100c", "-newer", str(timestamp_file), *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "*file*", + "-size", "100c", "-newer", str(timestamp_file), + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) log.info("Packing order verification successful!") finally: - # Dump daemon log to inspect scanned keys and ctime - # daemon_log = test_workspace.logdir / "daemon.log" - # if daemon_log.exists(): - ## log.info("DUMPING DAEMON LOG DEBUG ENTRIES:") - # with open(daemon_log, 'r') as f: - # for line in f: - # if "DEBUG" in line: - # print(line.strip()) - #else: - # log.warning(f"Daemon log not found at {daemon_log}") daemon.shutdown() diff --git a/tests/integration/directories/test_sfind.py b/tests/integration/directories/test_sfind.py index a770f0e77..d20b237ef 100644 --- a/tests/integration/directories/test_sfind.py +++ b/tests/integration/directories/test_sfind.py @@ -1,10 +1,10 @@ - -import pytest import logging -from harness.gkfs import Daemon, ShellClient, Client, find_command -import os import time +import pytest + +from harness.gkfs import Client, Daemon, ShellClient, find_command + log = logging.getLogger(__name__) SHELL_CHECK_TIMEOUT = 180 @@ -12,9 +12,9 @@ SHELL_CHECK_TIMEOUT = 180 @pytest.mark.parametrize("buff_size", ["4096", "5242880"]) @pytest.mark.parametrize("conf", [ {"compress": "OFF", "cache": "OFF"}, - {"compress": "ON", "cache": "OFF"}, + {"compress": "ON", "cache": "OFF"}, {"compress": "OFF", "cache": "ON"}, - {"compress": "ON", "cache": "ON"}, + {"compress": "ON", "cache": "ON"}, ]) def test_sfind_permutations(test_workspace, request, conf, buff_size): """ @@ -23,16 +23,17 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 1. Populate with Safe Mode (Comp=OFF) 2. Restart Daemon with Target Mode 3. Run sfind with Target Client Mode - 4. Run ls with Target Client Mode + 4. Run readdir with Target Client Mode """ - + # 1. Safe Population (Daemon Comp=OFF) log.info("--- Phase 1: Population (Safe Mode) ---") pop_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", "GKFS_USE_DIRENTS_COMPRESSION": "OFF" } - daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=pop_daemon_env) + daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=pop_daemon_env) daemon_pop.run() try: @@ -42,19 +43,16 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): "LIBGKFS_DENTRY_CACHE": "OFF", "LIBGKFS_LOG": "info" } - - client = ShellClient(test_workspace) + + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir test_dir = mount_dir / "testdir" - - # Ensure directory exists - client.run("mkdir", "-p", str(test_dir)) - # Populate using create_n_files (faster) - io_client = Client(test_workspace) - # Create 2000 files + ret = io_client.mkdir(str(test_dir), 0o755, env=pop_client_env) + assert ret.retval == 0, f"mkdir failed: {ret.errno}" + ret = io_client.create_n_files(str(test_dir), 2000, env=pop_client_env) - assert ret.retval == 0, f"Population failed: {ret.errno}" assert ret.files_created == 2000, f"Expected 2000 files, created {ret.files_created}" log.info("Population complete.") @@ -65,14 +63,15 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): # 2. Test Execution (Target Mode) log.info(f"--- Phase 2: Testing (Conf: {conf}) ---") - + test_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"] } - daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=test_daemon_env) + daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=test_daemon_env) daemon_test.run() - + try: # Client Env for Test test_client_env = { @@ -81,45 +80,36 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): "LIBGKFS_DIRENTS_BUFF_SIZE": buff_size, "LIBGKFS_LOG": "info" } - + sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - + # --- sfind Check --- - log.info(f"Running sfind...") + log.info("Running sfind...") # sfind -S 1 -M # Run sfind directly instead of wrapping it in `bash -c`: ShellClient # preloads the executed process, and preloading both bash and sfind can # make the shell crash during teardown after sfind already printed a # successful MATCHED line. - ret = client.run( + ret = shell.run( str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir), timeout=SHELL_CHECK_TIMEOUT, env=test_client_env) sfind_stderr = ret.stderr.decode() if ret.stderr else "" sfind_stdout = ret.stdout.decode() if ret.stdout else "" - assert ret.exit_code == 0, f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}" - assert "MATCHED 2000/2000" in sfind_stdout, f"sfind did not match 2000/2000. Output:\n{sfind_stdout}" + assert ret.exit_code == 0, \ + f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}" + assert "MATCHED 2000/2000" in sfind_stdout, \ + f"sfind did not match 2000/2000. Output:\n{sfind_stdout}" log.info("sfind verification successful.") - # --- ls Check --- - log.info(f"Running ls check...") - # Avoid `ls -l` here: long format stats every entry and is the first - # thing to time out on overloaded CI runners. - ret_ls = client.run( - "ls", "-1", str(test_dir), timeout=SHELL_CHECK_TIMEOUT, - env=test_client_env) - - ls_stderr = ret_ls.stderr.decode() if ret_ls.stderr else "" - ls_stdout = ret_ls.stdout.decode() if ret_ls.stdout else "" - - assert ret_ls.exit_code == 0, f"ls check failed with {ret_ls.exit_code}\nStderr: {ls_stderr}" - - count = sum(1 for line in ls_stdout.splitlines() - if line.startswith("file_")) - assert count == 2000, f"ls count expected 2000, got '{count}'" - log.info("ls verification successful.") + log.info("Running readdir check...") + ret_ls = io_client.readdir(str(test_dir), env=test_client_env) + assert ret_ls.errno == 0, f"readdir failed: {ret_ls.errno}" + count = sum(1 for entry in ret_ls.dirents if entry.d_name.startswith("file_")) + assert count == 2000, f"readdir count expected 2000, got '{count}'" + log.info("readdir verification successful.") finally: daemon_test.shutdown() diff --git a/tests/integration/directories/test_sfind_diagnostic.py b/tests/integration/directories/test_sfind_diagnostic.py new file mode 100644 index 000000000..82b9f5042 --- /dev/null +++ b/tests/integration/directories/test_sfind_diagnostic.py @@ -0,0 +1,147 @@ +import logging +import os +import time +from pathlib import Path + +import pytest + +from harness.gkfs import Client, Daemon, ShellClient, find_command + + +log = logging.getLogger(__name__) + +SFIND_DIAG_TIMEOUT = 180 + + +def _read_text(path, limit=12000): + try: + data = Path(path).read_text(errors="replace") + except OSError as exc: + return f"" + if len(data) <= limit: + return data + return data[-limit:] + + +def _collect_workspace_logs(test_workspace): + chunks = [] + for base in (test_workspace.logdir, test_workspace.twd): + if not Path(base).exists(): + continue + for path in sorted(Path(base).glob("**/*")): + if path.is_file() and path.suffix in (".log", ".txt"): + chunks.append(f"\n--- {path} ---\n{_read_text(path)}") + return "".join(chunks) if chunks else "" + + +def _sfind_env(conf, buff_size, iteration): + return { + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], + "LIBGKFS_DENTRY_CACHE": conf["cache"], + "LIBGKFS_DIRENTS_BUFF_SIZE": str(buff_size), + "LIBGKFS_LOG": "debug", + "LIBGKFS_LOG_PER_PROCESS": "ON", + "MALLOC_CHECK_": "3", + "MALLOC_PERTURB_": str(1 + (iteration % 254)), + "SFIND_NUM_THREADS": str(conf["threads"]), + } + + +@pytest.mark.parametrize("buff_size", ["4096", "5242880"]) +@pytest.mark.parametrize("conf", [ + {"compress": "OFF", "cache": "OFF", "threads": 1}, + {"compress": "ON", "cache": "OFF", "threads": 1}, + {"compress": "OFF", "cache": "ON", "threads": 1}, + {"compress": "ON", "cache": "ON", "threads": 1}, + {"compress": "OFF", "cache": "OFF", "threads": 20}, + {"compress": "ON", "cache": "ON", "threads": 20}, +]) +def test_sfind_repeated_teardown_diagnostic(test_workspace, request, conf, buff_size): + """Stress sfind until the random SIGSEGV is reproducible with logs. + + The observed failure exits with -11 after printing MATCHED 2000/2000, so + this diagnostic repeats only the sfind phase and records whether the crash + depends on compression, dentry cache, buffer size, or sfind thread count. + """ + + repetitions = int(os.environ.get("GKFS_SFIND_DIAG_REPETITIONS", "25")) + file_count = int(os.environ.get("GKFS_SFIND_DIAG_FILES", "2000")) + + populate_daemon = Daemon( + request.config.getoption("--interface"), + "rocksdb", + test_workspace, + env={ + "GKFS_DAEMON_LOG_LEVEL": "debug", + "GKFS_USE_DIRENTS_COMPRESSION": "OFF", + }) + populate_daemon.run() + + mount_dir = test_workspace.mountdir + test_dir = mount_dir / "sfind_diagnostic_dir" + io_client = Client(test_workspace) + try: + ret = io_client.mkdir(str(test_dir), 0o755) + assert ret.retval == 0, f"mkdir failed: errno={ret.errno}" + + ret = io_client.create_n_files( + str(test_dir), + file_count, + env={ + "GKFS_USE_DIRENTS_COMPRESSION": "OFF", + "LIBGKFS_DENTRY_CACHE": "OFF", + "LIBGKFS_LOG": "debug", + "LIBGKFS_LOG_PER_PROCESS": "ON", + }) + assert ret.retval == 0, f"population failed: errno={ret.errno}" + assert ret.files_created == file_count + finally: + populate_daemon.shutdown() + time.sleep(1) + + test_daemon = Daemon( + request.config.getoption("--interface"), + "rocksdb", + test_workspace, + env={ + "GKFS_DAEMON_LOG_LEVEL": "debug", + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], + }) + test_daemon.run() + + sfind_bin = find_command("sfind", test_workspace.bindirs) + assert sfind_bin, "sfind binary not found" + + try: + shell = ShellClient(test_workspace) + failures = [] + for iteration in range(repetitions): + env = _sfind_env(conf, buff_size, iteration) + ret = shell.run( + str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir), + timeout=SFIND_DIAG_TIMEOUT, env=env) + stdout = ret.stdout.decode(errors="replace") if ret.stdout else "" + stderr = ret.stderr.decode(errors="replace") if ret.stderr else "" + log.info( + "sfind diagnostic iteration=%s conf=%s buff_size=%s exit=%s\n" + "stdout:\n%s\nstderr:\n%s", + iteration, conf, buff_size, ret.exit_code, stdout, stderr) + + if ret.exit_code != 0 or f"MATCHED {file_count}/{file_count}" not in stdout: + failures.append((iteration, ret.exit_code, stdout, stderr)) + if ret.exit_code == -11: + break + + if failures: + iteration, exit_code, stdout, stderr = failures[0] + pytest.fail( + "sfind diagnostic reproduced failure\n" + f"iteration: {iteration}\n" + f"conf: {conf}\n" + f"buff_size: {buff_size}\n" + f"exit_code: {exit_code}\n" + f"stdout:\n{stdout}\n" + f"stderr:\n{stderr}\n" + f"workspace logs:\n{_collect_workspace_logs(test_workspace)}") + finally: + test_daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/directories/test_sfind_filtered.py b/tests/integration/directories/test_sfind_filtered.py index 23068bceb..6a7cf87a0 100644 --- a/tests/integration/directories/test_sfind_filtered.py +++ b/tests/integration/directories/test_sfind_filtered.py @@ -1,54 +1,48 @@ - -import pytest import logging -from harness.gkfs import Daemon, ShellClient, Client, find_command -import os -import time + +from harness.gkfs import Client, Daemon, ShellClient, find_command log = logging.getLogger(__name__) + def test_sfind_filtered(test_workspace, request): """ Test sfind with server-side filtering (-name). """ - + # 1. Start Daemon log.info("--- Starting Daemon ---") daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace) daemon.run() try: - client = ShellClient(test_workspace) + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir test_dir = mount_dir / "testdir_filtered" - - # Ensure directory exists - client.run("mkdir", "-p", str(test_dir)) - # 2. Populate Files - io_client = Client(test_workspace) - # Create 100 files, named file_0 to file_99 + ret = io_client.mkdir(str(test_dir), 0o755) + assert ret.retval == 0 + ret = io_client.create_n_files(str(test_dir), 100) assert ret.retval == 0 log.info("Population complete.") - # 3.Run sfind with -name filter + # 3. Run sfind with -name filter sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - - # Filter for "file_50" - # Usage: sfind [-name ] -M -S --server-side - # sfind_cmd = f"{sfind_bin} {test_dir} -name \"file_50\" -M {mount_dir} -S 1 --server-side" - + + # Usage: sfind [-name ] -M + # -S --server-side log.info(f"Running sfind command: {sfind_bin} {test_dir} ...") - print(f"DEBUG: Workspace: {test_workspace.twd}") - - ret = client.run(str(sfind_bin), str(test_dir), "-name", "file_50", "-M", str(mount_dir), "-S", "1", "--server-side") + + ret = shell.run(str(sfind_bin), str(test_dir), "-name", "file_50", + "-M", str(mount_dir), "-S", "1", "--server-side") sfind_stderr = ret.stderr.decode(errors='replace') if ret.stderr else "" sfind_stdout = ret.stdout.decode(errors='replace') if ret.stdout else "" - + print(f"STDOUT: {sfind_stdout}") print(f"STDERR: {sfind_stderr}") @@ -57,57 +51,54 @@ def test_sfind_filtered(test_workspace, request): assert "MATCHED" in sfind_stdout # Expect 1 match out of 100 checked assert "MATCHED 1/100" in sfind_stdout - + # 4. Verify Recursive Filtering (Deep Search) # Create a nested directory and file nested_dir = test_dir / "subdir" - client.run("mkdir", "-p", str(nested_dir)) - nested_file = nested_dir / "deep_file.01" - # We need to manually create this file as create_n_files is batch - # Using touch or simple write via ShellClient if io_client is limited - # Actually io_client.create_n_files can point to the subdir - ret = io_client.create_n_files(str(nested_dir), 10) # 10 files in subdir + ret = io_client.mkdir(str(nested_dir), 0o755) + assert ret.retval == 0 + + ret = io_client.create_n_files(str(nested_dir), 10) assert ret.retval == 0 - + # Search for "file_5" in the scoped subdir (should be file_5) - # create_n_files uses formatted names "file_"? - # Let's check how create_n_files names them. Usually "file_"? - # Assuming defaults. - - log.info(f"Running recursive sfind command: {sfind_bin} {test_dir} -name \"*file_5\" ...") + log.info( + f"Running recursive sfind command: {sfind_bin} {test_dir} " + "-name \"*file_5\" ...") # Should find file_5 in top dir AND file_5 in subdir - ret = client.run(str(sfind_bin), str(test_dir), "-name", "*file_5", "-M", str(mount_dir), "-S", "1", "--server-side") - + ret = shell.run(str(sfind_bin), str(test_dir), "-name", "*file_5", + "-M", str(mount_dir), "-S", "1", "--server-side") + sfind_stdout = ret.stdout.decode(errors='replace') if ret.stdout else "" print(f"RECURSIVE STDOUT: {sfind_stdout}") - + assert ret.exit_code == 0 assert "MATCHED" in sfind_stdout # We expect file_5 in top (0-99) and file_5 in subdir (0-9) # Total files: 100 + 10 = 110. # Matches: 2 assert "MATCHED 2/111" in sfind_stdout - + except Exception: # Dump logs client_log = test_workspace.logdir / 'gkfs_client.log' if client_log.exists(): print("\n=== CLIENT LOG ===") print(client_log.read_text()) - + daemon_log = test_workspace.logdir / 'gkfs_daemon.log' if daemon_log.exists(): print("\n=== DAEMON LOG ===") print(daemon_log.read_text()) - + print(f"DEBUG: Workspace: {test_workspace.twd}") - + # Check for sfind result file results_file = test_workspace.twd / 'gfind_results.rank-0.txt' if results_file.exists(): - print(f"\n=== SFIND RESULT FILE ===\n{results_file.read_text()}") + print(f"\n=== SFIND RESULT FILE ===\n{results_file.read_text()}") else: - print(f"\n=== SFIND RESULT FILE NOT FOUND ===") + print("\n=== SFIND RESULT FILE NOT FOUND ===") raise diff --git a/tests/integration/directories/test_symlinks.py b/tests/integration/directories/test_symlinks.py index 78263b360..f6e481472 100644 --- a/tests/integration/directories/test_symlinks.py +++ b/tests/integration/directories/test_symlinks.py @@ -1,12 +1,8 @@ -import harness -from pathlib import Path -import errno import stat import os -import ctypes -import sys + import pytest -from harness.logger import logger + @pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) def test_symlink_type(client_fixture, request, gkfs_daemon): @@ -25,9 +21,13 @@ def test_symlink_type(client_fixture, request, gkfs_daemon): ret = gkfs_client.symlink(str(target_name), str(link_name)) assert ret.retval == 0 + ret = gkfs_client.readlink(str(link_name)) + assert ret.retval == len(str(target_name)) + assert ret.target == str(target_name) + # Readdir and check type ret = gkfs_client.readdir(mountdir) - + # We expect ., .., target, mylink found_link = False found_target = False diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index d9843d85d..0aa4696c1 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -62,6 +62,7 @@ add_executable(gkfs.io gkfs.io/chdir.cpp gkfs.io/getcwd_validate.cpp gkfs.io/symlink.cpp + gkfs.io/readlink.cpp gkfs.io/directory_validate.cpp gkfs.io/unlink.cpp gkfs.io/access.cpp diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 4eaf56348..02aac04ed 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -130,6 +130,9 @@ getcwd_validate_init(CLI::App& app); void symlink_init(CLI::App& app); +void +readlink_init(CLI::App& app); + void unlink_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 552c24b3f..8fc028774 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -77,6 +77,7 @@ init_commands(CLI::App& app) { chdir_init(app); getcwd_validate_init(app); symlink_init(app); + readlink_init(app); unlink_init(app); dup_validate_init(app); syscall_coverage_init(app); diff --git a/tests/integration/harness/gkfs.io/readlink.cpp b/tests/integration/harness/gkfs.io/readlink.cpp new file mode 100644 index 000000000..f774fee0c --- /dev/null +++ b/tests/integration/harness/gkfs.io/readlink.cpp @@ -0,0 +1,120 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This software was partially supported by the + the European Union’s Horizon 2020 JTI-EuroHPC research and + innovation programme, by the project ADMIRE (Project ID: 956748, + admire-eurohpc.eu) + + This project was partially promoted by the Ministry for Digital Transformation + and the Civil Service, within the framework of the Recovery, + Transformation and Resilience Plan - Funded by the European Union + -NextGenerationEU. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include + +using json = nlohmann::json; + +struct readlink_options { + bool verbose{}; + std::string pathname; + ::size_t size = 4096; + + REFL_DECL_STRUCT(readlink_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, size)); +}; + +struct readlink_output { + std::string target; + long retval; + int errnum; + + REFL_DECL_STRUCT(readlink_output, REFL_DECL_MEMBER(std::string, target), + REFL_DECL_MEMBER(long, retval), + REFL_DECL_MEMBER(int, errnum)); +}; + +void +to_json(json& record, const readlink_output& out) { + record = serialize(out); +} + +void +readlink_exec(const readlink_options& opts) { + + std::string target(opts.size, '\0'); + errno = 0; + auto rv = ::readlink(opts.pathname.c_str(), target.data(), target.size()); + auto errnum = errno; + + if(rv >= 0) { + target.resize(static_cast(rv)); + } else { + target.clear(); + } + + if(opts.verbose) { + fmt::print( + "readlink(pathname=\"{}\", bufsize={}) = {}, target=\"{}\", errno: {} [{}]\n", + opts.pathname, opts.size, rv, target, errnum, ::strerror(errnum)); + return; + } + + json out = readlink_output{target, static_cast(rv), errnum}; + fmt::print("{}\n", out.dump(2)); +} + +void +readlink_init(CLI::App& app) { + + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("readlink", "Execute the readlink() system call"); + + cmd->add_flag("-v,--verbose", opts->verbose, + "Produce human readable output"); + + cmd->add_option("pathname", opts->pathname, "Symbolic link path") + ->required() + ->type_name(""); + + cmd->add_option("--size", opts->size, "Read buffer size") + ->default_val(opts->size); + + cmd->callback([opts]() { readlink_exec(*opts); }); +} \ No newline at end of file diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 5d677e347..33190f2d4 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -468,6 +468,18 @@ class SymlinkOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('SymlinkReturn', ['retval', 'errno'])(**data) +class ReadlinkOutputSchema(Schema): + """Schema to deserialize the results of a readlink() execution""" + + target = fields.String(required=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('ReadlinkReturn', + ['target', 'retval', 'errno'])(**data) + class UnlinkOutputSchema(Schema): """Schema to deserialize the results of an unlink() execution""" retval = fields.Integer(required=True) @@ -545,6 +557,7 @@ class IOParser: 'chdir' : ChdirOutputSchema(), 'getcwd_validate' : GetcwdvalidateOutputSchema(), 'symlink' : SymlinkOutputSchema(), + 'readlink': ReadlinkOutputSchema(), 'dup_validate' : DupValidateOutputSchema(), 'syscall_coverage' : SyscallCoverageOutputSchema(), 'create_n_files' : CreateNFilesOutputSchema(), diff --git a/tests/integration/operations/test_client_cache.py b/tests/integration/operations/test_client_cache.py index f7a5fd11b..4b15b45cb 100644 --- a/tests/integration/operations/test_client_cache.py +++ b/tests/integration/operations/test_client_cache.py @@ -1,6 +1,6 @@ import pytest import os -import stat + from harness.gkfs import Client as GKFSClient @@ -10,11 +10,10 @@ def gkfs_client_cache(test_workspace, gkfs_daemon, monkeypatch): Sets up a GKFSClient with caching enabled via environment variables. """ monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE", "ON") - # Threshold 10 means flush every 10 writes. - monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE_THRESHOLD", "10") + # Threshold 10 means flush every 10 writes. + monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE_THRESHOLD", "10") monkeypatch.setenv("LIBGKFS_DENTRY_CACHE", "ON") - - + client = GKFSClient(test_workspace) return client @@ -24,14 +23,16 @@ def test_write_size_cache(gkfs_daemon, gkfs_client_cache): Test write size cache by running a C++ helper that performs multiple writes. """ file = gkfs_daemon.mountdir / "cache_file" - + # Run the helper: writes 20 chunks of 100 bytes iterations = 20 chunk_size = 100 - + # Use gkfs.io write_sequential command - cmd = gkfs_client_cache.run("write_sequential", "--pathname", str(file), "--count", str(iterations), "--size", str(chunk_size)) - + cmd = gkfs_client_cache.run("write_sequential", "--pathname", str(file), + "--count", str(iterations), "--size", + str(chunk_size)) + assert cmd.retval == 0 # Verify file size using GKFSClient (which handles stat command parsing) @@ -44,29 +45,18 @@ def test_dentry_cache(gkfs_daemon, gkfs_client_cache): Test dentry cache by creating a directory structure and listing it repeatedly. """ subdir = gkfs_daemon.mountdir / "subdir" - + gkfs_client_cache.mkdir(subdir, 0o755) - + # Create files for i in range(10): f = subdir / f"file{i}" gkfs_client_cache.open(f, os.O_CREAT | os.O_WRONLY, 0o644) - - # Use ls -lR to trigger readdir and accessing attributes - # We use subprocess with client env to ensure caching is enabled - import subprocess - ls_cmd = ["ls", "-lR", str(gkfs_daemon.mountdir)] - - # First run: entries should be cached - proc = subprocess.Popen(ls_cmd, env=gkfs_client_cache._env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - assert proc.returncode == 0 - assert b"subdir" in stdout - assert b"file0" in stdout - - # Second run: entries should be served from cache - proc = subprocess.Popen(ls_cmd, env=gkfs_client_cache._env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - assert proc.returncode == 0 + first = gkfs_client_cache.readdir(gkfs_daemon.mountdir) + assert first.errno == 0 + assert any(entry.d_name == "subdir" for entry in first.dirents) + second = gkfs_client_cache.readdir(subdir) + assert second.errno == 0 + assert any(entry.d_name == "file0" for entry in second.dirents) diff --git a/tests/integration/syscalls/test_config_env.py b/tests/integration/syscalls/test_config_env.py index 6e670bd1d..2deb68542 100644 --- a/tests/integration/syscalls/test_config_env.py +++ b/tests/integration/syscalls/test_config_env.py @@ -1,135 +1,88 @@ - import pytest -import logging -import os -import time -from pathlib import Path -from harness.logger import logger -from harness.gkfs import Daemon, Client, ShellClient + +from harness.gkfs import Client, Daemon + @pytest.mark.parametrize("use_inline", ["ON", "OFF"]) def test_inline_data(test_workspace, request, use_inline): """ Verify inline data configuration via environment variables. - Both Client and Daemon need to agree (or at least Client needs to send it). - + If OFF: - Client writes small data. - - Should land in Chunk Storage (file in workspace/chunks). + - Should land in chunk storage. If ON: - - Client writes small data. - - Should landad in Metadata (no file in workspace/chunks for this file). + - Should land in metadata. + - Should not create chunk files for this small write. """ - - # 1. Start Daemon with env var - # We use same value for Daemon to be safe (though strictly Client decides to send inline) - # 1. Start Daemon with env var - # We use same value for Daemon to be safe (though strictly Client decides to send inline) daemon_env = {"GKFS_USE_INLINE_DATA": use_inline} - - interface = request.config.getoption('--interface') - backend = "rocksdb" - - daemon = Daemon(interface, backend, test_workspace, env=daemon_env) + + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() - + try: - # 2. Start Client with env var - # 2. Start Client with env var client_env = {"GKFS_USE_INLINE_DATA": use_inline} - - # We need a shell client or similar to execute commands - # We can use gkfs.io via Client class which wraps it, or just use shell client = Client(test_workspace) - - # 3. Write small file (100 bytes) - # using 'write_sequential' from previous task! + test_file = test_workspace.mountdir / "file_inline" - - # Write 100 bytes - cmd = client.run("write_sequential", "--pathname", str(test_file), "--count", "1", "--size", "100", env=client_env) + cmd = client.run("write_sequential", "--pathname", str(test_file), + "--count", "1", "--size", "100", env=client_env) assert cmd.retval == 0 - - # 4. Verify storage location - # Check chunks directory - # The chunks directory is in test_workspace.rootdir / "chunks" + chunk_dir = test_workspace.rootdir / "chunks" - - # We expect files in chunk_dir ONLY if use_inline == "OFF" - # Since we just started fresh, chunk_dir might be empty or contain structure. - # We look for any file recursively in chunk_dir - - found_chunks = list(chunk_dir.rglob("*")) - # Filter out directories - found_chunks = [f for f in found_chunks if f.is_file()] - + found_chunks = [f for f in chunk_dir.rglob("*") if f.is_file()] + if use_inline == "OFF": - # Should have chunks - assert len(found_chunks) > 0, "Expected chunks to be created when INLINE_DATA=OFF" + assert len(found_chunks) > 0, \ + "Expected chunks to be created when INLINE_DATA=OFF" else: - # Should NOT have chunks (for this file). - # Note: create_directories might have created empty subdirs, but we filtered for files. - assert len(found_chunks) == 0, f"Expected NO chunks when INLINE_DATA=ON (data should be inline). Found: {found_chunks}" - + assert len(found_chunks) == 0, \ + f"Expected NO chunks when INLINE_DATA=ON. Found: {found_chunks}" finally: daemon.shutdown() + @pytest.mark.parametrize("use_compression", ["ON", "OFF"]) def test_dirents_compression(test_workspace, request, use_compression): """ Verify dirents compression configuration. - Hard to verify compression effect directly without packet capture, - but we can verify that the system runs and respects the flag in logs. + + Hard to verify compression effect directly without packet capture, but we + can verify that the daemon logs the flag and that directory listing works. """ - - daemon_env = {"GKFS_USE_DIRENTS_COMPRESSION": use_compression} - # specific log level to see configuration output - daemon_env["GKFS_DAEMON_LOG_LEVEL"] = "info" - - interface = request.config.getoption('--interface') - backend = "rocksdb" - - daemon = Daemon(interface, backend, test_workspace, env=daemon_env) + daemon_env = { + "GKFS_USE_DIRENTS_COMPRESSION": use_compression, + "GKFS_DAEMON_LOG_LEVEL": "info", + } + + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() - + try: - # Check daemon logs for the configuration message - # We added: GKFS_DATA->spdlogger()->info("{}() Inline data: {} / Dirents compression: {}", ... - - # grep log file log_file = daemon.logdir / "gkfs_daemon.log" - passed = False expected_val = "true" if use_compression == "ON" else "false" with open(log_file, 'r') as f: log_content = f.read() - if f"Dirents compression: {expected_val}" in log_content: - passed = True - - if not passed: - print(f"DEBUG: Log content:\n{log_content}") - - assert passed, f"Daemon log did not confirm Dirents compression={use_compression} (expected {expected_val})" - - # Client side verification - # Client side verification + + assert f"Dirents compression: {expected_val}" in log_content, \ + f"Daemon log did not confirm Dirents compression={use_compression} " \ + f"(expected {expected_val})\n{log_content}" + client_env = {"GKFS_USE_DIRENTS_COMPRESSION": use_compression} - - # Just run a simple ls to trigger dirents client = Client(test_workspace) - shell = ShellClient(test_workspace) - - # Create dir using shell mkidr - behavior should be same regarding dirents later - cmd = shell.run("mkdir", "-p", str(test_workspace.mountdir / "dir"), env=client_env) - assert cmd.exit_code == 0 - - # Populate dir using Client write - cmd = client.run("write_sequential", "--pathname", str(test_workspace.mountdir / "dir" / "file"), "--count", "1", "--size", "100", env=client_env) + + test_dir = test_workspace.mountdir / "dir" + cmd = client.mkdir(str(test_dir), 0o755, env=client_env) assert cmd.retval == 0 - # List dir using shell ls - cmd = shell.run("ls", str(test_workspace.mountdir / "dir"), env=client_env) - assert cmd.exit_code == 0 - + cmd = client.run("write_sequential", "--pathname", str(test_dir / "file"), + "--count", "1", "--size", "100", env=client_env) + assert cmd.retval == 0 + + cmd = client.readdir(str(test_dir), env=client_env) + assert cmd.errno == 0 + assert any(entry.d_name == "file" for entry in cmd.dirents) finally: daemon.shutdown() - -- GitLab From 5b92785a69904be8f71179192566805e7126884c Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 14:02:18 +0200 Subject: [PATCH 14/21] fix(daemon): track redistribution state atomically Use atomic load/store accessors for the redistribution running flag and set it before starting malleability redistribution threads. Reset the flag on ABT thread creation failure so daemon state remains consistent. --- src/daemon/classes/fs_data.cpp | 4 ++-- src/daemon/malleability/malleable_manager.cpp | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index a107c3f44..768ed3b80 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -384,12 +384,12 @@ FsData::maintenance_mode(bool maintenance_mode) { bool FsData::redist_running() const { - return redist_running_; + return redist_running_.load(); } void FsData::redist_running(bool redist_running) { - redist_running_ = redist_running; + redist_running_.store(redist_running); } const std::shared_ptr& diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index e7a32df41..fbe6f7563 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -909,10 +909,12 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, "{}() {} active: old_hosts={}, new_hosts={}. Skipping eager data migration.", __func__, gkfs::env::EXPAND_ON_DEMAND, old_hosts_size_, hosts.size()); + GKFS_DATA->redist_running(true); auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_on_demand_abt, this, ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { + GKFS_DATA->redist_running(false); throw runtime_error(fmt::format( "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", __func__, abt_err)); @@ -921,9 +923,11 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, } // Use v2 pipeline for mutate + GKFS_DATA->redist_running(true); auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { + GKFS_DATA->redist_running(false); auto err_str = fmt::format( "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", __func__, abt_err); -- GitLab From a4004a61007bfd64ce0b86c3689a0b34afde359b Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 14:02:42 +0200 Subject: [PATCH 15/21] fix(fs): make redistribution state flag atomic Use `std::atomic` for `redist_running_` to ensure safe concurrent access when clients check whether redistribution is running. --- include/daemon/classes/fs_data.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 984a13a95..5ee983631 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -41,6 +41,7 @@ #include +#include #include #include #include //std::hash @@ -138,7 +139,7 @@ private: bool maintenance_mode_ = false; ABT_mutex maintenance_mode_mutex_; // redist_running_ indicates to client that redistribution is running - bool redist_running_ = false; + std::atomic redist_running_{false}; bool expand_on_demand_active_ = false; unsigned int expand_on_demand_old_hosts_size_ = 0; -- GitLab From fdc1e9dff5575df99e4e3cc1872c0d89332109bb Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 14:33:27 +0200 Subject: [PATCH 16/21] diags out --- .../directories/test_sfind_diagnostic.py | 147 ------------------ 1 file changed, 147 deletions(-) delete mode 100644 tests/integration/directories/test_sfind_diagnostic.py diff --git a/tests/integration/directories/test_sfind_diagnostic.py b/tests/integration/directories/test_sfind_diagnostic.py deleted file mode 100644 index 82b9f5042..000000000 --- a/tests/integration/directories/test_sfind_diagnostic.py +++ /dev/null @@ -1,147 +0,0 @@ -import logging -import os -import time -from pathlib import Path - -import pytest - -from harness.gkfs import Client, Daemon, ShellClient, find_command - - -log = logging.getLogger(__name__) - -SFIND_DIAG_TIMEOUT = 180 - - -def _read_text(path, limit=12000): - try: - data = Path(path).read_text(errors="replace") - except OSError as exc: - return f"" - if len(data) <= limit: - return data - return data[-limit:] - - -def _collect_workspace_logs(test_workspace): - chunks = [] - for base in (test_workspace.logdir, test_workspace.twd): - if not Path(base).exists(): - continue - for path in sorted(Path(base).glob("**/*")): - if path.is_file() and path.suffix in (".log", ".txt"): - chunks.append(f"\n--- {path} ---\n{_read_text(path)}") - return "".join(chunks) if chunks else "" - - -def _sfind_env(conf, buff_size, iteration): - return { - "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], - "LIBGKFS_DENTRY_CACHE": conf["cache"], - "LIBGKFS_DIRENTS_BUFF_SIZE": str(buff_size), - "LIBGKFS_LOG": "debug", - "LIBGKFS_LOG_PER_PROCESS": "ON", - "MALLOC_CHECK_": "3", - "MALLOC_PERTURB_": str(1 + (iteration % 254)), - "SFIND_NUM_THREADS": str(conf["threads"]), - } - - -@pytest.mark.parametrize("buff_size", ["4096", "5242880"]) -@pytest.mark.parametrize("conf", [ - {"compress": "OFF", "cache": "OFF", "threads": 1}, - {"compress": "ON", "cache": "OFF", "threads": 1}, - {"compress": "OFF", "cache": "ON", "threads": 1}, - {"compress": "ON", "cache": "ON", "threads": 1}, - {"compress": "OFF", "cache": "OFF", "threads": 20}, - {"compress": "ON", "cache": "ON", "threads": 20}, -]) -def test_sfind_repeated_teardown_diagnostic(test_workspace, request, conf, buff_size): - """Stress sfind until the random SIGSEGV is reproducible with logs. - - The observed failure exits with -11 after printing MATCHED 2000/2000, so - this diagnostic repeats only the sfind phase and records whether the crash - depends on compression, dentry cache, buffer size, or sfind thread count. - """ - - repetitions = int(os.environ.get("GKFS_SFIND_DIAG_REPETITIONS", "25")) - file_count = int(os.environ.get("GKFS_SFIND_DIAG_FILES", "2000")) - - populate_daemon = Daemon( - request.config.getoption("--interface"), - "rocksdb", - test_workspace, - env={ - "GKFS_DAEMON_LOG_LEVEL": "debug", - "GKFS_USE_DIRENTS_COMPRESSION": "OFF", - }) - populate_daemon.run() - - mount_dir = test_workspace.mountdir - test_dir = mount_dir / "sfind_diagnostic_dir" - io_client = Client(test_workspace) - try: - ret = io_client.mkdir(str(test_dir), 0o755) - assert ret.retval == 0, f"mkdir failed: errno={ret.errno}" - - ret = io_client.create_n_files( - str(test_dir), - file_count, - env={ - "GKFS_USE_DIRENTS_COMPRESSION": "OFF", - "LIBGKFS_DENTRY_CACHE": "OFF", - "LIBGKFS_LOG": "debug", - "LIBGKFS_LOG_PER_PROCESS": "ON", - }) - assert ret.retval == 0, f"population failed: errno={ret.errno}" - assert ret.files_created == file_count - finally: - populate_daemon.shutdown() - time.sleep(1) - - test_daemon = Daemon( - request.config.getoption("--interface"), - "rocksdb", - test_workspace, - env={ - "GKFS_DAEMON_LOG_LEVEL": "debug", - "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], - }) - test_daemon.run() - - sfind_bin = find_command("sfind", test_workspace.bindirs) - assert sfind_bin, "sfind binary not found" - - try: - shell = ShellClient(test_workspace) - failures = [] - for iteration in range(repetitions): - env = _sfind_env(conf, buff_size, iteration) - ret = shell.run( - str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir), - timeout=SFIND_DIAG_TIMEOUT, env=env) - stdout = ret.stdout.decode(errors="replace") if ret.stdout else "" - stderr = ret.stderr.decode(errors="replace") if ret.stderr else "" - log.info( - "sfind diagnostic iteration=%s conf=%s buff_size=%s exit=%s\n" - "stdout:\n%s\nstderr:\n%s", - iteration, conf, buff_size, ret.exit_code, stdout, stderr) - - if ret.exit_code != 0 or f"MATCHED {file_count}/{file_count}" not in stdout: - failures.append((iteration, ret.exit_code, stdout, stderr)) - if ret.exit_code == -11: - break - - if failures: - iteration, exit_code, stdout, stderr = failures[0] - pytest.fail( - "sfind diagnostic reproduced failure\n" - f"iteration: {iteration}\n" - f"conf: {conf}\n" - f"buff_size: {buff_size}\n" - f"exit_code: {exit_code}\n" - f"stdout:\n{stdout}\n" - f"stderr:\n{stderr}\n" - f"workspace logs:\n{_collect_workspace_logs(test_workspace)}") - finally: - test_daemon.shutdown() \ No newline at end of file -- GitLab From 54dd5c7d7f836ff87f4f3de5dac34f282cbfcd37 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 14:49:15 +0200 Subject: [PATCH 17/21] Update README --- CHANGELOG.md | 10 ++- README.md | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 210 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a60aa05e..3f0b2b4cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,8 +47,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Client interface and hostfile removel ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313)) - Demon option to avoid removing hostfile on exit : ENV variable , GKFS_KEEP_HOSTS_FILE or option --keep-hosts - Client Interface selection avoiding loopback setup : LIBGKFS_OFI_INTERFACE=ib0 - - + - Random Slicing + cutshift ([!316](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/316)) + - Added marker-based expand, shrink, and mixed mutate workflow using one shared hostfile. + - Added Random Slicing placement with CutShift for expand-only resizes to reduce inter-node data movement. + - Added optional expand-on-demand data movement for pure expand operations. + - Added malleability integration coverage and user documentation. ### Changed @@ -60,6 +63,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - LIBGKFS/ GKFS _SYMLINK_SUPPORT, _RENAME_SUPPORT and _CREATE_CHECK_PARENTS. - Now all the performance options are in config.hpp and env variables. - Added DIRECT_IO on server side to increase performance ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312)) + - Shared metrics/message routing variables now use the common `GKFS_` prefix. + ### Fixed - SYS_lstat does not exists on some architectures, change to newfstatat ([!269](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/269)) @@ -71,6 +76,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Fix remove chunk bug ([!294](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/294)) - Fix decompress_and_parse_entries_standard() Unexpected end of buffer while parsing name bug ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312)) - Fix client dissapearing on malleability ends with an error. ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313)) + - Fixed buffer-size validation issues in filtered directory listing paths. ## [0.9.5] - 2025-08 diff --git a/README.md b/README.md index 16f36ae9e..83b7537db 100644 --- a/README.md +++ b/README.md @@ -551,6 +551,201 @@ scripts/run/gkfs -c gkfs.conf stop For shrink-only operations, no new daemons are needed; just mark leaving entries with `-`. For expand-only operations, no `-` entries are needed; start the added daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+` entries. +### User-level resize examples + +The examples below use plain GekkoFS commands: start daemons with `gkfs_daemon` and control topology changes with +`gkfs_malleability`. They assume GekkoFS was configured with tools enabled, for example `-DGKFS_BUILD_TOOLS=ON`, and that +`gkfs_daemon`, `gkfs_malleability`, and `libgkfs_intercept.so` come from the same installation. + +Common shell setup: + +```bash +# installation and workspace paths +export GKFS_INSTALL=/path/to/install +export GKFS_WORKDIR=/scratch/$USER/gkfs +export GKFS_HOSTFILE=$GKFS_WORKDIR/gkfs_hosts.txt +export GKFS_ROOTDIR=/local/ssd/$USER/gkfs-root +export GKFS_MOUNTDIR=$GKFS_WORKDIR/mount + +mkdir -p "$GKFS_WORKDIR" "$GKFS_MOUNTDIR" + +# client and tool input: one shared hostfile for the current GekkoFS instance +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTFILE + +# daemon-side hostfile path; the same path is also passed with -H below +export GKFS_HOSTS_FILE=$GKFS_HOSTFILE + +# keep the hosts file while daemons are stopped or resized; this is important for marker-based mutations +export GKFS_DAEMON_KEEP_HOSTS_FILE=ON + +# choose one placement strategy and export it everywhere: daemons, clients, proxies, and tools +export GKFS_DISTRIBUTION_STRATEGY=simple_hash +# CutShift substantially reduces inter-node data movement during expand-only resizes +# alternative: +# export GKFS_DISTRIBUTION_STRATEGY=random_slicing +# export GKFS_RANDOM_SLICING_CUTSHIFT=ON + +# recommended on multi-node systems to avoid clients choosing loopback +export LIBGKFS_OFI_INTERFACE=ib0 +``` + +Client environment for applications: + +```bash +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTFILE +export LD_PRELOAD=$GKFS_INSTALL/lib/libgkfs_intercept.so +export GKFS_DISTRIBUTION_STRATEGY=simple_hash # or random_slicing, matching daemon/tool env +export LIBGKFS_OFI_INTERFACE=ib0 +``` + +Before every resize operation, stop or pause applications so that no process accesses the mounted GekkoFS namespace while +redistribution is running. After `mutate finalize`, restart applications or force them to reload the updated hostfile. If +the GekkoFS proxy is used, restart proxies manually after the topology change. + +Use the following helper loop to wait for a topology change to finish when running `gkfs_malleability` manually: + +```bash +wait_for_mutate() { + while ! "$GKFS_INSTALL/bin/gkfs_malleability" mutate status 2>&1 | grep -q "No mutate"; do + sleep 2 + done +} +``` + +#### Expand: add daemon nodes + +Start the initial instance, for example on two nodes: + +```bash +rm -f "$GKFS_HOSTFILE" +printf "node01\nnode02\n" > "$GKFS_WORKDIR/nodes-current.txt" + +srun --nodelist="$GKFS_WORKDIR/nodes-current.txt" \ + --ntasks=2 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & +``` + +To expand from two to four daemons, start only the new daemons with `GKFS_DAEMON_EXPAND=ON`. The added daemons append +`+` entries to the existing hostfile; do not delete the existing hostfile. + +```bash +# applications must be stopped or paused here +printf "node03\nnode04\n" > "$GKFS_WORKDIR/nodes-add.txt" + +GKFS_DAEMON_EXPAND=ON \ +srun --nodelist="$GKFS_WORKDIR/nodes-add.txt" \ + --ntasks=2 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & + +# run redistribution, poll status, and finalize the marked hostfile +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize +``` + +After a successful finalize, `gkfs_hosts.txt` contains four unmarked active entries. Keep an up-to-date list of all active +daemon nodes for later administration: + +```bash +printf "node01\nnode02\nnode03\nnode04\n" > "$GKFS_WORKDIR/nodes-current.txt" +``` + +Optional lazy data movement for pure expand: + +```bash +export GKFS_EXPAND_ON_DEMAND=ON +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize +``` + +With `GKFS_EXPAND_ON_DEMAND=ON`, metadata is still redistributed during `mutate start`, but data chunks are fetched and +materialized lazily on first access. Use this only for pure expand operations; shrink and mixed mutate use eager data +migration. + +#### Shrink: remove daemon nodes + +For shrink-only operations, do not start new daemons. Mark each daemon that should leave by prefixing its hostfile line +with `-`, then run mutate. The leaving daemons must remain running and reachable until finalize completes; `mutate +finalize` stops the daemons marked with `-` automatically. + +Example shrink from four to two daemons: + +```bash +# applications must be stopped or paused here +cp "$GKFS_HOSTFILE" "$GKFS_HOSTFILE.bak" + +# mark node03 and node04 for removal. Match the real host names as they appear in the hostfile. +sed -i -E 's/^(node03[[:space:]])/-\1/' "$GKFS_HOSTFILE" +sed -i -E 's/^(node04[[:space:]])/-\1/' "$GKFS_HOSTFILE" + +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize +``` + +After finalize, the hostfile contains only the remaining active daemon entries and the daemons that left the topology have +been stopped automatically. Update your active-node list: + +```bash +printf "node01\nnode02\n" > "$GKFS_WORKDIR/nodes-current.txt" +``` + +#### Mutate: replace nodes in one operation + +A mixed mutate, or node swap, combines shrink and expand. Mark the leaving daemon lines with `-`, start replacement +daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+` entries, then run the same mutate workflow. + +Example replacement of `node02` by `node05` while keeping the daemon count constant: + +```bash +# applications must be stopped or paused here +cp "$GKFS_HOSTFILE" "$GKFS_HOSTFILE.bak" + +# mark the old daemon that should leave +sed -i -E 's/^(node02[[:space:]])/-\1/' "$GKFS_HOSTFILE" + +# start the replacement daemon; it appends a '+node05 ...' entry +printf "node05\n" > "$GKFS_WORKDIR/nodes-add.txt" +GKFS_DAEMON_EXPAND=ON \ +srun --nodelist="$GKFS_WORKDIR/nodes-add.txt" \ + --ntasks=1 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & + +# redistribute from the before topology to the after topology, stop '-' daemons, and clean markers +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize + +# update the active-node list for future administration +printf "node01\nnode03\nnode04\nnode05\n" > "$GKFS_WORKDIR/nodes-current.txt" +``` + +Useful daemon and tool options for these examples: + +| Option or command | Meaning | +|---|---| +| `gkfs_daemon -r, --rootdir ` | Local data directory for each daemon | +| `gkfs_daemon -m, --mountdir ` | GekkoFS mount directory visible to clients | +| `gkfs_daemon -H, --hosts-file ` | Shared hostfile written by daemons and read by clients/tools | +| `gkfs_daemon -l ` | Network interface or address to bind, for example `ib0` | +| `gkfs_daemon -P ` | RPC protocol, for example `ofi+sockets` or `ofi+verbs` | +| `gkfs_malleability mutate start` | Start redistribution from the marked hostfile topology | +| `gkfs_malleability mutate status` | Poll redistribution status | +| `gkfs_malleability mutate finalize` | Stop daemons marked with `-`, clean the hostfile markers, and complete the topology change | + ### Random Slicing and CutShift Set the same distribution variables for daemons, clients, proxies, and tools: @@ -566,6 +761,11 @@ For `random_slicing`, `mutate start` writes the chosen interval table into the w CutShift is applied for expand-only Random Slicing operations (`+` entries present, no `-` entries). Other Random Slicing mutate cases currently fall back to the final equal RS layout. +Using `GKFS_DISTRIBUTION_STRATEGY=random_slicing` together with `GKFS_RANDOM_SLICING_CUTSHIFT=ON` is recommended for +expand-heavy workflows because it avoids rebuilding an entirely new balanced placement from scratch. Instead, CutShift +adjusts the existing Random Slicing interval layout and assigns ranges to the added daemons, so most data remains on its +current owner and only a smaller subset has to move between nodes. + Reference: Random Slicing is based on "Random slicing: Efficient and scalable data placement for large-scale storage systems" by Alberto Miranda, Sascha Effert, Yangwook Kang, Ethan L. Miller, Ivan Popov, Andre Brinkmann, Tom Friedetzky, and Toni Cortes, published in ACM Transactions on Storage 10(3), 2014. See the [Google Scholar entry](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=hK-ogNAAAAAJ&cstart=20&pagesize=80&sortby=pubdate&citation_for_view=hK-ogNAAAAAJ:RGFaLdJalmkC). ### Manual `gkfs_malleability` workflow @@ -588,7 +788,7 @@ unset GKFS_DAEMON_EXPAND gkfs_malleability mutate start while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done -# finalize: stop '-' daemons, promote '+', preserve RS interval comments +# finalize: stop '-' daemons automatically, promote '+', preserve RS interval comments gkfs_malleability mutate finalize ``` @@ -601,6 +801,7 @@ sed -i 's/^node4 /-node4 /' /path/to/gkfs_workspace.txt export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt gkfs_malleability mutate start while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done +# finalize stops the daemon marked with '-' automatically gkfs_malleability mutate finalize ``` -- GitLab From f3b2964fe64a945b63794633dca2c35f576386fa Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 28 Aug 2026 22:04:54 +0200 Subject: [PATCH 18/21] fix bugs --- include/client/rpc/forward_malleability.hpp | 3 + include/common/common_defs.hpp | 1 + include/common/rpc/rpc_types_thallium.hpp | 9 + include/daemon/classes/fs_data.hpp | 17 ++ include/daemon/handler/rpc_defs.hpp | 4 + .../daemon/malleability/malleable_manager.hpp | 3 + src/client/malleability.cpp | 33 +++- src/client/rpc/forward_malleability.cpp | 75 ++++++++ src/daemon/classes/fs_data.cpp | 22 +++ src/daemon/daemon.cpp | 7 +- src/daemon/handler/srv_malleability.cpp | 23 ++- src/daemon/malleability/malleable_manager.cpp | 180 ++++++++++++++++++ src/daemon/ops/data.cpp | 84 ++++---- tests/integration/harness/gkfs.py | 6 + .../malleability/test_expand_on_demand.py | 79 ++++++++ .../test_malleability_performance.py | 99 +++++----- .../unit/test_random_slicing_distributor.cpp | 67 +++++++ tests/unit/test_random_slicing_pipeline.cpp | 113 +++++++++++ 18 files changed, 719 insertions(+), 106 deletions(-) diff --git a/include/client/rpc/forward_malleability.hpp b/include/client/rpc/forward_malleability.hpp index 6cecd857d..aa7ae94f6 100644 --- a/include/client/rpc/forward_malleability.hpp +++ b/include/client/rpc/forward_malleability.hpp @@ -54,6 +54,9 @@ forward_mutate_status(); int forward_mutate_finalize(); +int +forward_mutate_reload(const std::string& hosts_file); + int forward_mutate_shutdown_removed(const std::string& hostfile); diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index f12ba717d..0580bab95 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -132,6 +132,7 @@ namespace malleable::rpc::tag { constexpr auto mutate_start = "rpc_srv_mutate_start"; constexpr auto mutate_status = "rpc_srv_mutate_status"; constexpr auto mutate_finalize = "rpc_srv_mutate_finalize"; +constexpr auto mutate_reload = "rpc_srv_mutate_reload"; constexpr auto mutate_shutdown = "rpc_srv_mutate_shutdown"; // Migrate metadata (used by forward_metadata for RocksDB redistribution) constexpr auto migrate_metadata = "rpc_srv_migrate_metadata"; diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index 901563bed..6b0e06e94 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -458,6 +458,15 @@ struct rpc_mutate_start_in_t { } }; +struct rpc_mutate_reload_in_t { + std::string hosts_file; + template + void + serialize(Archive& ar) { + ar(hosts_file); + } +}; + struct rpc_migrate_metadata_in_t { std::string key; std::string value; diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 5ee983631..2faa58aac 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -47,6 +47,7 @@ #include //std::hash #include #include +#include /* Forward declarations */ namespace gkfs { @@ -143,7 +144,10 @@ private: bool expand_on_demand_active_ = false; unsigned int expand_on_demand_old_hosts_size_ = 0; + uint64_t expand_on_demand_old_local_host_id_ = + std::numeric_limits::max(); std::shared_ptr expand_on_demand_old_distributor_; + std::map expand_on_demand_old_rpc_endpoints_; std::shared_ptr malleable_manager_; @@ -171,6 +175,12 @@ public: void expand_on_demand_old_hosts_size(unsigned int hosts_size); + uint64_t + expand_on_demand_old_local_host_id() const; + + void + expand_on_demand_old_local_host_id(uint64_t host_id); + std::shared_ptr expand_on_demand_old_distributor() const; @@ -178,6 +188,13 @@ public: expand_on_demand_old_distributor( std::shared_ptr distributor); + const std::map& + expand_on_demand_old_rpc_endpoints() const; + + void + expand_on_demand_old_rpc_endpoints( + std::map endpoints); + // getter/setter const std::shared_ptr& diff --git a/include/daemon/handler/rpc_defs.hpp b/include/daemon/handler/rpc_defs.hpp index 6fa00ddb5..338fcaa9a 100644 --- a/include/daemon/handler/rpc_defs.hpp +++ b/include/daemon/handler/rpc_defs.hpp @@ -149,6 +149,10 @@ rpc_srv_mutate_status(const tl::request& req); void rpc_srv_mutate_finalize(const tl::request& req); +void +rpc_srv_mutate_reload(const tl::request& req, + const gkfs::rpc::rpc_mutate_reload_in_t& in); + void rpc_srv_mutate_shutdown(const tl::request& req); diff --git a/include/daemon/malleability/malleable_manager.hpp b/include/daemon/malleability/malleable_manager.hpp index 8593fb9fe..014fb0d94 100644 --- a/include/daemon/malleability/malleable_manager.hpp +++ b/include/daemon/malleability/malleable_manager.hpp @@ -85,6 +85,9 @@ public: void mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file); + + void + reload_hosts_file(const std::string& hosts_file); }; } // namespace gkfs::malleable diff --git a/src/client/malleability.cpp b/src/client/malleability.cpp index 9019d29c4..0894980aa 100644 --- a/src/client/malleability.cpp +++ b/src/client/malleability.cpp @@ -125,16 +125,29 @@ int mutate_finalize() { LOG(INFO, "{}() enter", __func__); auto res = gkfs::malleable::rpc::forward_mutate_finalize(); - if(res == 0) { - if(const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE")) { - auto shutdown_res = - gkfs::malleable::rpc::forward_mutate_shutdown_removed(hf); - if(shutdown_res != 0) { - LOG(ERROR, - "{}() failed to gracefully shutdown removed daemons: {}", - __func__, shutdown_res); - return shutdown_res; - } + const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE"); + if(res == 0 && hf != nullptr) { + // Keep the marker file intact while removed-daemon endpoints are + // collected and shut down. Cleanup below removes those markers only + // after the old daemons have been told to exit. + auto shutdown_res = + gkfs::malleable::rpc::forward_mutate_shutdown_removed(hf); + if(shutdown_res != 0) { + LOG(ERROR, "{}() failed to gracefully shutdown removed daemons: {}", + __func__, shutdown_res); + return shutdown_res; + } + + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + auto rs_intervals = gkfs::malleable::parse_rs_interval_comments(hf); + gkfs::malleable::write_clean_hostfile(hf, markers, rs_intervals); + LOG(INFO, "{}() wrote final clean hosts file '{}'", __func__, hf); + res = gkfs::malleable::rpc::forward_mutate_reload(hf); + } catch(const std::exception& e) { + LOG(ERROR, "{}() failed to clean/reload final hosts file: {}", + __func__, e.what()); + res = EIO; } } LOG(INFO, "{}() mutate operation finalized. ", __func__); diff --git a/src/client/rpc/forward_malleability.cpp b/src/client/rpc/forward_malleability.cpp index 3c63d0b11..a8c3b75c0 100644 --- a/src/client/rpc/forward_malleability.cpp +++ b/src/client/rpc/forward_malleability.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -159,6 +160,12 @@ forward_mutate_start(int old_server_conf, int new_server_conf, LOG(INFO, "{}() enter", __func__); auto targets = mutate_endpoints(&new_hosts_file, true); + if(targets.empty()) { + LOG(ERROR, "No mutate targets found for hosts file '{}'", + new_hosts_file); + return EINVAL; + } + auto err = 0; std::vector waiters; waiters.reserve(targets.size()); @@ -182,6 +189,7 @@ forward_mutate_start(int old_server_conf, int new_server_conf, waiter_targets.push_back(target); } catch(const std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); + err = EBUSY; } } @@ -296,6 +304,73 @@ forward_mutate_finalize() { return err; } +int +forward_mutate_reload(const std::string& hosts_file) { + LOG(INFO, "{}() enter", __func__); + std::vector> targets; + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hosts_file); + std::set seen_uris; + for(const auto& entry : markers.active) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + targets.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); + } + for(const auto& entry : markers.adding) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + targets.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); + } + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to parse final hosts file '{}': {}", hosts_file, + ex.what()); + return EINVAL; + } + if(targets.empty()) { + LOG(ERROR, "Final hosts file '{}' contains no active hosts", + hosts_file); + return EINVAL; + } + auto reload_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_reload); + + int err = 0; + std::vector waiters; + std::vector waiter_targets; + waiters.reserve(targets.size()); + waiter_targets.reserve(targets.size()); + for(const auto& [target, endpoint] : targets) { + try { + gkfs::rpc::rpc_mutate_reload_in_t in; + in.hosts_file = hosts_file; + waiters.push_back(reload_rpc.on(endpoint).async(in)); + waiter_targets.push_back(target); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to send reload RPC to host {}: {}", target, + ex.what()); + err = EBUSY; + } + } + for(size_t i = 0; i < waiters.size(); ++i) { + try { + const auto out = waiters[i].wait().as(); + if(out.err != 0) { + LOG(ERROR, "Failed reload on host '{}'", waiter_targets[i]); + err = out.err; + } + } catch(const std::exception& ex) { + LOG(ERROR, "Reload RPC wait failed for host {}: {}", + waiter_targets[i], ex.what()); + err = EBUSY; + } + } + return err; +} + int forward_mutate_shutdown_removed(const std::string& hostfile) { LOG(INFO, "{}() enter", __func__); diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 768ed3b80..4be3852dd 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -74,6 +74,16 @@ FsData::expand_on_demand_old_hosts_size(unsigned int hosts_size) { expand_on_demand_old_hosts_size_ = hosts_size; } +uint64_t +FsData::expand_on_demand_old_local_host_id() const { + return expand_on_demand_old_local_host_id_; +} + +void +FsData::expand_on_demand_old_local_host_id(uint64_t host_id) { + expand_on_demand_old_local_host_id_ = host_id; +} + std::shared_ptr FsData::expand_on_demand_old_distributor() const { return expand_on_demand_old_distributor_; @@ -85,6 +95,17 @@ FsData::expand_on_demand_old_distributor( expand_on_demand_old_distributor_ = std::move(distributor); } +const std::map& +FsData::expand_on_demand_old_rpc_endpoints() const { + return expand_on_demand_old_rpc_endpoints_; +} + +void +FsData::expand_on_demand_old_rpc_endpoints( + std::map endpoints) { + expand_on_demand_old_rpc_endpoints_ = std::move(endpoints); +} + const std::shared_ptr& FsData::spdlogger() const { return spdlogger_; @@ -376,6 +397,7 @@ FsData::maintenance_mode(bool maintenance_mode) { auto err_str = "Critical error: Maintenance mode enabled twice, e.g., due to multiple expand requests. This is not a allowed and should not happen."; spdlogger()->error(err_str); + ABT_mutex_unlock(maintenance_mode_mutex_); throw std::runtime_error(err_str); } maintenance_mode_ = maintenance_mode; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 4c99809b1..07f0b7801 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -342,6 +342,8 @@ register_server_rpcs(std::shared_ptr engine) { rpc_srv_mutate_status); engine->define(gkfs::malleable::rpc::tag::mutate_finalize, rpc_srv_mutate_finalize); + engine->define(gkfs::malleable::rpc::tag::mutate_reload, + rpc_srv_mutate_reload); engine->define(gkfs::malleable::rpc::tag::mutate_shutdown, rpc_srv_mutate_shutdown); engine->define(gkfs::malleable::rpc::tag::migrate_metadata, @@ -620,7 +622,10 @@ init_environment() { throw runtime_error("Failed to write root metadentry to KV store: "s + e.what()); } - // setup hostfile to let clients know that a daemon is running on this host + + // Publish the hostfile entry before distributor initialization. The + // distributor may read random-slicing metadata from this file, and fresh + // workspaces do not contain it yet. if(!GKFS_DATA->hosts_file().empty()) { gkfs::utils::populate_hosts_file(GKFS_DATA->expand_mode()); } diff --git a/src/daemon/handler/srv_malleability.cpp b/src/daemon/handler/srv_malleability.cpp index 02bde6764..9b9e8aeba 100644 --- a/src/daemon/handler/srv_malleability.cpp +++ b/src/daemon/handler/srv_malleability.cpp @@ -58,6 +58,7 @@ void rpc_srv_mutate_start(const tl::request& req, const gkfs::rpc::rpc_mutate_start_in_t& in) { gkfs::rpc::rpc_err_out_t out; + bool entered_maintenance = false; GKFS_DATA->spdlogger()->debug( "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", @@ -65,13 +66,16 @@ rpc_srv_mutate_start(const tl::request& req, in.new_hosts_file); try { GKFS_DATA->maintenance_mode(true); + entered_maintenance = true; GKFS_DATA->malleable_manager()->mutate_start( in.old_server_conf, in.new_server_conf, in.new_hosts_file); out.err = 0; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error("{}() Failed to start mutate: '{}' ", __func__, e.what()); - GKFS_DATA->maintenance_mode(false); + if(entered_maintenance) { + GKFS_DATA->maintenance_mode(false); + } out.err = -1; } @@ -116,6 +120,23 @@ rpc_srv_mutate_finalize(const tl::request& req) { gkfs::utils::safe_respond(req, out); } +void +rpc_srv_mutate_reload(const tl::request& req, + const gkfs::rpc::rpc_mutate_reload_in_t& in) { + gkfs::rpc::rpc_err_out_t out; + GKFS_DATA->spdlogger()->debug("{}() Got RPC for hosts file '{}'", __func__, + in.hosts_file); + try { + GKFS_DATA->malleable_manager()->reload_hosts_file(in.hosts_file); + out.err = 0; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error("{}() Failed to reload hosts file: '{}'", + __func__, e.what()); + out.err = -1; + } + gkfs::utils::safe_respond(req, out); +} + void rpc_srv_mutate_shutdown(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index fbe6f7563..84b352a6b 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -155,14 +155,45 @@ rs_comments_to_old_id_intervals( const vector>& before_hosts) { vector intervals; intervals.reserve(comments.size()); + unordered_map seen_hosts; for(const auto& comment : comments) { if(comment.host_id >= before_hosts.size()) { return {}; } + if(!std::isfinite(comment.start) || !std::isfinite(comment.end) || + comment.start < 0.0 || comment.end > 1.0 || + comment.start >= comment.end) { + return {}; + } + seen_hosts.emplace(static_cast(comment.host_id), + true); intervals.push_back({static_cast(comment.start), static_cast(comment.end), static_cast(comment.host_id)}); } + + // A raced hostfile may already contain the post-shrink table. Such a table + // can still cover [0, 1), but it is not a valid old layout when it lacks + // one or more old host IDs. Reject it so the caller rebuilds the complete + // pre-shrink layout instead of mixing post-shrink intervals with old IDs. + if(seen_hosts.size() != before_hosts.size()) { + return {}; + } + + auto sorted = intervals; + sort(sorted.begin(), sorted.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + constexpr double epsilon = 1.0e-5; + if(sorted.empty() || std::abs(sorted.front().start) > epsilon || + std::abs(sorted.back().end - 1.0) > epsilon) { + return {}; + } + for(size_t i = 1; i < sorted.size(); ++i) { + if(std::abs(sorted[i].start - sorted[i - 1].end) > epsilon) { + return {}; + } + } + return intervals; } @@ -209,6 +240,38 @@ remap_compact_survivor_partitions_to_after_ids( return partitions; } +static string +rs_intervals_to_string(const vector& intervals) { + ostringstream os; + os << "["; + for(size_t i = 0; i < intervals.size(); ++i) { + if(i != 0) { + os << ", "; + } + os << "{host=" << intervals[i].host_id << ", start=" << fixed + << setprecision(9) << intervals[i].start + << ", end=" << intervals[i].end << "}"; + } + os << "]"; + return os.str(); +} + +static string +rs_partitions_to_string(const vector& partitions) { + ostringstream os; + os << "["; + for(size_t i = 0; i < partitions.size(); ++i) { + if(i != 0) { + os << ", "; + } + os << "{host=" << partitions[i].host_id + << ", intervals=" << rs_intervals_to_string(partitions[i].intervals) + << "}"; + } + os << "]"; + return os.str(); +} + static vector remap_existing_rs_comments( const vector& comments, @@ -309,6 +372,7 @@ MalleableManager::load_hostfile(const std::string& path) { if(idx != string::npos) h.first.erase(idx, h.first.length()); } + return hosts; } @@ -732,6 +796,41 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, h.first.erase(idx, h.first.length()); } + // Save the actual pre-mutate map before connect_to_hosts() replaces it + // with the final topology. IDs in this map belong to the old distributor. + auto old_rpc_endpoints = RPC_DATA->rpc_endpoints(); + auto old_local_host_id = RPC_DATA->local_host_id(); + if(old_server_conf < new_server_conf && markers.removing.empty()) { + if(old_rpc_endpoints.size() < old_hosts_size_) { + vector> old_hosts; + old_hosts.reserve(markers.active.size()); + for(const auto& e : markers.active) { + old_hosts.emplace_back(e.hostname, e.uri); + } + sort(old_hosts.begin(), old_hosts.end()); + old_rpc_endpoints.clear(); + old_local_host_id = std::numeric_limits::max(); + for(size_t id = 0; id < old_hosts.size(); ++id) { + if(old_hosts[id].second == RPC_DATA->self_addr_str()) { + old_local_host_id = static_cast(id); + } + try { + old_rpc_endpoints.emplace( + static_cast(id), + RPC_DATA->client_rpc_engine()->lookup( + old_hosts[id].second)); + } catch(const std::exception& e) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to lookup old host '{}': {}", + __func__, old_hosts[id].second, e.what())); + } + } + } + } else { + old_rpc_endpoints.clear(); + old_local_host_id = std::numeric_limits::max(); + } + if(hosts.size() != static_cast(new_server_conf)) { throw runtime_error( fmt::format("MalleableManager::{}() Something is wrong. " @@ -843,18 +942,36 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, ? vector{} : rs_comments_to_old_id_intervals( comments, before_hosts); + if(!comments.empty() && old_intervals.empty()) { + GKFS_DATA->spdlogger()->warn( + "{}() No usable pre-shrink RS interval comments found; rebuilding equal old layout", + __func__); + } auto old_partitions = old_intervals.empty() ? make_equal_rs_partitions_for_before_hosts( before_hosts) : intervals_to_partitions(old_intervals); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink old RS intervals: {}", __func__, + rs_intervals_to_string(old_intervals)); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink old partitions: {}", __func__, + rs_partitions_to_string(old_partitions)); + updated_partitions = gkfs::rpc::shrink_with_cutshift( old_partitions, removed_old_ids); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink compact partitions before after-id remap: {}", + __func__, rs_partitions_to_string(updated_partitions)); updated_partitions = remap_compact_survivor_partitions_to_after_ids( std::move(updated_partitions), before_hosts, hosts, removed_old_ids); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink final after-id partitions: {}", + __func__, rs_partitions_to_string(updated_partitions)); } if(!added_ids.empty() && !updated_partitions.empty()) { @@ -904,7 +1021,10 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, if(expand_on_demand && pure_expand) { GKFS_DATA->expand_on_demand_active(true); GKFS_DATA->expand_on_demand_old_hosts_size(old_hosts_size_); + GKFS_DATA->expand_on_demand_old_local_host_id(old_local_host_id); GKFS_DATA->expand_on_demand_old_distributor(std::move(old_distributor)); + GKFS_DATA->expand_on_demand_old_rpc_endpoints( + std::move(old_rpc_endpoints)); GKFS_DATA->spdlogger()->info( "{}() {} active: old_hosts={}, new_hosts={}. Skipping eager data migration.", __func__, gkfs::env::EXPAND_ON_DEMAND, old_hosts_size_, @@ -935,4 +1055,64 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, } } +void +MalleableManager::reload_hosts_file(const std::string& hosts_file) { + const auto hosts = [&]() { + try { + return load_hostfile(hosts_file); + } catch(const std::exception& e) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to load final hosts file '{}': {}", + __func__, hosts_file, e.what())); + } + }(); + + if(hosts.empty()) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Final hosts file '{}' is empty", + __func__, hosts_file)); + } + + connect_to_hosts(hosts, false); + + gkfs::rpc::DistributionConfig config; + if(const auto* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + + auto distributor = gkfs::rpc::create_from_config( + config, static_cast(RPC_DATA->local_host_id()), + static_cast(hosts.size()), 0); + if(!distributor) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to recreate distributor", + __func__)); + } + + if(config.is_random_slicing()) { + auto comments = gkfs::malleable::parse_rs_interval_comments(hosts_file); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + auto* rs = dynamic_cast( + distributor.get()); + if(rs && !intervals.empty() && !rs->set_intervals(intervals)) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Invalid final random-slicing intervals", + __func__)); + } + } + + RPC_DATA->distributor(std::move(distributor)); + GKFS_DATA->spdlogger()->info( + "{}() Reloaded final hosts file '{}' with {} hosts", __func__, + hosts_file, hosts.size()); +} + } // namespace gkfs::malleable diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index 13d4a73cd..d760ffe05 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -46,7 +46,6 @@ #include #include #include -#include extern "C" { #include @@ -80,35 +79,10 @@ expand_on_demand_owns_final_missing_candidate(const std::string& path, old_owner = old_dist->locate_data( path, static_cast(chunk_id), 0); - return old_owner != local && + return old_owner != GKFS_DATA->expand_on_demand_old_local_host_id() && old_owner < GKFS_DATA->expand_on_demand_old_hosts_size() && - RPC_DATA->rpc_endpoints().find(old_owner) != - RPC_DATA->rpc_endpoints().end(); -} - -void -async_materialize_local_full_read(const std::string& path, uint64_t chunk_id, - const char* buf, ssize_t size, - off64_t offset) { - if(size <= 0 || offset != 0) { - return; - } - - std::vector materialized(buf, buf + size); - std::thread([path, chunk_id, data = std::move(materialized)]() { - try { - GKFS_DATA->storage()->write_chunk( - path, static_cast(chunk_id), - data.data(), data.size(), 0); - GKFS_DATA->spdlogger()->debug( - "{}() expand-on-demand materialized chunk '{}' for '{}' ({} bytes)", - __func__, chunk_id, path, data.size()); - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->warn( - "{}() expand-on-demand async materialization failed for '{}' chunk '{}': {}", - __func__, path, chunk_id, e.what()); - } - }).detach(); + GKFS_DATA->expand_on_demand_old_rpc_endpoints().find(old_owner) != + GKFS_DATA->expand_on_demand_old_rpc_endpoints().end(); } } // namespace @@ -122,8 +96,9 @@ expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, return {ENOENT, 0}; } + std::vector full_chunk(gkfs::config::rpc::chunksize); std::vector> segments = { - std::make_pair(buf, size)}; + std::make_pair(full_chunk.data(), full_chunk.size())}; tl::bulk bulk_handle; try { bulk_handle = RPC_DATA->client_rpc_engine()->expose( @@ -137,7 +112,7 @@ expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, gkfs::rpc::rpc_read_data_in_t in{}; in.path = path; - in.offset = offset; + in.offset = 0; in.host_id = old_owner; in.host_size = GKFS_DATA->expand_on_demand_old_hosts_size(); std::vector bitset(1, 1); @@ -145,23 +120,47 @@ expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, in.chunk_n = 1; in.chunk_start = chunk_id; in.chunk_end = chunk_id; - in.total_chunk_size = size; + in.total_chunk_size = full_chunk.size(); in.bulk_handle = bulk_handle; try { auto read_data = RPC_DATA->client_rpc_engine()->define(gkfs::rpc::tag::read); - auto out = read_data.on(RPC_DATA->rpc_endpoints().at(old_owner))(in) - .as(); - if(out.err == 0 && out.io_size > 0) { - async_materialize_local_full_read(path, chunk_id, buf, - static_cast(out.io_size), - offset); + auto out = + read_data + .on(GKFS_DATA->expand_on_demand_old_rpc_endpoints().at( + old_owner))(in) + .as(); + if(out.err != 0 || out.io_size == 0) { + GKFS_DATA->spdlogger()->debug( + "{}() expand-on-demand fallback read '{}' chunk '{}' from old owner '{}': err={} size={}", + __func__, path, chunk_id, old_owner, out.err, out.io_size); + return {out.err, static_cast(out.io_size)}; + } + + try { + GKFS_DATA->storage()->write_chunk( + path, static_cast(chunk_id), + full_chunk.data(), static_cast(out.io_size), 0); + } catch(const gkfs::data::ChunkStorageException& e) { + GKFS_DATA->spdlogger()->error( + "{}() failed to materialize fallback chunk '{}' for '{}': {}", + __func__, chunk_id, path, e.what()); + return {e.code().value(), 0}; + } + const auto available = + offset >= 0 && static_cast(offset) < out.io_size + ? out.io_size - static_cast(offset) + : 0; + const auto copied = std::min(size, available); + if(copied > 0) { + std::memcpy(buf, full_chunk.data() + static_cast(offset), + copied); } GKFS_DATA->spdlogger()->debug( "{}() expand-on-demand fallback read '{}' chunk '{}' from old owner '{}': err={} size={}", __func__, path, chunk_id, old_owner, out.err, out.io_size); - return {out.err, static_cast(out.io_size)}; + return {0, static_cast(copied)}; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() expand-on-demand fallback read failed for '{}' chunk '{}' from old owner '{}': {}", @@ -205,13 +204,6 @@ expand_on_demand_materialize_for_partial_write(const std::string& path, if(read_size <= 0) { return 0; } - try { - GKFS_DATA->storage()->write_chunk( - path, static_cast(chunk_id), chunk.data(), - static_cast(read_size), 0); - } catch(const gkfs::data::ChunkStorageException& e) { - return e.code().value(); - } return 0; } diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index c84144010..ac19a5333 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -1541,6 +1541,12 @@ class FwdDaemon: # ... or it might just be lazy. let's give it some more time logger.debug(f"daemon {pid} found, retrying...") + if self._proc.poll() is not None: + raise RuntimeError( + f"daemon {pid} exited during initialization with code " + f"{self._proc.returncode}; log: " + f"{self.logdir / gkfwd_daemon_log_file}" + ) time.sleep(1) raise RuntimeError("initialization timeout exceeded") diff --git a/tests/integration/malleability/test_expand_on_demand.py b/tests/integration/malleability/test_expand_on_demand.py index 9f781ae39..acbd25723 100644 --- a/tests/integration/malleability/test_expand_on_demand.py +++ b/tests/integration/malleability/test_expand_on_demand.py @@ -144,6 +144,85 @@ def test_expand_on_demand_skips_eager_data_and_keeps_reads_correct( assert ret.retval == len(PARTIAL_OVERWRITE) assert _read_md5(gkfs_client, path) == expected_after_partial_md5 + finally: + for daemon in daemons: + daemon.shutdown() + + +def test_expand_on_demand_2_to_4_read_and_partial_write( + monkeypatch, gkfwd_daemon_factory, gkfs_client, gkfs_shell +): + """Verify lazy full-chunk reads and partial writes across a 2->4 expand.""" + monkeypatch.setenv("GKFS_EXPAND_ON_DEMAND", "ON") + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", "simple_hash") + + daemons = [] + try: + old_daemons = [ + gkfwd_daemon_factory.create(), + gkfwd_daemon_factory.create(), + ] + daemons.extend(old_daemons) + time.sleep(3) + + hostfile = Path(old_daemons[0].hostfile) + md5_map = {} + for i in range(FILE_COUNT): + path = old_daemons[0].mountdir / f"expand_on_demand_2to4_{i:03d}.dat" + md5_map[path] = _write_file(gkfs_client, path) + + for path, expected_md5 in md5_map.items(): + assert _read_md5(gkfs_client, path) == expected_md5 + + new_daemons = [ + gkfwd_daemon_factory.create(expand_mode=True), + gkfwd_daemon_factory.create(expand_mode=True), + ] + daemons.extend(new_daemons) + time.sleep(3) + + lines = hostfile.read_text().splitlines() + for daemon in new_daemons: + port = daemon.address.rsplit(":", 1)[-1] + matching = [line for line in lines if f":{port}" in line] + assert len(matching) == 1, f"Missing hostfile entry for {daemon.address}" + assert matching[0].startswith("+"), ( + f"Expected '+' marker for {daemon.address}: {matching[0]}" + ) + + _run_mutate_cmd(gkfs_shell, hostfile, "mutate start", 340) + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") + + # A full read must fetch a missing chunk from the old 2-node layout, + # materialize it on the final owner, and return the original bytes. + for path, expected_md5 in md5_map.items(): + assert _read_md5(gkfs_client, path) == expected_md5 + + partial_path = next(iter(md5_map)) + expected_payload = bytearray( + (ord("0") + (i % 10)) for i in range(FILE_SIZE) + ) + expected_payload[ + PARTIAL_OVERWRITE_OFFSET : + PARTIAL_OVERWRITE_OFFSET + len(PARTIAL_OVERWRITE) + ] = PARTIAL_OVERWRITE + ret = gkfs_client.pwrite( + partial_path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + assert _read_md5(gkfs_client, partial_path) == hashlib.md5( + expected_payload + ).hexdigest() + + # Read another file after the partial write to ensure no stale or + # unrelated materialization changed other chunks. + for path, expected_md5 in list(md5_map.items())[1:]: + assert _read_md5(gkfs_client, path) == expected_md5 finally: for daemon in daemons: daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py index 18186e84d..df4336e9b 100644 --- a/tests/integration/malleability/test_malleability_performance.py +++ b/tests/integration/malleability/test_malleability_performance.py @@ -388,25 +388,19 @@ def describe_returncode(returncode): def count_accessible_files(file_paths, client, timeout=60): - """Count accessible files with one intercepted shell process.""" + """Count files visible through the GekkoFS client API.""" if not file_paths: return 0 - quoted = " ".join(shlex.quote(str(p)) for p in file_paths) - cmd = f"ok=0; for f in {quoted}; do [ -f \"$f\" ] && ok=$((ok+1)); done; echo $ok" - completed = subprocess.run( - ["bash", "-c", cmd], - env=getattr(client, "_env", os.environ), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - timeout=timeout, - ) - if completed.returncode != 0: - logger.warning(f"fast accessibility check failed: {completed.stderr.decode()[:300]}") - return sum(1 for fpath in file_paths if client.stat(fpath, timeout=15).retval == 0) - try: - return int(completed.stdout.decode().strip() or "0") - except ValueError: - return 0 + accessible = 0 + for fpath in file_paths: + for attempt in range(3): + result = client.stat(fpath, timeout=min(timeout, 15)) + if result.retval == 0: + accessible += 1 + break + if attempt < 2: + time.sleep(0.25) + return accessible def verify_files_with_md5(file_md5_map, client, mountdir, max_read_checks=None): @@ -664,6 +658,20 @@ def _dump_mutate_failure_context(label, cmd, workspace_file): logger.error(f"failed to read {daemon_log}: {exc}") +def _run_malleability(gkfs_shell, action, workspace_file, timeout=60): + """Run gkfs_malleability without an extra shell process.""" + return gkfs_shell.run( + "gkfs_malleability", + "mutate", + action, + timeout=timeout, + env={ + "LIBGKFS_HOSTS_FILE": str(workspace_file), + "LD_PRELOAD": "", + }, + ) + + # ============ # Test: shrink # ============ @@ -753,8 +761,9 @@ def test_shrink_performance_multi_run(client_fixture, # Execute shrink t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) if cmd.exit_code != 0: _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) assert cmd.exit_code == 0, f"Shrink start failed: {cmd.stderr.decode()[:300]}" @@ -762,8 +771,7 @@ def test_shrink_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "status", workspace_file) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(MUTATE_STATUS_POLL_INTERVAL) @@ -774,8 +782,7 @@ def test_shrink_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -936,8 +943,9 @@ def test_expand_performance_multi_run(client_fixture, # Execute expand t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) if cmd.exit_code != 0: _dump_mutate_failure_context("Expand mutate start", cmd, workspace_file) assert cmd.exit_code == 0, f"Expand start failed: {cmd.stderr.decode()[:300]}" @@ -945,8 +953,7 @@ def test_expand_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "status", workspace_file) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(MUTATE_STATUS_POLL_INTERVAL) @@ -957,8 +964,7 @@ def test_expand_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -1109,8 +1115,9 @@ def test_mutate_performance_multi_run(client_fixture, # Execute mutate t0 = perf_counter() t_wall = time.time() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) if cmd.exit_code != 0: _dump_mutate_failure_context("Mutate start", cmd, workspace_file) result = OperationResult( @@ -1129,8 +1136,7 @@ def test_mutate_performance_multi_run(client_fixture, # Wait for completion deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "status", workspace_file) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(MUTATE_STATUS_POLL_INTERVAL) @@ -1139,8 +1145,7 @@ def test_mutate_performance_multi_run(client_fixture, wall = time.time() - t_wall # Finalize - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_file) @@ -1276,23 +1281,22 @@ def test_comprehensive_cycle_performance(client_fixture, ) shrink_t0 = perf_counter() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + cmd = _run_malleability( + gkfs_shell, "start", workspace_shrink, DEFAULT_TIMEOUT + ) if cmd.exit_code != 0: _dump_mutate_failure_context("Cycle shrink mutate start", cmd, workspace_shrink) assert cmd.exit_code == 0, f"Cycle shrink start failed: {cmd.stderr.decode()[:300]}" deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "status", workspace_shrink) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(MUTATE_STATUS_POLL_INTERVAL) shrink_elapsed = perf_counter() - shrink_t0 - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "finalize", workspace_shrink) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_shrink) @@ -1341,13 +1345,14 @@ def test_comprehensive_cycle_performance(client_fixture, ) expand_t0 = perf_counter() - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate start"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=DEFAULT_TIMEOUT) + cmd = _run_malleability( + gkfs_shell, "start", workspace_expand, DEFAULT_TIMEOUT + ) if cmd.exit_code != 0: _dump_mutate_failure_context("Cycle expand mutate start", cmd, workspace_expand) assert cmd.exit_code == 0, ( "Cycle expand start failed:\n" - f"cmd: {cmd_str}\n" + f"cmd: gkfs_malleability mutate start\n" f"stdout: {cmd.stdout.decode(errors='replace')[:1000]}\n" f"stderr: {cmd.stderr.decode(errors='replace')[:1000]}\n" f"workspace:\n{workspace_expand.read_text(errors='replace')[:2000]}" @@ -1355,15 +1360,13 @@ def test_comprehensive_cycle_performance(client_fixture, deadline = time.time() + 120 while time.time() < deadline: - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate status"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "status", workspace_expand) if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(MUTATE_STATUS_POLL_INTERVAL) expand_elapsed = perf_counter() - expand_t0 - cmd_str = f'bash -c "source {env_file} && gkfs_malleability mutate finalize"' - cmd = gkfs_shell.script(cmd_str, intercept_shell=False) + cmd = _run_malleability(gkfs_shell, "finalize", workspace_expand) assert cmd.exit_code == 0 _set_client_hostfile(client, workspace_expand) diff --git a/tests/unit/test_random_slicing_distributor.cpp b/tests/unit/test_random_slicing_distributor.cpp index e002cb561..a1ac78436 100644 --- a/tests/unit/test_random_slicing_distributor.cpp +++ b/tests/unit/test_random_slicing_distributor.cpp @@ -25,11 +25,40 @@ #include #include +#include #include #include #include #include +namespace { + +float +coverage(const std::vector& intervals) { + float sum = 0.0f; + for(const auto& interval : intervals) { + sum += interval.end - interval.start; + } + return sum; +} + +std::vector +make_equal_partitions(unsigned int hosts) { + std::vector partitions; + partitions.reserve(hosts); + const auto step = 1.0f / static_cast(hosts); + for(unsigned int host = 0; host < hosts; ++host) { + gkfs::rpc::Partition partition; + partition.host_id = host; + partition.intervals.push_back({host * step, (host + 1) * step, host}); + partition.total_capacity = step; + partitions.push_back(partition); + } + return partitions; +} + +} // namespace + // ===== Basic interface tests ===== TEST_CASE("RandomSlicingDistributor basic construction", "[common][distributor][random_slicing]") { @@ -241,6 +270,44 @@ TEST_CASE("RandomSlicing interval comments round-trip through hostfile", std::filesystem::remove(path); } +TEST_CASE("CutShift shrink 4 to 2 handles arbitrary removed old ids", + "[common][malleability][random_slicing][cutshift]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + auto shrunk = gkfs::rpc::shrink_with_cutshift( + make_equal_partitions(4), removed); + + REQUIRE(shrunk.size() == 2); + REQUIRE(shrunk[0].host_id == 0); + REQUIRE(shrunk[1].host_id == 1); + REQUIRE(coverage(shrunk[0].intervals) == Catch::Approx(0.5f)); + REQUIRE(coverage(shrunk[1].intervals) == Catch::Approx(0.5f)); + + std::vector intervals; + for(const auto& partition : shrunk) { + for(const auto& interval : partition.intervals) { + REQUIRE(interval.host_id == partition.host_id); + intervals.push_back(interval); + } + } + + std::sort(intervals.begin(), intervals.end(), [](const auto& a, + const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } + } +} + // ===== Compare with SimpleHash: deterministic across same config ===== TEST_CASE("RandomSlicingDistributor produces different mapping than SimpleHash", "[common][distributor][random_slicing]") { diff --git a/tests/unit/test_random_slicing_pipeline.cpp b/tests/unit/test_random_slicing_pipeline.cpp index 2f9f855d5..e58afde9e 100644 --- a/tests/unit/test_random_slicing_pipeline.cpp +++ b/tests/unit/test_random_slicing_pipeline.cpp @@ -262,6 +262,119 @@ TEST_CASE("Pipeline: RandomSlicing CutShift shrink only moves removed-node chunk REQUIRE(moved_from_survivors == 0); } +TEST_CASE("Pipeline: RandomSlicing 4-to-2 preserves physical survivor ownership", + "[pipeline][cutshift][random_slicing][shrink]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + std::vector old_partitions; + for(host_t host = 0; host < 4; ++host) { + Partition partition; + partition.host_id = host; + partition.total_capacity = 0.25f; + partition.intervals.push_back( + {static_cast(host) / 4.0f, + static_cast(host + 1) / 4.0f, host}); + old_partitions.push_back(partition); + } + + auto new_partitions = shrink_with_cutshift(old_partitions, removed); + REQUIRE(new_partitions.size() == 2); + + std::vector survivors; + for(host_t host = 0; host < 4; ++host) { + if(std::find(removed.begin(), removed.end(), host) == removed.end()) { + survivors.push_back(host); + } + } + + for(size_t compact_id = 0; compact_id < survivors.size(); ++compact_id) { + REQUIRE(new_partitions[compact_id].host_id == compact_id); + bool original_range_preserved = false; + float preserved = 0.0f; + const auto old_start = static_cast(survivors[compact_id]) / 4.0f; + const auto old_end = static_cast(survivors[compact_id] + 1) / 4.0f; + for(const auto& interval : new_partitions[compact_id].intervals) { + REQUIRE(interval.host_id == compact_id); + preserved += std::max(0.0f, std::min(interval.end, old_end) - + std::max(interval.start, old_start)); + } + original_range_preserved = preserved == Catch::Approx(0.25f); + REQUIRE(original_range_preserved); + } + + std::vector intervals; + for(const auto& partition : new_partitions) { + intervals.insert(intervals.end(), partition.intervals.begin(), + partition.intervals.end()); + } + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } + } +} + +TEST_CASE("Pipeline: RandomSlicing 4-to-2 hash ownership matches migration layout", + "[pipeline][cutshift][random_slicing][shrink][integrity]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + RandomSlicingDistributor old_dist(0, 4); + const auto old_partitions = old_dist.get_partitions_copy(); + const auto compact_partitions = + shrink_with_cutshift(old_partitions, removed); + + RandomSlicingDistributor new_dist(0, 2); + std::vector final_intervals; + for(const auto& partition : compact_partitions) { + final_intervals.insert(final_intervals.end(), + partition.intervals.begin(), + partition.intervals.end()); + } + REQUIRE(new_dist.set_intervals(final_intervals)); + + std::vector survivor_to_compact; + for(host_t old_host = 0; old_host < 4; ++old_host) { + if(std::find(removed.begin(), removed.end(), old_host) == + removed.end()) { + survivor_to_compact.push_back(old_host); + } + } + + for(uint64_t file = 0; file < 256; ++file) { + const auto path = "/integrity-4to2/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < 256; ++chunk) { + const auto old_owner = old_dist.locate_data(path, chunk, 0); + const auto new_owner = new_dist.locate_data(path, chunk, 0); + REQUIRE(new_owner < 2); + + if(std::find(removed.begin(), removed.end(), old_owner) == + removed.end()) { + const auto survivor_index = static_cast( + std::find(survivor_to_compact.begin(), + survivor_to_compact.end(), old_owner) - + survivor_to_compact.begin()); + CAPTURE(file, chunk, old_owner, new_owner, survivor_index); + REQUIRE(new_owner == survivor_index); + } + } + } + } +} + TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", "[pipeline][migrator][random_slicing]") { auto rs = create_distributor_from_string("random_slicing", 0, 3); -- GitLab From f70d54e52ddcdf277370e5758b9e8c19e8f47498 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Sat, 29 Aug 2026 11:07:26 +0200 Subject: [PATCH 19/21] Enhance client endpoint lookups with retry logic to improve test reliability. Fix a buffer overread in sfind.cpp by validating d_reclen against struct offsets and remaining bytes. Standardize the malleability test suite by extracting a shared execution helper and removing redundant shell wrappers. --- examples/gfind/sfind.cpp | 16 +++-- include/daemon/classes/fs_data.hpp | 13 +++++ src/client/rpc/forward_malleability.cpp | 58 +++++++++++++++++-- src/daemon/classes/fs_data.cpp | 31 ++++++++++ src/daemon/handler/srv_malleability.cpp | 14 +++++ src/daemon/malleability/malleable_manager.cpp | 21 ++++--- .../test_malleability_performance.py | 6 +- 7 files changed, 141 insertions(+), 18 deletions(-) diff --git a/examples/gfind/sfind.cpp b/examples/gfind/sfind.cpp index eb15a3594..d0769fca4 100644 --- a/examples/gfind/sfind.cpp +++ b/examples/gfind/sfind.cpp @@ -302,11 +302,15 @@ worker_routine(void* arg) { if(n > 0 && entries && !data->opt->just_count) { char* ptr = reinterpret_cast(entries); - int bytes_processed = 0; + size_t bytes_processed = 0; while(bytes_processed < n) { struct dirent_extended* temp = reinterpret_cast(ptr); - if(temp->d_reclen == 0) + const auto remaining = + static_cast(n) - bytes_processed; + if(temp->d_reclen < + offsetof(struct dirent_extended, d_name) + 1 || + temp->d_reclen > remaining) break; local_found++; @@ -360,11 +364,15 @@ worker_routine(void* arg) { } char* ptr = reinterpret_cast(entries); - int bytes_processed = 0; + size_t bytes_processed = 0; while(bytes_processed < n) { struct dirent_extended* temp = reinterpret_cast(ptr); - if(temp->d_reclen == 0) + const auto remaining = + static_cast(n) - bytes_processed; + if(temp->d_reclen < + offsetof(struct dirent_extended, d_name) + 1 || + temp->d_reclen > remaining) break; if(temp->d_type != 1) { diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 2faa58aac..a3c27c0fe 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -139,6 +139,10 @@ private: // indicates for clients: try again. Is set to true when redist is running bool maintenance_mode_ = false; ABT_mutex maintenance_mode_mutex_; + bool mutate_start_active_ = false; + int mutate_old_server_conf_ = 0; + int mutate_new_server_conf_ = 0; + std::string mutate_hosts_file_; // redist_running_ indicates to client that redistribution is running std::atomic redist_running_{false}; @@ -370,6 +374,15 @@ public: void maintenance_mode(bool maintenance_mode); + // Returns true when this request is a duplicate of the active mutation, + // false when a new mutation was registered, and throws for a conflict. + bool + begin_mutate_start(int old_server_conf, int new_server_conf, + const std::string& hosts_file); + + void + end_mutate_start(); + bool redist_running() const; diff --git a/src/client/rpc/forward_malleability.cpp b/src/client/rpc/forward_malleability.cpp index a8c3b75c0..0854ebafc 100644 --- a/src/client/rpc/forward_malleability.cpp +++ b/src/client/rpc/forward_malleability.cpp @@ -31,12 +31,33 @@ #include #include +#include #include #include +#include #include namespace { +thallium::endpoint +lookup_mutate_endpoint(const std::string& uri) { + constexpr unsigned int max_attempts = 10; + for(unsigned int attempt = 0; attempt < max_attempts; ++attempt) { + try { + return CTX->rpc_engine()->lookup(uri); + } catch(const std::exception& ex) { + if(attempt + 1 == max_attempts) { + throw; + } + LOG(DEBUG, + "Lookup of mutate endpoint '{}' failed (attempt {}/{}): {}", + uri, attempt + 1, max_attempts, ex.what()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + throw std::runtime_error("unreachable endpoint lookup failure"); +} + std::vector> mutate_endpoints_from_loaded_hosts() { std::vector> endpoints; @@ -74,7 +95,7 @@ mutate_endpoints_from_markers(const std::string& hostfile, try { LOG(DEBUG, "Looking up mutate endpoint '{}'", entry.uri); endpoints.emplace_back(entry.uri, - CTX->rpc_engine()->lookup(entry.uri)); + lookup_mutate_endpoint(entry.uri)); } catch(const std::exception& ex) { LOG(ERROR, "Failed to lookup mutate endpoint '{}': {}", entry.uri, ex.what()); @@ -109,7 +130,7 @@ mutate_removed_endpoints_from_markers(const std::string& hostfile) { try { LOG(DEBUG, "Looking up removed mutate endpoint '{}'", entry.uri); endpoints.emplace_back(entry.uri, - CTX->rpc_engine()->lookup(entry.uri)); + lookup_mutate_endpoint(entry.uri)); } catch(const std::exception& ex) { LOG(ERROR, "Failed to lookup removed mutate endpoint '{}': {}", entry.uri, ex.what()); @@ -160,10 +181,33 @@ forward_mutate_start(int old_server_conf, int new_server_conf, LOG(INFO, "{}() enter", __func__); auto targets = mutate_endpoints(&new_hosts_file, true); - if(targets.empty()) { + std::size_t expected_targets = targets.size(); + try { + const auto markers = + gkfs::malleable::parse_hostfile_markers(new_hosts_file); + std::set expected_uris; + for(const auto& entry : markers.active) { + expected_uris.insert(entry.uri); + } + for(const auto& entry : markers.adding) { + expected_uris.insert(entry.uri); + } + for(const auto& entry : markers.removing) { + expected_uris.insert(entry.uri); + } + expected_targets = expected_uris.size(); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to validate mutate targets from '{}': {}", + new_hosts_file, ex.what()); + return EINVAL; + } + + if(targets.empty() || targets.size() != expected_targets) { LOG(ERROR, "No mutate targets found for hosts file '{}'", new_hosts_file); - return EINVAL; + LOG(ERROR, "Expected {} mutate targets, resolved {}", expected_targets, + targets.size()); + return EBUSY; } auto err = 0; @@ -201,7 +245,11 @@ forward_mutate_start(int old_server_conf, int new_server_conf, try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); if(out.err != 0) { - err = out.err; + LOG(ERROR, "Mutate start failed on target '{}' with error {}", + waiter_targets[i], out.err); + if(err == 0) { + err = out.err; + } } } catch(const std::exception& ex) { LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index 4be3852dd..6680ad76e 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -404,6 +404,37 @@ FsData::maintenance_mode(bool maintenance_mode) { ABT_mutex_unlock(maintenance_mode_mutex_); } +bool +FsData::begin_mutate_start(int old_server_conf, int new_server_conf, + const std::string& hosts_file) { + ABT_mutex_lock(maintenance_mode_mutex_); + if(mutate_start_active_) { + const bool duplicate = mutate_old_server_conf_ == old_server_conf && + mutate_new_server_conf_ == new_server_conf && + mutate_hosts_file_ == hosts_file; + ABT_mutex_unlock(maintenance_mode_mutex_); + if(duplicate) { + return true; + } + throw std::runtime_error( + "A different mutate operation is already active"); + } + mutate_start_active_ = true; + mutate_old_server_conf_ = old_server_conf; + mutate_new_server_conf_ = new_server_conf; + mutate_hosts_file_ = hosts_file; + ABT_mutex_unlock(maintenance_mode_mutex_); + return false; +} + +void +FsData::end_mutate_start() { + ABT_mutex_lock(maintenance_mode_mutex_); + mutate_start_active_ = false; + mutate_hosts_file_.clear(); + ABT_mutex_unlock(maintenance_mode_mutex_); +} + bool FsData::redist_running() const { return redist_running_.load(); diff --git a/src/daemon/handler/srv_malleability.cpp b/src/daemon/handler/srv_malleability.cpp index 9b9e8aeba..65bff96ab 100644 --- a/src/daemon/handler/srv_malleability.cpp +++ b/src/daemon/handler/srv_malleability.cpp @@ -59,12 +59,22 @@ rpc_srv_mutate_start(const tl::request& req, const gkfs::rpc::rpc_mutate_start_in_t& in) { gkfs::rpc::rpc_err_out_t out; bool entered_maintenance = false; + bool duplicate_request = false; + bool registered_mutate = false; GKFS_DATA->spdlogger()->debug( "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", __func__, in.old_server_conf, in.new_server_conf, in.new_hosts_file); try { + duplicate_request = GKFS_DATA->begin_mutate_start( + in.old_server_conf, in.new_server_conf, in.new_hosts_file); + if(duplicate_request) { + out.err = 0; + gkfs::utils::safe_respond(req, out); + return; + } + registered_mutate = true; GKFS_DATA->maintenance_mode(true); entered_maintenance = true; GKFS_DATA->malleable_manager()->mutate_start( @@ -76,6 +86,9 @@ rpc_srv_mutate_start(const tl::request& req, if(entered_maintenance) { GKFS_DATA->maintenance_mode(false); } + if(registered_mutate) { + GKFS_DATA->end_mutate_start(); + } out.err = -1; } @@ -108,6 +121,7 @@ rpc_srv_mutate_finalize(const tl::request& req) { try { GKFS_DATA->maintenance_mode(false); GKFS_DATA->keep_hosts_file(true); + GKFS_DATA->end_mutate_start(); out.err = 0; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error("{}() Failed to finalize mutate: '{}'", diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index 84b352a6b..466812d2f 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -408,9 +408,9 @@ MalleableManager::connect_to_hosts( const auto& local_uri = RPC_DATA->self_addr_str(); bool local_host_found = false; - RPC_DATA->hosts_size(hosts.size()); - RPC_DATA->rpc_endpoints().clear(); - RPC_DATA->rpc_endpoints_str().clear(); + std::map new_rpc_endpoints; + std::map new_rpc_endpoints_str; + uint64_t new_local_host_id = std::numeric_limits::max(); vector host_ids(hosts.size()); // populate vector with [0, ..., host_size - 1] ::iota(::begin(host_ids), ::end(host_ids), 0); @@ -432,8 +432,8 @@ MalleableManager::connect_to_hosts( for(uint32_t i = 0; i < 4; i++) { try { auto svr_addr = RPC_DATA->client_rpc_engine()->lookup(uri); - RPC_DATA->rpc_endpoints().insert(make_pair(id, svr_addr)); - RPC_DATA->rpc_endpoints_str().insert(make_pair(id, uri)); + new_rpc_endpoints.insert(make_pair(id, svr_addr)); + new_rpc_endpoints_str.insert(make_pair(id, uri)); break; } catch(const std::exception& e) { // still not working after 5 tries. @@ -459,7 +459,7 @@ MalleableManager::connect_to_hosts( : (hostname == local_hostname))) { GKFS_DATA->spdlogger()->debug("{}() Found local host: {} (uri: {})", __func__, hostname, uri); - RPC_DATA->local_host_id(id); + new_local_host_id = id; local_host_found = true; } GKFS_DATA->spdlogger()->debug("{}() Found daemon: id '{}' uri '{}'", @@ -470,7 +470,7 @@ MalleableManager::connect_to_hosts( GKFS_DATA->spdlogger()->info( "{}() Local host '{}' (uri: '{}') not found in new hosts file. This node is being removed.", __func__, local_hostname, local_uri); - RPC_DATA->local_host_id(std::numeric_limits::max()); + new_local_host_id = std::numeric_limits::max(); } else { auto err_msg = fmt::format( "{}() Local host '{}' (uri: '{}') not found in hosts file. This should not happen.", @@ -478,6 +478,11 @@ MalleableManager::connect_to_hosts( throw runtime_error(err_msg); } } + + RPC_DATA->hosts_size(hosts.size()); + RPC_DATA->rpc_endpoints(std::move(new_rpc_endpoints)); + RPC_DATA->rpc_endpoints_str(std::move(new_rpc_endpoints_str)); + RPC_DATA->local_host_id(new_local_host_id); } int @@ -867,7 +872,7 @@ MalleableManager::mutate_start(int old_server_conf, int new_server_conf, } auto old_distributor = gkfs::rpc::create_from_config( - config, static_cast(RPC_DATA->local_host_id()), + config, static_cast(old_local_host_id), old_hosts_size_, 0); if(!old_distributor) { throw runtime_error("Failed to recreate old distributor for mutate"); diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py index df4336e9b..0c98628af 100644 --- a/tests/integration/malleability/test_malleability_performance.py +++ b/tests/integration/malleability/test_malleability_performance.py @@ -766,7 +766,11 @@ def test_shrink_performance_multi_run(client_fixture, ) if cmd.exit_code != 0: _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) - assert cmd.exit_code == 0, f"Shrink start failed: {cmd.stderr.decode()[:300]}" + assert cmd.exit_code == 0, ( + "Shrink start failed:\n" + f"stdout: {cmd.stdout.decode(errors='replace')[:1000]}\n" + f"stderr: {cmd.stderr.decode(errors='replace')[:1000]}" + ) # Wait for completion deadline = time.time() + 120 -- GitLab From 1527977f5b7b93af21a7f387aed00e48dac088e8 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Sat, 29 Aug 2026 11:40:46 +0200 Subject: [PATCH 20/21] fix: remove redundant zero-read checks in client read functions Remove the explicit check that converts a zero read count to an error in gkfs_preadv, and delete the corresponding assertion in gkfs_readv. Zero-length reads are valid and should be allowed to propagate instead of being treated as failures. --- src/client/gkfs_data.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/client/gkfs_data.cpp b/src/client/gkfs_data.cpp index 748c4df1f..88f85f0ce 100644 --- a/src/client/gkfs_data.cpp +++ b/src/client/gkfs_data.cpp @@ -703,9 +703,6 @@ gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { } } - if(read == 0) { - return -1; - } return read; } @@ -727,7 +724,6 @@ gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { } auto pos = gkfs_fd->pos(); // retrieve the current offset auto ret = gkfs_preadv(fd, iov, iovcnt, pos); - assert(ret != 0); if(ret < 0) { return -1; } -- GitLab From 57821d891600afe9abf890eed8db2e66bc21d4e3 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Sat, 29 Aug 2026 13:28:16 +0200 Subject: [PATCH 21/21] fix(malleability): use atomic rename to prevent hostfile race condition write_rs_interval_comments now writes to a temporary file and renames it to the target path instead of truncating the file in place. This prevents a race condition where concurrently running daemons could observe an empty or partially written hostfile. Added cleanup for the temporary file on failure and updated error messages accordingly. --- src/common/malleability_markers.cpp | 30 ++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/common/malleability_markers.cpp b/src/common/malleability_markers.cpp index 5c5c959f3..7cadbf49b 100644 --- a/src/common/malleability_markers.cpp +++ b/src/common/malleability_markers.cpp @@ -33,9 +33,12 @@ #include #include #include +#include #include #include +#include + namespace gkfs { namespace malleable { @@ -258,11 +261,17 @@ write_rs_interval_comments( } in.close(); - std::ofstream out(path, std::ios::trunc); + // Write to a sibling temporary file and rename it into place. Mutate + // start sends this shared hostfile to every daemon concurrently; opening + // the destination with std::ios::trunc leaves a window in which a peer can + // observe an empty or partially-written hostfile. + const auto temporary_path = + path + ".tmp." + std::to_string(static_cast(getpid())); + std::ofstream out(temporary_path, std::ios::trunc); if(!out.is_open()) { throw std::runtime_error(fmt::format( - "Failed to write hostfile for RS intervals: '{}': {}", path, - strerror(errno))); + "Failed to write temporary hostfile for RS intervals: '{}': {}", + temporary_path, strerror(errno))); } for(const auto& kept_line : lines) { out << kept_line << "\n"; @@ -272,6 +281,21 @@ write_rs_interval_comments( << " start=" << std::fixed << std::setprecision(9) << interval.start << " end=" << interval.end << "\n"; } + out.close(); + if(!out) { + std::remove(temporary_path.c_str()); + throw std::runtime_error(fmt::format( + "Failed to write temporary hostfile for RS intervals: '{}': {}", + temporary_path, strerror(errno))); + } + + if(std::rename(temporary_path.c_str(), path.c_str()) != 0) { + const auto error = errno; + std::remove(temporary_path.c_str()); + throw std::runtime_error(fmt::format( + "Failed to replace hostfile for RS intervals: '{}': {}", path, + strerror(error))); + } } } // namespace malleable -- GitLab