diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c4c30b864f8dd2149b998d88d47118659c0aa589..05e49f3010a91dfe08f24a738087c0008be94be8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,10 +181,6 @@ gkfs:integration-2: ${INTEGRATION_TESTS_BIN_PATH}/rename ${INTEGRATION_TESTS_BIN_PATH}/position ${INTEGRATION_TESTS_BIN_PATH}/status - ${INTEGRATION_TESTS_BIN_PATH}/malleability - ${INTEGRATION_TESTS_BIN_PATH}/forwarding - ${INTEGRATION_TESTS_BIN_PATH}/concurrency - ${INTEGRATION_TESTS_BIN_PATH}/resilience --basetemp=${BUILD_PATH}/tests/run/ --junit-xml=report-2.xml @@ -212,6 +208,56 @@ gkfs:integration-2: reports: junit: ${BUILD_PATH}/tests/integration/report-2.xml + +gkfs:integration-3: + stage: test + image: ${TESTING} + interruptible: true + needs: ['gkfs'] + + script: + ## run tests + - export PATH=${PATH}:/usr/local/bin + - export GKFS_MALLEABILITY_CI_FAST=ON + - export GKFS_PERF_REPETITIONS=1 + - export GKFS_PERF_NUM_FILES=1 + - export GKFS_PERF_FILE_SIZE=$((1024 * 1024)) + - export GKFS_MALLEABILITY_OLD_NODES=2 + - export GKFS_MALLEABILITY_NEW_NODES=1 + - mkdir -p ${BUILD_PATH}/tests/run + - cd ${BUILD_PATH}/tests/integration + - ${PYTEST} -v -n $(nproc) + ${INTEGRATION_TESTS_BIN_PATH}/malleability + ${INTEGRATION_TESTS_BIN_PATH}/forwarding + ${INTEGRATION_TESTS_BIN_PATH}/concurrency + ${INTEGRATION_TESTS_BIN_PATH}/resilience + --basetemp=${BUILD_PATH}/tests/run/ + --junit-xml=report-3.xml + + ## capture coverage information + - cd ${CI_PROJECT_DIR} + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration3.info + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + after_script: + - perl -i.orig + -pe 's%file="(.*?)"%file="tests/integration/$1"%g;' + -pe 's%(../)+install/share/gkfs/%%g;' + ${BUILD_PATH}/tests/integration/report-3.xml + + artifacts: + expire_in: 1 day + when: always + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/integration/report-3.xml + ## == integration tests for gkfs =========== gkfs:integration: stage: test @@ -566,7 +612,7 @@ coverage: stage: report image: ${TESTING} #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] + needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:integration-3', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ] script: # use ccache - ccache --zero-stats diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a60aa05ea2ca298e7835c17115d1107952f28ba..3f0b2b4cfbe27428cbe0abac02b9cbf6235f903b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,8 +47,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Client interface and hostfile removel ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313)) - Demon option to avoid removing hostfile on exit : ENV variable , GKFS_KEEP_HOSTS_FILE or option --keep-hosts - Client Interface selection avoiding loopback setup : LIBGKFS_OFI_INTERFACE=ib0 - - + - Random Slicing + cutshift ([!316](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/316)) + - Added marker-based expand, shrink, and mixed mutate workflow using one shared hostfile. + - Added Random Slicing placement with CutShift for expand-only resizes to reduce inter-node data movement. + - Added optional expand-on-demand data movement for pure expand operations. + - Added malleability integration coverage and user documentation. ### Changed @@ -60,6 +63,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - LIBGKFS/ GKFS _SYMLINK_SUPPORT, _RENAME_SUPPORT and _CREATE_CHECK_PARENTS. - Now all the performance options are in config.hpp and env variables. - Added DIRECT_IO on server side to increase performance ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312)) + - Shared metrics/message routing variables now use the common `GKFS_` prefix. + ### Fixed - SYS_lstat does not exists on some architectures, change to newfstatat ([!269](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/269)) @@ -71,6 +76,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Fix remove chunk bug ([!294](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/294)) - Fix decompress_and_parse_entries_standard() Unexpected end of buffer while parsing name bug ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312)) - Fix client dissapearing on malleability ends with an error. ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313)) + - Fixed buffer-size validation issues in filtered directory listing paths. ## [0.9.5] - 2025-08 diff --git a/README.md b/README.md index c7fed24cd6edae86fe793f6a4d0df060459645fc..83b7537dbdfd055feab70cb2f8e68fc9401b0f4c 100644 --- a/README.md +++ b/README.md @@ -33,8 +33,8 @@ to I/O, which reduces interferences and improves performance. - [Client-side metrics via MessagePack and ZeroMQ](#client-side-metrics-via-messagepack-and-zeromq) - [Server-side statistics via Prometheus](#server-side-statistics-via-prometheus) - [GekkoFS proxy](#gekkofs-proxy) - - [File system expansion](#file-system-expansion) - - [File system shrinking](#file-system-shrinking) + - [File system malleability: expand, shrink, and mutate](#file-system-malleability-expand-shrink-and-mutate) + - [Expand-on-demand malleability](#expand-on-demand-malleability) - [Miscellaneous](#miscellaneous) - [External functions](#external-functions) - [Data placement](#data-placement) @@ -193,7 +193,7 @@ gkfs_daemon --keep-hosts -r -m -H -m -H ``` @@ -209,9 +209,9 @@ modify `scripts/run/gkfs.conf` to mold default configurations to their environme The following options are available for `scripts/run/gkfs`: ```bash -usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args ] [--proxy ] [-f/--foreground ] - [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-v/--verbose ] - {start,expand,stop} +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} This script simplifies the starting and stopping GekkoFS daemons. If daemons are started on multiple nodes, @@ -219,7 +219,7 @@ usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args additional permanent configurations can be set. positional arguments: - COMMAND Command to execute: 'start', 'stop', 'expand' + COMMAND Command to execute: 'start', 'stop', 'mutate', 'status', 'finalize' optional arguments: -h, --help Shows this help message and exits @@ -235,7 +235,7 @@ usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args Nodelist is extracted from Slurm via the SLURM_JOB_ID env variable. --cpuspertask <#cores> Set the number of cores the daemons can use. Must use '--srun'. -c, --config Path to configuration file. By defaults looks for a 'gkfs.conf' in this directory. - -e, --expand_hostfile Path to the hostfile with new nodes where GekkoFS should be extended to (hostfile contains one line per node). + -H, --hostsfile Path to the GekkoFS hostfile. For mutate, this may already contain '+' and '-' markers. -v, --verbose Increase verbosity ``` @@ -494,129 +494,352 @@ Press 'q' to exit Please consult `include/config.hpp` for additional configuration options. Note, GekkoFS proxy does not support replication. -## File system expansion +## File system malleability: expand, shrink, and mutate -GekkoFS supports extending the current daemon configuration to additional compute nodes. This includes redistribution of -the existing data and metadata and therefore scales file system performance and capacity of existing data. Note, -that it is the user's responsibility to not access the GekkoFS file system during redistribution. A corresponding -feature that is transparent to the user is planned. Note also, if the GekkoFS proxy is used, they need to be manually -restarted, after expansion. +GekkoFS supports changing the daemon topology while preserving existing data and metadata. The same mechanism covers: -To enable this feature, the following CMake compilation flags are required to build the `gkfs_malleability` tool: -`-DGKFS_BUILD_TOOLS=ON`. The `gkfs_malleability` tool is then available in the `build/tools` directory. Please consult -`-h` for its arguments. While the tool can be used manually to expand the file system, the `scripts/run/gkfs` script -should be used instead which invokes the `gkfs_malleability` tool. +* **expand**: add daemon nodes +* **shrink**: remove daemon nodes +* **mutate**: add and remove nodes in one operation, for example a node swap -The only requirement for extending the file system is a hostfile containing the hostnames/IPs of the new nodes (one line -per host). Example starting the file system. The `DAEMON_NODELIST` in the `gkfs.conf` is set to a hostfile containing -the initial set of file system nodes.: +During redistribution, applications must not access the mounted GekkoFS file system. If the GekkoFS proxy is used, proxies must be restarted manually after the topology change. + +To build the control tool, configure with `-DGKFS_BUILD_TOOLS=ON`. This builds `gkfs_malleability`. The control tool reads the marker hostfile directly; it does not need separate expand or shrink hostfiles and it does not start daemons. The `scripts/run/gkfs` wrapper uses the same model: prepare one workspace hostfile with `+` and `-` markers, make sure added daemons are already running, then run `scripts/run/gkfs mutate` to call `gkfs_malleability mutate start/status/finalize` and let finalize clean the workspace hostfile. + +### Hostfile marker model + +Malleability uses one workspace hostfile. Entries in this file are the source of truth: + +| Line form | Meaning | +|---|---| +| `node uri ...` | active daemon | +| `+node uri ...` | daemon being added; new daemons write this automatically when started with `GKFS_DAEMON_EXPAND=ON` | +| `-node uri ...` | daemon being removed | +| `# GKFS_RS_INTERVAL host= start= end=` | Random Slicing interval-table comment; ignored by normal hostfile parsing but loaded by Random Slicing clients/daemons | + +Important: `+` and `-` daemons are still reachable during `mutate start`. The marker describes the **future** topology after finalize, not process liveness. + +### `scripts/run/gkfs` wrapper ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf start -* [gkfs] Starting GekkoFS daemons (4 nodes) ... -* [gkfs] GekkoFS daemons running -* [gkfs] Startup time: 10.853 seconds +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} ``` -... Some computation ... +The marker hostfile is enough. `scripts/run/gkfs mutate` does not take expand or shrink hostfiles. If the topology grows, start the added daemons first with `GKFS_DAEMON_EXPAND=ON`; they append `+` entries to the workspace hostfile. If the topology shrinks, mark the leaving daemon lines with `-` before running mutate. -Expanding the file system. Using `-e ` to specify the new nodes. Redistribution is done automatically with a -progress bar. When finished, the file system is ready to use in the new configuration: +Common workflows: ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf -e ~/hostfile_expand expand -* [gkfs] Starting GekkoFS daemons (8 nodes) ... -* [gkfs] GekkoFS daemons running -* [gkfs] Startup time: 1.058 seconds -Expansion process from 4 nodes to 12 nodes launched... -* [gkfs] Expansion progress: -[####################] 0/4 left -* [gkfs] Redistribution process done. Finalizing ... -* [gkfs] Expansion done. +# Start daemons from gkfs.conf +scripts/run/gkfs -c gkfs.conf start + +# Mutate after the workspace hostfile is marked and added daemons, if any, are running. +scripts/run/gkfs -c gkfs.conf mutate + +# Same, but use an explicit marked hostfile instead of LIBGKFS_HOSTS_FILE from gkfs.conf. +scripts/run/gkfs -c gkfs.conf -H /path/to/marked_hosts.txt mutate + +# Poll or finalize manually if needed. +scripts/run/gkfs -c gkfs.conf status +scripts/run/gkfs -c gkfs.conf finalize + +# Stop daemons. +scripts/run/gkfs -c gkfs.conf stop +``` + +For shrink-only operations, no new daemons are needed; just mark leaving entries with `-`. For expand-only operations, no `-` entries are needed; start the added daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+` entries. + +### User-level resize examples + +The examples below use plain GekkoFS commands: start daemons with `gkfs_daemon` and control topology changes with +`gkfs_malleability`. They assume GekkoFS was configured with tools enabled, for example `-DGKFS_BUILD_TOOLS=ON`, and that +`gkfs_daemon`, `gkfs_malleability`, and `libgkfs_intercept.so` come from the same installation. + +Common shell setup: + +```bash +# installation and workspace paths +export GKFS_INSTALL=/path/to/install +export GKFS_WORKDIR=/scratch/$USER/gkfs +export GKFS_HOSTFILE=$GKFS_WORKDIR/gkfs_hosts.txt +export GKFS_ROOTDIR=/local/ssd/$USER/gkfs-root +export GKFS_MOUNTDIR=$GKFS_WORKDIR/mount + +mkdir -p "$GKFS_WORKDIR" "$GKFS_MOUNTDIR" + +# client and tool input: one shared hostfile for the current GekkoFS instance +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTFILE + +# daemon-side hostfile path; the same path is also passed with -H below +export GKFS_HOSTS_FILE=$GKFS_HOSTFILE + +# keep the hosts file while daemons are stopped or resized; this is important for marker-based mutations +export GKFS_DAEMON_KEEP_HOSTS_FILE=ON + +# choose one placement strategy and export it everywhere: daemons, clients, proxies, and tools +export GKFS_DISTRIBUTION_STRATEGY=simple_hash +# CutShift substantially reduces inter-node data movement during expand-only resizes +# alternative: +# export GKFS_DISTRIBUTION_STRATEGY=random_slicing +# export GKFS_RANDOM_SLICING_CUTSHIFT=ON + +# recommended on multi-node systems to avoid clients choosing loopback +export LIBGKFS_OFI_INTERFACE=ib0 ``` -Stop the file system: +Client environment for applications: ```bash -~/gekkofs/scripts/run/gkfs -c ~/run/gkfs_verbs_expandtest.conf stop -* [gkfs] Stopping daemon with pid 16462 -srun: sending Ctrl-C to StepId=282378.1 -* [gkfs] Stopping daemon with pid 16761 -srun: sending Ctrl-C to StepId=282378.2 -* [gkfs] Shutdown time: 1.032 seconds +export LIBGKFS_HOSTS_FILE=$GKFS_HOSTFILE +export LD_PRELOAD=$GKFS_INSTALL/lib/libgkfs_intercept.so +export GKFS_DISTRIBUTION_STRATEGY=simple_hash # or random_slicing, matching daemon/tool env +export LIBGKFS_OFI_INTERFACE=ib0 ``` -## File system shrinking +Before every resize operation, stop or pause applications so that no process accesses the mounted GekkoFS namespace while +redistribution is running. After `mutate finalize`, restart applications or force them to reload the updated hostfile. If +the GekkoFS proxy is used, restart proxies manually after the topology change. -GekkoFS supports **shrinking** the current daemon configuration, removing one or more nodes from the cluster while -safely redistributing all existing data and metadata to the remaining nodes. As with expansion, it is the user's -responsibility not to access the file system during redistribution. +Use the following helper loop to wait for a topology change to finish when running `gkfs_malleability` manually: -The same `gkfs_malleability` tool (built with `-DGKFS_BUILD_TOOLS=ON`) is used. Shrinking requires two hostfiles: +```bash +wait_for_mutate() { + while ! "$GKFS_INSTALL/bin/gkfs_malleability" mutate status 2>&1 | grep -q "No mutate"; do + sleep 2 + done +} +``` -| File | Description | -|---|---| -| `gkfs_hosts.txt` | Current (old) hostfile — set via `LIBGKFS_HOSTS_FILE` | -| `gkfs_hosts_new.txt` | New hostfile listing **only** the surviving nodes | +#### Expand: add daemon nodes + +Start the initial instance, for example on two nodes: + +```bash +rm -f "$GKFS_HOSTFILE" +printf "node01\nnode02\n" > "$GKFS_WORKDIR/nodes-current.txt" + +srun --nodelist="$GKFS_WORKDIR/nodes-current.txt" \ + --ntasks=2 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & +``` + +To expand from two to four daemons, start only the new daemons with `GKFS_DAEMON_EXPAND=ON`. The added daemons append +`+` entries to the existing hostfile; do not delete the existing hostfile. + +```bash +# applications must be stopped or paused here +printf "node03\nnode04\n" > "$GKFS_WORKDIR/nodes-add.txt" + +GKFS_DAEMON_EXPAND=ON \ +srun --nodelist="$GKFS_WORKDIR/nodes-add.txt" \ + --ntasks=2 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & + +# run redistribution, poll status, and finalize the marked hostfile +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize +``` + +After a successful finalize, `gkfs_hosts.txt` contains four unmarked active entries. Keep an up-to-date list of all active +daemon nodes for later administration: + +```bash +printf "node01\nnode02\nnode03\nnode04\n" > "$GKFS_WORKDIR/nodes-current.txt" +``` + +Optional lazy data movement for pure expand: + +```bash +export GKFS_EXPAND_ON_DEMAND=ON +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize +``` -### Step-by-step +With `GKFS_EXPAND_ON_DEMAND=ON`, metadata is still redistributed during `mutate start`, but data chunks are fetched and +materialized lazily on first access. Use this only for pure expand operations; shrink and mixed mutate use eager data +migration. -**1. Create the new hostfile** containing only the nodes that should remain after shrink. -The format is identical to `gkfs_hosts.txt`. The order does not matter — any nodes present in the old file -but absent from the new file will be removed. +#### Shrink: remove daemon nodes + +For shrink-only operations, do not start new daemons. Mark each daemon that should leave by prefixing its hostfile line +with `-`, then run mutate. The leaving daemons must remain running and reachable until finalize completes; `mutate +finalize` stops the daemons marked with `-` automatically. + +Example shrink from four to two daemons: ```bash -# Example: remove node4, keep node1–node3 -grep -v node4 gkfs_hosts.txt > gkfs_hosts_new.txt +# applications must be stopped or paused here +cp "$GKFS_HOSTFILE" "$GKFS_HOSTFILE.bak" + +# mark node03 and node04 for removal. Match the real host names as they appear in the hostfile. +sed -i -E 's/^(node03[[:space:]])/-\1/' "$GKFS_HOSTFILE" +sed -i -E 's/^(node04[[:space:]])/-\1/' "$GKFS_HOSTFILE" + +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize ``` -**2. Start the shrink** process. Each surviving node redistributes the data that was owned by the removed nodes, -and the removed nodes forward all their data before stopping: +After finalize, the hostfile contains only the remaining active daemon entries and the daemons that left the topology have +been stopped automatically. Update your active-node list: ```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt \ - gkfs_malleability shrink --new-hosts-file gkfs_hosts_new.txt start -Shrink process from 4 nodes to 3 nodes launched... +printf "node01\nnode02\n" > "$GKFS_WORKDIR/nodes-current.txt" ``` -The old and new node counts are **auto-detected** from the respective hostfiles. They can be overridden with -`--old-nodes ` and `--new-nodes ` if needed. +#### Mutate: replace nodes in one operation + +A mixed mutate, or node swap, combines shrink and expand. Mark the leaving daemon lines with `-`, start replacement +daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+` entries, then run the same mutate workflow. -**3. Poll status** until all nodes have finished: +Example replacement of `node02` by `node05` while keeping the daemon count constant: ```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt gkfs_malleability shrink status -No shrink running/finished. +# applications must be stopped or paused here +cp "$GKFS_HOSTFILE" "$GKFS_HOSTFILE.bak" + +# mark the old daemon that should leave +sed -i -E 's/^(node02[[:space:]])/-\1/' "$GKFS_HOSTFILE" + +# start the replacement daemon; it appends a '+node05 ...' entry +printf "node05\n" > "$GKFS_WORKDIR/nodes-add.txt" +GKFS_DAEMON_EXPAND=ON \ +srun --nodelist="$GKFS_WORKDIR/nodes-add.txt" \ + --ntasks=1 --ntasks-per-node=1 \ + "$GKFS_INSTALL/bin/gkfs_daemon" \ + -r "$GKFS_ROOTDIR" \ + -m "$GKFS_MOUNTDIR" \ + -H "$GKFS_HOSTFILE" \ + -l ib0 -P ofi+verbs & + +# redistribute from the before topology to the after topology, stop '-' daemons, and clean markers +"$GKFS_INSTALL/bin/gkfs_malleability" mutate start +wait_for_mutate +"$GKFS_INSTALL/bin/gkfs_malleability" mutate finalize + +# update the active-node list for future administration +printf "node01\nnode03\nnode04\nnode05\n" > "$GKFS_WORKDIR/nodes-current.txt" ``` -When active: `Shrink in progress: 2 nodes not finished.` +Useful daemon and tool options for these examples: -**4. Finalize** the shrink. This disables maintenance mode on all remaining daemons and atomically replaces -`gkfs_hosts.txt` with `gkfs_hosts_new.txt`: +| Option or command | Meaning | +|---|---| +| `gkfs_daemon -r, --rootdir ` | Local data directory for each daemon | +| `gkfs_daemon -m, --mountdir ` | GekkoFS mount directory visible to clients | +| `gkfs_daemon -H, --hosts-file ` | Shared hostfile written by daemons and read by clients/tools | +| `gkfs_daemon -l ` | Network interface or address to bind, for example `ib0` | +| `gkfs_daemon -P ` | RPC protocol, for example `ofi+sockets` or `ofi+verbs` | +| `gkfs_malleability mutate start` | Start redistribution from the marked hostfile topology | +| `gkfs_malleability mutate status` | Poll redistribution status | +| `gkfs_malleability mutate finalize` | Stop daemons marked with `-`, clean the hostfile markers, and complete the topology change | + +### Random Slicing and CutShift + +Set the same distribution variables for daemons, clients, proxies, and tools: ```bash -LIBGKFS_HOSTS_FILE=gkfs_hosts.txt \ - gkfs_malleability shrink --new-hosts-file gkfs_hosts_new.txt finalize -Shrink finalize 0 -Hosts file updated: gkfs_hosts_new.txt -> gkfs_hosts.txt +export GKFS_DISTRIBUTION_STRATEGY=random_slicing +export GKFS_RANDOM_SLICING_CUTSHIFT=ON ``` -After finalize, `gkfs_hosts.txt` contains only the surviving nodes and all clients automatically use the -updated configuration on their next initialization. +These are normal GekkoFS environment variables, not `scripts/run/gkfs` options. The wrapper does not interpret them; it only inherits and forwards its process environment when starting daemons or running `gkfs_malleability`. + +For `random_slicing`, `mutate start` writes the chosen interval table into the workspace hostfile as `# GKFS_RS_INTERVAL ...` comments. `mutate finalize` preserves those comments while removing `-` entries and promoting `+` entries. Later clients and restarted daemons load the comments and reconstruct the same Random Slicing layout instead of rebuilding a fresh equal layout. + +CutShift is applied for expand-only Random Slicing operations (`+` entries present, no `-` entries). Other Random Slicing mutate cases currently fall back to the final equal RS layout. + +Using `GKFS_DISTRIBUTION_STRATEGY=random_slicing` together with `GKFS_RANDOM_SLICING_CUTSHIFT=ON` is recommended for +expand-heavy workflows because it avoids rebuilding an entirely new balanced placement from scratch. Instead, CutShift +adjusts the existing Random Slicing interval layout and assigns ranges to the added daemons, so most data remains on its +current owner and only a smaller subset has to move between nodes. + +Reference: Random Slicing is based on "Random slicing: Efficient and scalable data placement for large-scale storage systems" by Alberto Miranda, Sascha Effert, Yangwook Kang, Ethan L. Miller, Ivan Popov, Andre Brinkmann, Tom Friedetzky, and Toni Cortes, published in ACM Transactions on Storage 10(3), 2014. See the [Google Scholar entry](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=hK-ogNAAAAAJ&cstart=20&pagesize=80&sortby=pubdate&citation_for_view=hK-ogNAAAAAJ:RGFaLdJalmkC). + +### Manual `gkfs_malleability` workflow + +Use this when you manage daemon startup and hostfile markers yourself. + +Expand manually: -**5. Shut down the removed daemons** (they have already forwarded all data but are still running): +```bash +# workspace hostfile initially has only active entries +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +export GKFS_DAEMON_KEEP_HOSTS_FILE=ON + +# start added daemons; they append '+' lines to the workspace hostfile +export GKFS_DAEMON_EXPAND=ON +srun -w newnode1,newnode2 gkfs_daemon -r -m -H "$LIBGKFS_HOSTS_FILE" ... +unset GKFS_DAEMON_EXPAND + +# start redistribution and wait +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done + +# finalize: stop '-' daemons automatically, promote '+', preserve RS interval comments +gkfs_malleability mutate finalize +``` + +Shrink manually: ```bash -# Send SIGTERM to each daemon on the removed nodes -pdsh -w node4 'kill $(cat /tmp/gkfs_daemon.pid)' +# mark existing entries to remove +sed -i 's/^node4 /-node4 /' /path/to/gkfs_workspace.txt + +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done +# finalize stops the daemon marked with '-' automatically +gkfs_malleability mutate finalize ``` -### Environment variables +Swap/mutate is the combination: mark leaving daemons with `-`, start replacement daemons with `GKFS_DAEMON_EXPAND=ON` so they append `+`, then run `mutate start/status/finalize`. + +### Environment variables for malleability | Variable | Description | |---|---| -| `LIBGKFS_HOSTS_FILE` | Path to the **current** (old) hosts file | -| `LIBGKFS_HOSTS_FILE_NEW` | Alternative to `--new-hosts-file` for the new hosts file | +| `LIBGKFS_HOSTS_FILE` | Workspace hostfile for clients and `gkfs_malleability` | +| `GKFS_HOSTS_FILE` | Hostfile path consumed by daemons; normally passed with `-H` or via the wrapper | +| `GKFS_DAEMON_EXPAND` | Set to `ON` only for added daemons so they append `+` hostfile entries | +| `GKFS_DAEMON_KEEP_HOSTS_FILE` | Set to `ON` to prevent daemon shutdown from deleting the workspace hostfile | +| `GKFS_DISTRIBUTION_STRATEGY` | Common placement strategy. Supported values include `simple_hash` and `random_slicing` | +| `GKFS_RANDOM_SLICING_CUTSHIFT` | Set to `ON` to use CutShift for Random Slicing expand-only operations | +| `GKFS_EXPAND_ON_DEMAND` | Set to `ON` to enable v1 expand-only lazy data movement | + +## Expand-on-demand malleability + +`GKFS_EXPAND_ON_DEMAND=ON` enables the v1 lazy data path for pure expand operations. Metadata is still redistributed eagerly during `mutate start`, but data chunks are not moved eagerly. When a read reaches the new final owner and the local chunk is missing, the daemon fetches the chunk from the old owner through the normal read RPC, serves the client, and then best-effort materializes the chunk locally. Partial writes first materialize the old chunk synchronously so bytes outside the overwrite range are preserved. + +Example: + +```bash +export GKFS_EXPAND_ON_DEMAND=ON +export LIBGKFS_HOSTS_FILE=/path/to/gkfs_workspace.txt +gkfs_malleability mutate start +while ! gkfs_malleability mutate status 2>&1 | grep -q "No mutate"; do sleep 2; done +gkfs_malleability mutate finalize +``` + +Current v1 limits: + +* only pure expand is handled lazily; shrink or mixed mutate falls back to eager data migration; +* state is in memory only and is lost on daemon restart; +* old daemons must stay alive and reachable until cold chunks are materialized or no longer needed; +* existing daemon host order/IDs must remain stable across the hostfile rebuild; +* no custom materialization RPC, batching, throttling, or persistent epoch metadata exists yet. # Miscellaneous @@ -712,12 +935,30 @@ Once it is enabled, `--dbbackend` option will be functional. ## Environment variables The GekkoFS daemon, client, and proxy support a number of environment variables to augment its functionality: +Environment variable prefixes are scoped by component: + +| Prefix | Scope | +|---|---| +| `LIBGKFS_` | Client-side variables consumed by the preload/libc/user/FUSE client side | +| `GKFS_DAEMON_` | Daemon-only variables | +| `GKFS_PROXY_` | Proxy-only variables | +| `GKFS_` | Common variables shared by daemon, client, proxy, or tools | + +`GKFS_HOSTS_FILE` is retained as a legacy daemon-side hostfile variable. The client hostfile remains `LIBGKFS_HOSTS_FILE`. + +### Common +- `GKFS_DISTRIBUTION_STRATEGY` - Common data/metadata placement strategy. Set the same value on daemons, clients, proxies, and malleability tools. Supported values: `simple_hash` (default), `random_slicing`. +- `GKFS_RANDOM_SLICING_CUTSHIFT` - Enable CutShift for random-slicing add/expand operations (default: OFF). +- `GKFS_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. +- `GKFS_SYMLINK_SUPPORT` - Enable support for symbolic links. +- `GKFS_RENAME_SUPPORT` - Enable support for rename. +- `GKFS_USE_INLINE_DATA` - Enable inline data storage for small files (default: ON). +- `GKFS_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). +- `GKFS_HOSTS_FILE` - Legacy daemon-side hostfile path. + ### Client #### Core - `LIBGKFS_HOSTS_FILE` - Path to the hostsfile (created by the daemon and mandatory for the client). -- `LIBGKFS_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. -- `LIBGKFS_SYMLINK_SUPPORT` - Enable support for symbolic links. -- `LIBGKFS_RENAME_SUPPORT` - Enable support for rename. - `LIBGKFS_ENABLE_FORK` - Enable fork support in the client library, used for example in DLIO. - `LIBGKFS_OFI_INTERFACE` - Force the client-side libfabric interface to use (equivalent to `FI_SOCKETS_IFACE`). This prevents clients from binding to loopback (`127.0.0.1`) when daemons are on real NICs (e.g., `ib0`). @@ -744,16 +985,14 @@ Client-metrics require the CMake argument `-DGKFS_ENABLE_CLIENT_METRICS=ON` (see - `LIBGKFS_PROXY_PID_FILE` - Path to the proxy pid file (when using the GekkoFS proxy). - `LIBGKFS_NUM_REPL` - Number of replicas for data. #### Optimization -- `LIBGKFS_USE_INLINE_DATA` - Enable inline data storage for small files (default: ON). - `LIBGKFS_CREATE_WRITE_OPTIMIZATION` - Optimization for write operations (default: OFF). - `LIBGKFS_READ_INLINE_PREFETCH` - Prefetch inline data when opening files (default: OFF). -- `LIBGKFS_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). - `LIBGKFS_DIRENTS_BUFF_SIZE` - Buffer size for directory entries (default: 8MB). - `LIBGKFS_ASYNC_WRITE` - Enable client-side asynchronous write cache (default: OFF). -- `GKFS_FUSE_ENTRY_TIMEOUT` - Caching timeout for dentry entries in the FUSE client (default: 1.0). -- `GKFS_FUSE_ATTR_TIMEOUT` - Caching timeout for file attributes in the FUSE client (default: 1.0). -- `GKFS_FUSE_NEGATIVE_TIMEOUT` - Caching timeout for negative lookups in the FUSE client (default: 1.0). -- `GKFS_FUSE_WRITEBACK` - Enable writeback cache in the FUSE client (default: OFF). +- `LIBGKFS_FUSE_ENTRY_TIMEOUT` - Caching timeout for dentry entries in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_ATTR_TIMEOUT` - Caching timeout for file attributes in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_NEGATIVE_TIMEOUT` - Caching timeout for negative lookups in the FUSE client (default: 1.0). +- `LIBGKFS_FUSE_WRITEBACK` - Enable writeback cache in the FUSE client (default: OFF). #### Caching ##### Dentry cache @@ -800,23 +1039,17 @@ During write/pwrite operations, when the asynchronous write cache is enabled, th ### Daemon #### Core -- `GKFS_DAEMON_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. - `GKFS_DAEMON_CREATE_EXIST_CHECK` - Check for existence of file metadata before create in RocksDB. -- `GKFS_DAEMON_SYMLINK_SUPPORT` - Enable support for symbolic links. -- `GKFS_DAEMON_RENAME_SUPPORT` - Enable support for rename. -- `GKFS_KEEP_HOSTS_FILE` - Preserve the hosts file on daemon shutdown instead of destroying it (default: OFF, use with `--keep-hosts` CLI flag). +- `GKFS_DAEMON_KEEP_HOSTS_FILE` - Preserve the hosts file on daemon shutdown instead of destroying it (default: OFF, use with `--keep-hosts` CLI flag). +- `GKFS_DAEMON_EXPAND` - Set to `ON` on new daemons so they auto-write `+` marker entries to the workspace hostfile during marker-based expansion/mutate. +- `GKFS_DAEMON_ENABLE_WAL` - Enable RocksDB write-ahead logging for daemon metadata persistence tests/recovery scenarios (default follows build/runtime config). #### Logging - `GKFS_DAEMON_LOG_PATH` - Path to the log file of the daemon. - `GKFS_DAEMON_LOG_LEVEL` - Log level of the daemon. Available levels are: `off`, `critical`, `err`, `warn`, `info`, `debug`, `trace`. -#### Optimization -- `GKFS_DAEMON_USE_INLINE_DATA` - Enable inline data storage (default: ON). -- `GKFS_DAEMON_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). ### Proxy #### Logging - `GKFS_PROXY_LOG_PATH` - Path to the log file of the proxy. - `GKFS_PROXY_LOG_LEVEL` - Log level of the proxy. Available levels are: `off`, `critical`, `err`, `warn`, `info`, `debug`, `trace`. -#### Optimization -- `GKFS_PROXY_USE_DIRENTS_COMPRESSION` - Enable compression for directory entries (default: OFF). # Acknowledgment diff --git a/examples/gfind/sfind.cpp b/examples/gfind/sfind.cpp index eb15a35947a4c16dd2e6ce6ef4ae98f135e9a27c..d0769fca4e22b781ac7011ec55abf7f2affae9db 100644 --- a/examples/gfind/sfind.cpp +++ b/examples/gfind/sfind.cpp @@ -302,11 +302,15 @@ worker_routine(void* arg) { if(n > 0 && entries && !data->opt->just_count) { char* ptr = reinterpret_cast(entries); - int bytes_processed = 0; + size_t bytes_processed = 0; while(bytes_processed < n) { struct dirent_extended* temp = reinterpret_cast(ptr); - if(temp->d_reclen == 0) + const auto remaining = + static_cast(n) - bytes_processed; + if(temp->d_reclen < + offsetof(struct dirent_extended, d_name) + 1 || + temp->d_reclen > remaining) break; local_found++; @@ -360,11 +364,15 @@ worker_routine(void* arg) { } char* ptr = reinterpret_cast(entries); - int bytes_processed = 0; + size_t bytes_processed = 0; while(bytes_processed < n) { struct dirent_extended* temp = reinterpret_cast(ptr); - if(temp->d_reclen == 0) + const auto remaining = + static_cast(n) - bytes_processed; + if(temp->d_reclen < + offsetof(struct dirent_extended, d_name) + 1 || + temp->d_reclen > remaining) break; if(temp->d_type != 1) { diff --git a/include/client/env.hpp b/include/client/env.hpp index b5425f2c62644ff8c4de312e6227d4d0110a7db0..a8711678510b9a9883ab372c068d77d678c368ca 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -83,8 +83,6 @@ static constexpr auto PROTECT_FILES_CONSUMER = ADD_PREFIX("PROTECT_FILES_CONSUMER"); static constexpr auto RANGE_FD = ADD_PREFIX("RANGE_FD"); static constexpr auto DIRENTS_BUFF_SIZE = ADD_PREFIX("DIRENTS_BUFF_SIZE"); -static constexpr auto USE_DIRENTS_COMPRESSION = - ADD_PREFIX("USE_DIRENTS_COMPRESSION"); static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL"); static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE"); @@ -95,10 +93,6 @@ static constexpr auto WRITE_SIZE_THRESHOLD = ADD_PREFIX("WRITE_SIZE_CACHE_THRESHOLD"); } // namespace cache -static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS"); -static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT"); -static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT"); -static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA"); static constexpr auto CREATE_WRITE_OPTIMIZATION = ADD_PREFIX("CREATE_WRITE_OPTIMIZATION"); static constexpr auto READ_INLINE_PREFETCH = ADD_PREFIX("READ_INLINE_PREFETCH"); @@ -107,6 +101,11 @@ static constexpr auto METADATA_BATCH = ADD_PREFIX("METADATA_BATCH"); static constexpr auto METADATA_BATCH_THRESHOLD = ADD_PREFIX("METADATA_BATCH_THRESHOLD"); static constexpr auto ASYNC_WRITE = ADD_PREFIX("ASYNC_WRITE"); +static constexpr auto FUSE_ENTRY_TIMEOUT = ADD_PREFIX("FUSE_ENTRY_TIMEOUT"); +static constexpr auto FUSE_ATTR_TIMEOUT = ADD_PREFIX("FUSE_ATTR_TIMEOUT"); +static constexpr auto FUSE_NEGATIVE_TIMEOUT = + ADD_PREFIX("FUSE_NEGATIVE_TIMEOUT"); +static constexpr auto FUSE_WRITEBACK = ADD_PREFIX("FUSE_WRITEBACK"); // Libfabric interface pinning (consumed by libfabric at HG_init() time) // OFI_INTERFACE is used with the GKFS_ prefix (e.g., LIBGKFS_OFI_INTERFACE) diff --git a/include/client/rpc/forward_malleability.hpp b/include/client/rpc/forward_malleability.hpp index d15226469f99b1af2883315837889e0628d51e98..aa7ae94f619050241df8fc6dbd6ec88da07c4fa9 100644 --- a/include/client/rpc/forward_malleability.hpp +++ b/include/client/rpc/forward_malleability.hpp @@ -45,23 +45,21 @@ namespace gkfs::malleable::rpc { int -forward_expand_start(int old_server_conf, int new_server_conf); +forward_mutate_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file); int -forward_expand_status(); +forward_mutate_status(); int -forward_expand_finalize(); +forward_mutate_finalize(); int -forward_shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file); +forward_mutate_reload(const std::string& hosts_file); int -forward_shrink_status(); +forward_mutate_shutdown_removed(const std::string& hostfile); -int -forward_shrink_finalize(); } // namespace gkfs::malleable::rpc #endif // GEKKOFS_CLIENT_FORWARD_MALLEABILITY_HPP diff --git a/include/client/rpc/utils.hpp b/include/client/rpc/utils.hpp index 80bd300a78923bde89cf5131018227c7dcb7f45b..d2f31072a0b038ebabf3938dd66c306cce5dc4f4 100644 --- a/include/client/rpc/utils.hpp +++ b/include/client/rpc/utils.hpp @@ -37,6 +37,7 @@ #include #include #include +#include namespace gkfs::rpc { @@ -54,8 +55,9 @@ namespace gkfs::rpc { */ template std::vector> -decompress_and_parse_entries(const OutputOrErr& out, - const void* compressed_buffer) { +decompress_and_parse_entries( + const OutputOrErr& out, const void* compressed_buffer, + std::size_t buffer_size = std::numeric_limits::max()) { if(out.err != 0) { throw std::runtime_error("Server returned an error: " + std::to_string(out.err)); @@ -63,6 +65,12 @@ decompress_and_parse_entries(const OutputOrErr& out, if(out.dirents_size == 0) { return {}; } + if(out.dirents_size > buffer_size) { + throw std::runtime_error( + "Server returned dirents payload larger than exposed client buffer: " + + std::to_string(out.dirents_size) + " > " + + std::to_string(buffer_size)); + } const char* p = nullptr; const char* end = nullptr; diff --git a/include/client/user_functions.hpp b/include/client/user_functions.hpp index 3b15537ba41b797f2189d6222ed73328bb10f433..7fc7d3fbccf3ba804f4316cbced388f79c99830a 100644 --- a/include/client/user_functions.hpp +++ b/include/client/user_functions.hpp @@ -157,54 +157,59 @@ gkfs_msync(void* addr, size_t length, int flags); namespace malleable { /** - * @brief Start an expansion of the file system - * @param old_server_conf old number of nodes - * @param new_server_conf new number of nodes + * @brief Start a mutation of the file system cluster topology. + * + * MARKER-BASED WORKFLOW: + * This function uses a marker-based single-hostfile approach. The workspace + * hostfile (LIBGKFS_HOSTS_FILE) contains markers that define the before/after + * state: + * - Normal lines (no prefix) = active daemons + * - Lines with '-' prefix = to-be-removed daemons (user writes + * manually) + * - Lines with '+' prefix = to-be-added daemons (auto-written by + * daemons with GKFS_DAEMON_EXPAND=ON) + * + * The CLI auto-discovers before/after state from markers in + * LIBGKFS_HOSTS_FILE. No separate --new-hosts-file is needed. + * + * SHRINK: User writes '-' on N daemons in LIBGKFS_HOSTS_FILE, then calls mutate + * start. The daemon drops the -marked nodes after redistribution. + * + * EXPAND: New daemons start with GKFS_DAEMON_EXPAND=ON (auto-write '+' + * prefix), then user calls mutate start. The daemon promotes + nodes after + * redistribution. + * + * MUTATE: Combination of shrink + expand in one operation. + * + * @param old_server_conf old number of nodes (active + removing, auto-detected + * if -1) + * @param new_server_conf new number of nodes (active + adding, auto-detected if + * -1) + * @param new_hosts_file path to workspace hostfile with markers + * (LIBGKFS_HOSTS_FILE) * @return error code */ int -expand_start(int old_server_conf, int new_server_conf); - -/** - * @brief Check for the current status of the expansion process - * @return 0 when finished, positive numbers indicate how many daemons - * are still redistributing data - */ -int -expand_status(); - -/** - * @brief Finalize the expansion process - * @return error code - */ -int -expand_finalize(); - -/** - * @brief Start a shrinking of the file system - * @param old_server_conf old number of nodes - * @param new_server_conf new number of nodes - * @param new_hosts_file path to hostfile containing only the surviving nodes - * @return error code - */ -int -shrink_start(int old_server_conf, int new_server_conf, +mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file); /** - * @brief Check for the current status of the shrinking process + * @brief Check for the current status of the mutate process * @return 0 when finished, positive numbers indicate how many daemons * are still redistributing data */ int -shrink_status(); +mutate_status(); /** - * @brief Finalize the shrinking process + * @brief Finalize the mutate process. + * Rewrites the workspace hostfile clean (promotes + to active, removes + * - lines). * @return error code */ int -shrink_finalize(); +mutate_finalize(); + } // namespace malleable } // namespace gkfs diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index 897842a3c73688fe240c4c9f44ba16796c7e7225..0580bab95161d0b9cbc2888ae4c642baaa488f7e 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -9,7 +9,7 @@ ADA-FS project under the SPPEXA project funded by the DFG. This software was partially supported by the - the European Union’s Horizon 2020 JTI-EuroHPC research and + the European Union's Horizon 2020 JTI-EuroHPC research and innovation programme, by the project ADMIRE (Project ID: 956748, admire-eurohpc.eu) @@ -124,16 +124,17 @@ constexpr auto all_remote_protocols = {ofi_sockets, ofi_tcp, ofi_verbs, ucx_rc, ucx_ud}; #pragma GCC diagnostic pop } // namespace protocol + } // namespace rpc namespace malleable::rpc::tag { -constexpr auto expand_start = "rpc_srv_expand_start"; -constexpr auto expand_status = "rpc_srv_expand_status"; -constexpr auto expand_finalize = "rpc_srv_expand_finalize"; -constexpr auto shrink_start = "rpc_srv_shrink_start"; -constexpr auto shrink_status = "rpc_srv_shrink_status"; -constexpr auto shrink_finalize = "rpc_srv_shrink_finalize"; -// migrate data uses the write rpc +// Primary mutate RPC tags +constexpr auto mutate_start = "rpc_srv_mutate_start"; +constexpr auto mutate_status = "rpc_srv_mutate_status"; +constexpr auto mutate_finalize = "rpc_srv_mutate_finalize"; +constexpr auto mutate_reload = "rpc_srv_mutate_reload"; +constexpr auto mutate_shutdown = "rpc_srv_mutate_shutdown"; +// Migrate metadata (used by forward_metadata for RocksDB redistribution) constexpr auto migrate_metadata = "rpc_srv_migrate_metadata"; } // namespace malleable::rpc::tag diff --git a/include/common/env.hpp b/include/common/env.hpp index c5ce6ad6f682852f64cf165034f4ecf805feecf0..cf0803eab8cb43dfc1120dab320752b0bb2b6fa2 100644 --- a/include/common/env.hpp +++ b/include/common/env.hpp @@ -43,7 +43,8 @@ #define ADD_PREFIX(str) COMMON_ENV_PREFIX str -/* Environment variables shared by GekkoFS clients and daemons */ +/* Environment variables shared by GekkoFS daemons, clients, proxies, and tools. + */ namespace gkfs::env { #ifdef GKFS_ENABLE_CLIENT_METRICS @@ -52,6 +53,18 @@ static constexpr auto METRICS_IP_PORT = ADD_PREFIX("METRICS_IP_PORT"); static constexpr auto METRICS_AGGREGATOR = ADD_PREFIX("METRICS_AGGREGATOR"); #endif +static constexpr auto DISTRIBUTION_STRATEGY = + ADD_PREFIX("DISTRIBUTION_STRATEGY"); +static constexpr auto RANDOM_SLICING_CUTSHIFT = + ADD_PREFIX("RANDOM_SLICING_CUTSHIFT"); +static constexpr auto EXPAND_ON_DEMAND = ADD_PREFIX("EXPAND_ON_DEMAND"); +static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS"); +static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT"); +static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT"); +static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA"); +static constexpr auto USE_DIRENTS_COMPRESSION = + ADD_PREFIX("USE_DIRENTS_COMPRESSION"); + } // namespace gkfs::env #undef ADD_PREFIX diff --git a/include/common/malleability_markers.hpp b/include/common/malleability_markers.hpp new file mode 100644 index 0000000000000000000000000000000000000000..a7452cda8ed039626d84a386806d5eab0f1da4cf --- /dev/null +++ b/include/common/malleability_markers.hpp @@ -0,0 +1,147 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#ifndef GEKKOFS_MALLEABILITY_MARKERS_HPP +#define GEKKOFS_MALLEABILITY_MARKERS_HPP + +#include +#include +#include + +namespace gkfs { +namespace malleable { + +// Marker characters for hostfile lines +constexpr char MARKER_ADD = '+'; +constexpr char MARKER_REMOVE = '-'; + +// Node states in marker-based hostfile +enum class NodeState { + ACTIVE, // Normal line, no marker - currently serving + REMOVING, // Starts with '-' - will be stopped after redistribute + ADDING // Starts with '+' - not yet serving until promoted +}; + +// Information about a single host entry in a marker-based hostfile +struct HostEntry { + std::string marker; // "+", "-", or "" (empty for active) + std::string hostname; // First field (e.g., "lo" or "hostname#suffix") + std::string uri; // Second field (e.g., "fi+sockets://...") + std::vector extra_fields; // Remaining 10 fields + std::string raw_line; // Original line without marker prefix +}; + +// Parsed result from a marker-based hostfile +struct HostfileMarkers { + std::vector active; // Normal lines (no marker) + std::vector removing; // Lines with '-' prefix + std::vector adding; // Lines with '+' prefix + + // Returns true if any markers were found + bool + has_markers() const { + return !removing.empty() || !adding.empty(); + } + + // Total number of entries (all states) + size_t + total_count() const { + return active.size() + removing.size() + adding.size(); + } +}; + +struct RandomSlicingIntervalComment { + uint64_t host_id{0}; + double start{0.0}; + double end{0.0}; +}; + +/** + * @brief Check if a line starts with a malleability marker + * @param line The line to check + * @return true if line starts with '+' or '-', false otherwise + */ +bool +is_marker_line(const std::string& line); + +/** + * @brief Extract the marker character from a line + * @param line The line to parse + * @return The marker character ('+', '-', or '\0' if no marker) + */ +char +get_marker(const std::string& line); + +/** + * @brief Parse a marker-based hostfile and categorize entries by state + * @param path Path to the hostfile + * @return HostfileMarkers struct with active/removing/adding entries + * @throws std::runtime_error if file cannot be opened or parsed + * + * Format expected: + * [MARKER]hostname uri field1 ... field12 + * Where MARKER is one of: (none), '+', '-' + * Lines starting with '#' are treated as comments and skipped. + */ +HostfileMarkers +parse_hostfile_markers(const std::string& path); + +/** + * @brief Strip marker prefix from a hostfile line + * @param line The line (possibly with marker prefix) + * @return The line without the marker prefix + */ +std::string +strip_marker(const std::string& line); + +/** + * @brief Write a clean hostfile (promote adding to active, remove removing) + * @param path Output path for the clean hostfile + * @param markers The parsed markers struct + * @throws std::runtime_error if file cannot be written + */ +void +write_clean_hostfile(const std::string& path, const HostfileMarkers& markers); + +std::vector +parse_rs_interval_comments(const std::string& path); + +void +write_clean_hostfile( + const std::string& path, const HostfileMarkers& markers, + const std::vector& rs_intervals); + +void +write_rs_interval_comments( + const std::string& path, + const std::vector& rs_intervals); + +} // namespace malleable +} // namespace gkfs + +#endif // GEKKOFS_MALLEABILITY_MARKERS_HPP \ No newline at end of file diff --git a/include/common/rpc/cutshift_sorted.hpp b/include/common/rpc/cutshift_sorted.hpp new file mode 100644 index 0000000000000000000000000000000000000000..0207aece009d0a955543f00c49c6eec29f9049f0 --- /dev/null +++ b/include/common/rpc/cutshift_sorted.hpp @@ -0,0 +1,79 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_CUTSHIFT_SORTED_H +#define GKFS_RPC_CUTSHIFT_SORTED_H + +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Collect gaps from shrinking nodes using CutShift+Sorted algorithm +/// @param old_partitions Current partitions before expansion +/// @param reductions Map of host_id -> capacity reduction amount +/// @return Sorted list of gaps (intervals with host_id=0 indicating empty +/// space) +std::vector +collect_gaps_cutshift(const std::vector& old_partitions, + const std::unordered_map& reductions); + +/// Assign gaps to new nodes using greedy largest-first packing +/// @param gaps List of available gaps +/// @param new_hosts List of new host IDs needing intervals +/// @param old_partitions Reference to old partitions (for capacity +/// calculations) +/// @return Updated partitions for new nodes +std::vector +assign_gaps_to_new_nodes(std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions); + +/// Expand the cluster by adding new nodes with minimal disruption +/// @param current_partitions Current partition table +/// @param new_hosts New host IDs to add +/// @param old_total_capacity Total capacity before expansion +/// @param new_total_capacity Total capacity after expansion +/// @return Updated partition table (old + new nodes) +std::vector +expand_with_cutshift(std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, float new_total_capacity); + +/// Shrink the cluster with minimum data movement. +/// +/// Existing survivor intervals stay owned by the same physical survivor. Output +/// host ids are compacted to the post-shrink rank order, i.e. sorted survivor +/// old ids map to 0..survivor_count-1. Only intervals owned by removed hosts +/// are redistributed to survivors to restore equal capacity. +/// @param current_partitions Current pre-shrink partition table using old host +/// ids +/// @param removed_hosts Old host ids that disappear after mutate finishes +/// @return Post-shrink compact partition table for surviving nodes +std::vector +shrink_with_cutshift(std::vector current_partitions, + const std::vector& removed_hosts); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_CUTSHIFT_SORTED_H \ No newline at end of file diff --git a/include/common/rpc/data_migration_executor.hpp b/include/common/rpc/data_migration_executor.hpp new file mode 100644 index 0000000000000000000000000000000000000000..da05bb1c7f0a52e061ef20cd836fc9ccf7000e44 --- /dev/null +++ b/include/common/rpc/data_migration_executor.hpp @@ -0,0 +1,90 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DATA_MIGRATION_EXECUTOR_H +#define GKFS_RPC_DATA_MIGRATION_EXECUTOR_H + +#include "common/rpc/data_migrator.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Callback type for migration progress reporting +using MigrationProgressCallback = + std::function; + +/// Migration result status +enum class MigrationStatus { Success, PartialFailure, AllFailed, Cancelled }; + +/// Execute a migration plan +class DataMigrationExecutor { +public: + /// Per-job migration function type: returns 0 on success, -1 on failure + using JobHandler = std::function; + + /// Execute all migration jobs with optional progress callback + MigrationStatus + execute(std::vector& jobs, + MigrationProgressCallback progress = nullptr, + JobHandler handler = nullptr); + + /// Execute migration jobs in batches of given size + MigrationStatus + execute_batched(std::vector& jobs, size_t batch_size, + MigrationProgressCallback progress = nullptr, + JobHandler handler = nullptr); + + /// Get total bytes migrated (for accounting) + size_t + total_bytes_migrated() const { + return total_bytes_.load(); + } + + /// Reset counters + void + reset() { + total_bytes_.store(0); + success_count_.store(0); + fail_count_.store(0); + } + + /// Get migration statistics + struct Stats { + size_t total_bytes; + size_t success_count; + size_t fail_count; + }; + Stats + get_stats() const { + return {total_bytes_.load(), success_count_.load(), fail_count_.load()}; + } + +private: + std::atomic total_bytes_{0}; + std::atomic success_count_{0}; + std::atomic fail_count_{0}; +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DATA_MIGRATION_EXECUTOR_H \ No newline at end of file diff --git a/include/common/rpc/data_migrator.hpp b/include/common/rpc/data_migrator.hpp new file mode 100644 index 0000000000000000000000000000000000000000..b6c9f956e19e3a3836c6ca220443708aa2a18dae --- /dev/null +++ b/include/common/rpc/data_migrator.hpp @@ -0,0 +1,90 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DATA_MIGRATOR_H +#define GKFS_RPC_DATA_MIGRATOR_H + +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include +#include + +namespace gkfs { +namespace rpc { + +/// A single migration job: move chunks from old mapping to new mapping +struct MigrationJob { + std::string path; + chunkid_t chunk_id; + host_t source_node; + host_t target_node; +}; + +/// Helper to find which host owns a position in a partition table +host_t +find_host_for(const std::vector& partitions, const std::string& path, + chunkid_t chnk_id); + +/// Compare old vs new partitions and compute migration jobs +class DataMigrator { +public: + /// Compute which chunks need to move between old and new partitioning + /// schemes. + /// @param old_partitions The previous partition layout + /// @param new_partitions The new partition layout + /// @param chunk_sample_size Number of chunk IDs to sample per path for + /// migration estimation + /// @return List of migration jobs needed + std::vector + compute_migrations(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); + + /// Get statistics about a migration plan + struct MigrationStats { + std::vector jobs; + std::unordered_map + from_counts; // chunks leaving each host + std::unordered_map + to_counts; // chunks entering each host + size_t total_chunks; + size_t migrating_chunks; + double migration_ratio; // migrating_chunks / total_chunks + }; + + MigrationStats + get_migration_stats(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size = 256); + + /// Print migration summary to stdout + void + print_migration_summary(const MigrationStats& stats); + + /// Check if migration is needed + static bool + needs_migration(const std::vector& old_partitions, + const std::vector& new_partitions); +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DATA_MIGRATOR_H \ No newline at end of file diff --git a/include/common/rpc/distribution_config.hpp b/include/common/rpc/distribution_config.hpp new file mode 100644 index 0000000000000000000000000000000000000000..14c30928863c22283718e699f6ea2a45fe8aff27 --- /dev/null +++ b/include/common/rpc/distribution_config.hpp @@ -0,0 +1,86 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DISTRIBUTION_CONFIG_H +#define GKFS_RPC_DISTRIBUTION_CONFIG_H + +#include "common/rpc/distributor_factory.hpp" +#include + +namespace gkfs { +namespace rpc { + +/// Distribution configuration - reads strategy from env/config +class DistributionConfig { +public: + /// Get the current distribution strategy + DistributionStrategy + get_strategy() const { + return strategy_; + } + + /// Set strategy from string + void + set_strategy(const std::string& strategy) { + strategy_ = string_to_strategy(strategy); + } + + /// Get strategy string for display + std::string + get_strategy_string() const { + return strategy_to_string(strategy_); + } + + /// Check if random slicing is active + bool + is_random_slicing() const { + return strategy_ == DistributionStrategy::RandomSlicing; + } + + /// Check if simple hash is active + bool + is_simple_hash() const { + return strategy_ == DistributionStrategy::SimpleHash; + } + + /// Get the default strategy + static DistributionStrategy + default_strategy() { + return DistributionStrategy::SimpleHash; + } + +private: + DistributionStrategy strategy_ = DistributionStrategy::SimpleHash; +}; + +/// Create a distributor from a DistributionConfig +std::unique_ptr +create_from_config(const DistributionConfig& config, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); + +/// Read strategy from environment variable GKFS_DISTRIBUTION_STRATEGY +/// Falls back to DistributionConfig::default_strategy() if not set +DistributionStrategy +read_strategy_from_env(); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DISTRIBUTION_CONFIG_H \ No newline at end of file diff --git a/include/common/rpc/distributor_factory.hpp b/include/common/rpc/distributor_factory.hpp new file mode 100644 index 0000000000000000000000000000000000000000..9fa2c09f5eb9bd2943fd56fdf39fad9639956547 --- /dev/null +++ b/include/common/rpc/distributor_factory.hpp @@ -0,0 +1,66 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_DISTRIBUTOR_FACTORY_H +#define GKFS_RPC_DISTRIBUTOR_FACTORY_H + +#include "common/rpc/distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +/// Supported distribution strategies +enum class DistributionStrategy { + SimpleHash, // current default: modulo-based placement + RandomSlicing, // new: interval-based placement + LocalOnly, // all data on local node + Forwarder // data forwarded to a specific host +}; + +/// Convert strategy enum to string +const char* +strategy_to_string(DistributionStrategy strategy); + +/// Convert string to strategy enum +DistributionStrategy +string_to_strategy(const std::string& str); + +/// Create a distributor based on the given strategy and parameters. +/// @param strategy The distribution strategy to use +/// @param localhost The local node's host ID +/// @param hosts_size The number of hosts in the cluster +/// @param fwd_host Optional forwarder host ID (used when strategy is Forwarder) +/// @return A uniquely-owned distributor, or nullptr on error +std::unique_ptr +create_distributor(DistributionStrategy strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); + +/// Convenience factory: creates a distributor from a string strategy name. +/// Strings: "simple_hash", "random_slicing", "local_only", "forwarder" +std::unique_ptr +create_distributor_from_string(const std::string& strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host = 0); + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_DISTRIBUTOR_FACTORY_H \ No newline at end of file diff --git a/include/common/rpc/random_slicing_distributor.hpp b/include/common/rpc/random_slicing_distributor.hpp new file mode 100644 index 0000000000000000000000000000000000000000..4bb3e93eebb22656284e3143f91e16dbed90401b --- /dev/null +++ b/include/common/rpc/random_slicing_distributor.hpp @@ -0,0 +1,167 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#ifndef GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H +#define GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H + +#include "common/rpc/distributor.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace gkfs { +namespace rpc { + +/// A contiguous numeric interval within [0, 1). +struct Interval { + float start; + float end; + host_t host_id{0}; +}; + +/// A partition on a single node: a node may own multiple disjoint intervals. +struct Partition { + host_t host_id{0}; + float total_capacity{0.0f}; // relative capacity c_i + std::vector intervals; + + float + coverage() const { + float sum = 0.0f; + for(const auto& iv : intervals) sum += (iv.end - iv.start); + return sum; + } +}; + +/// Flat sorted interval list with binary-search lookup. +/// ponycastle: simpler than a BST — a sorted vector + upper_bound is sufficient +/// and has lower constant factor for our use case (~200K intervals max). +class IntervalIndex { +public: + IntervalIndex() = default; + + /// Build from a list of partitions (flattens and sorts). + void + build(const std::vector& partitions); + + /// Find which partition owns position x in [0, 1). Returns -1 if none. + int + find_partition(float x) const; + + /// Find the host_id for position x. + host_t + find_host(float x) const; + + const std::vector& + intervals() const { + return intervals_; + } + +private: + std::vector intervals_; // sorted by start +}; + +class RandomSlicingDistributor : public Distributor { +private: + host_t localhost_; + unsigned int hosts_size_{0}; + std::vector all_hosts_; + std::vector partitions_; + IntervalIndex interval_idx_; + + // PRNG: minstd_rand per thesis recommendation #9 (fast, acceptable quality) + // use std::minstd_rand (Knuth PRNG, LCG with p=2^31-1, a=16807) + std::minstd_rand prng_; + + // Internal helpers + void + init_partitions_from_hosts(); + uint64_t + hash_seed(const std::string& path, chunkid_t chnk_id) const; + +public: + explicit RandomSlicingDistributor(host_t localhost, + unsigned int hosts_size); + RandomSlicingDistributor() = default; + + // Distributor interface + host_t + localhost() const override; + unsigned int + hosts_size() const override; + void + hosts_size(unsigned int size) override; + + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + const int num_copy) const override; + host_t + locate_data(const std::string& path, const chunkid_t& chnk_id, + unsigned int hosts_size, const int num_copy) override; + host_t + locate_file_metadata(const std::string& path, + const int num_copy) const override; + std::vector + locate_directory_metadata() const override; + + // Random Slicing-specific methods + void + add_nodes(std::vector new_nodes); + void + remove_nodes(std::vector old_nodes); + void + reconfigure(); + + bool + set_intervals(const std::vector& intervals); + + std::vector + get_intervals() const; + + // Interval table persistence (disabled - intervals rebuilt each + // launch) + void + save_interval_table(const std::string& path) + const { /* disabled: intervals rebuilt each launch */ } + bool + load_interval_table(const std::string& path) { /* disabled: intervals + rebuilt each launch */ + return false; + } + + // Access to current partitions (for migration tracking) + const std::vector& + get_partitions() const { + return partitions_; + } + std::vector + get_partitions_copy() { + return partitions_; + } +}; + +} // namespace rpc +} // namespace gkfs + +#endif // GKFS_RPC_RANDOM_SLICING_DISTRIBUTOR_H \ No newline at end of file diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index 34d7251de9b381cabed74f65b4240fedd73763a2..6b0e06e946c58a3f36f78077286c9905ee79379b 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -447,24 +447,23 @@ struct rpc_proxy_daemon_read_in_t { }; // Malleability -struct rpc_expand_start_in_t { +struct rpc_mutate_start_in_t { uint32_t old_server_conf; uint32_t new_server_conf; + std::string new_hosts_file; template void serialize(Archive& ar) { - ar(old_server_conf, new_server_conf); + ar(old_server_conf, new_server_conf, new_hosts_file); } }; -struct rpc_shrink_start_in_t { - uint32_t old_server_conf; - uint32_t new_server_conf; - std::string new_hosts_file; +struct rpc_mutate_reload_in_t { + std::string hosts_file; template void serialize(Archive& ar) { - ar(old_server_conf, new_server_conf, new_hosts_file); + ar(hosts_file); } }; diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index b97c427cb8c5025583b02e730ff9c1c74277d42f..a3c27c0feda976b0e5f0e147b3702b118baa7580 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -41,10 +41,13 @@ #include +#include #include #include #include //std::hash #include +#include +#include /* Forward declarations */ namespace gkfs { @@ -65,6 +68,10 @@ namespace malleable { class MalleableManager; } +namespace rpc { +class Distributor; +} + namespace daemon { class FsData { @@ -122,6 +129,9 @@ private: // it. bool keep_hosts_file_ = false; + // When true, daemon writes '+' prefix to hostfile (for expand operations) + bool expand_mode_ = false; + // Prometheus std::string prometheus_gateway_ = gkfs::config::stats::prometheus_gateway; @@ -129,8 +139,19 @@ private: // indicates for clients: try again. Is set to true when redist is running bool maintenance_mode_ = false; ABT_mutex maintenance_mode_mutex_; + bool mutate_start_active_ = false; + int mutate_old_server_conf_ = 0; + int mutate_new_server_conf_ = 0; + std::string mutate_hosts_file_; // redist_running_ indicates to client that redistribution is running - bool redist_running_ = false; + std::atomic redist_running_{false}; + + bool expand_on_demand_active_ = false; + unsigned int expand_on_demand_old_hosts_size_ = 0; + uint64_t expand_on_demand_old_local_host_id_ = + std::numeric_limits::max(); + std::shared_ptr expand_on_demand_old_distributor_; + std::map expand_on_demand_old_rpc_endpoints_; std::shared_ptr malleable_manager_; @@ -146,6 +167,38 @@ public: void operator=(FsData const&) = delete; + bool + expand_on_demand_active() const; + + void + expand_on_demand_active(bool active); + + unsigned int + expand_on_demand_old_hosts_size() const; + + void + expand_on_demand_old_hosts_size(unsigned int hosts_size); + + uint64_t + expand_on_demand_old_local_host_id() const; + + void + expand_on_demand_old_local_host_id(uint64_t host_id); + + std::shared_ptr + expand_on_demand_old_distributor() const; + + void + expand_on_demand_old_distributor( + std::shared_ptr distributor); + + const std::map& + expand_on_demand_old_rpc_endpoints() const; + + void + expand_on_demand_old_rpc_endpoints( + std::map endpoints); + // getter/setter const std::shared_ptr& @@ -321,6 +374,15 @@ public: void maintenance_mode(bool maintenance_mode); + // Returns true when this request is a duplicate of the active mutation, + // false when a new mutation was registered, and throws for a conflict. + bool + begin_mutate_start(int old_server_conf, int new_server_conf, + const std::string& hosts_file); + + void + end_mutate_start(); + bool redist_running() const; @@ -339,6 +401,12 @@ public: void keep_hosts_file(bool keep); + + bool + expand_mode() const; + + void + expand_mode(bool expand_mode); }; diff --git a/include/daemon/daemon.hpp b/include/daemon/daemon.hpp index dfdc0998f6f654a642cf084daab9b74cee9f3f84..57156d767e1ba59e5664e33f6afba05c5d7882c7 100644 --- a/include/daemon/daemon.hpp +++ b/include/daemon/daemon.hpp @@ -85,4 +85,11 @@ struct formatter : formatter { ///< the RPCData singleton ///< across the daemon +namespace gkfs::daemon { + +void +request_shutdown(); + +} // namespace gkfs::daemon + #endif // GKFS_DAEMON_DAEMON_HPP diff --git a/include/daemon/env.hpp b/include/daemon/env.hpp index e1bccf4112f6335ded902cd86008ce58d9ed2602..1cfe7eecee3f87a2c6eb3a6608af85ec16d054b8 100644 --- a/include/daemon/env.hpp +++ b/include/daemon/env.hpp @@ -46,26 +46,18 @@ #include #include -#define ADD_PREFIX(str) COMMON_ENV_PREFIX str +#define ADD_PREFIX(str) DAEMON_ENV_PREFIX str /* Environment variables for the GekkoFS daemon */ namespace gkfs::env { static constexpr auto METADATA_DB_PATH = ADD_PREFIX("METADATA_DB_PATH"); -static constexpr auto USE_DIRENTS_COMPRESSION = - ADD_PREFIX("DAEMON_USE_DIRENTS_COMPRESSION"); -static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE"); -static constexpr auto DAEMON_CREATE_CHECK_PARENTS = - ADD_PREFIX("DAEMON_CREATE_CHECK_PARENTS"); +static constexpr auto HOSTS_FILE = COMMON_ENV_PREFIX "HOSTS_FILE"; static constexpr auto DAEMON_CREATE_EXIST_CHECK = - ADD_PREFIX("DAEMON_CREATE_EXIST_CHECK"); -static constexpr auto DAEMON_SYMLINK_SUPPORT = - ADD_PREFIX("DAEMON_SYMLINK_SUPPORT"); -static constexpr auto DAEMON_RENAME_SUPPORT = - ADD_PREFIX("DAEMON_RENAME_SUPPORT"); -static constexpr auto DAEMON_USE_INLINE_DATA = - ADD_PREFIX("DAEMON_USE_INLINE_DATA"); + ADD_PREFIX("CREATE_EXIST_CHECK"); static constexpr auto KEEP_HOSTS_FILE = ADD_PREFIX("KEEP_HOSTS_FILE"); +static constexpr auto DAEMON_EXPAND_MODE = ADD_PREFIX("EXPAND"); +static constexpr auto ENABLE_WAL = ADD_PREFIX("ENABLE_WAL"); } // namespace gkfs::env diff --git a/include/daemon/handler/rpc_defs.hpp b/include/daemon/handler/rpc_defs.hpp index 81d39011ee30d2fa20b607427352e093b9e4289c..338fcaa9adefefefe0348d91c94e2493edcb1f1c 100644 --- a/include/daemon/handler/rpc_defs.hpp +++ b/include/daemon/handler/rpc_defs.hpp @@ -44,6 +44,7 @@ #define GKFS_DAEMON_RPC_DEFS_HPP #include +#include // client <-> daemon RPCs @@ -135,31 +136,25 @@ rpc_srv_get_chunk_stat(const tl::request& req, // void proxy_rpc_srv_read(const tl::request& req, ...); // void proxy_rpc_srv_write(const tl::request& req, ...); -// malleability - -void -rpc_srv_expand_start(const tl::request& req, - const gkfs::rpc::rpc_expand_start_in_t& in); - void -rpc_srv_expand_status(const tl::request& req); - +rpc_srv_migrate_metadata(const tl::request& req, + const gkfs::rpc::rpc_migrate_metadata_in_t& in); void -rpc_srv_expand_finalize(const tl::request& req); +rpc_srv_mutate_start(const tl::request& req, + const gkfs::rpc::rpc_mutate_start_in_t& in); void -rpc_srv_shrink_start(const tl::request& req, - const gkfs::rpc::rpc_shrink_start_in_t& in); +rpc_srv_mutate_status(const tl::request& req); void -rpc_srv_shrink_status(const tl::request& req); +rpc_srv_mutate_finalize(const tl::request& req); void -rpc_srv_shrink_finalize(const tl::request& req); +rpc_srv_mutate_reload(const tl::request& req, + const gkfs::rpc::rpc_mutate_reload_in_t& in); void -rpc_srv_migrate_metadata(const tl::request& req, - const gkfs::rpc::rpc_migrate_metadata_in_t& in); +rpc_srv_mutate_shutdown(const tl::request& req); // inline data operations void diff --git a/include/daemon/malleability/malleable_manager.hpp b/include/daemon/malleability/malleable_manager.hpp index 1c6b33b39b40bd786784c9f7a1e98f9f3ed7356b..014fb0d94256049b01fd08e19ce4df21699d339c 100644 --- a/include/daemon/malleability/malleable_manager.hpp +++ b/include/daemon/malleability/malleable_manager.hpp @@ -39,6 +39,8 @@ #define GEKKOFS_DAEMON_MALLEABLE_MANAGER_HPP #include +#include +#include namespace gkfs::malleable { @@ -46,6 +48,9 @@ class MalleableManager { private: ABT_thread redist_thread_; + // Tracks old hosts_size before expansion/shrink + unsigned int old_hosts_size_{0}; + // TODO next 3 functions are mostly copy paste from preload_util. FIX std::vector> @@ -62,19 +67,27 @@ private: int redistribute_metadata(); + /// Migration job handler: called by DataMigrationExecutor for each job + int + do_migration(gkfs::rpc::MigrationJob& job); + + /// New data redistribution using DataMigrationExecutor pipeline void - redistribute_data(); + redistribute_data_v2(); static void - expand_abt(void* _arg); + expand_abt_v2(void* _arg); + + static void + expand_on_demand_abt(void* _arg); public: void - expand_start(int old_server_conf, int new_server_conf); + mutate_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file); void - shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file); + reload_hosts_file(const std::string& hosts_file); }; } // namespace gkfs::malleable diff --git a/include/daemon/ops/data.hpp b/include/daemon/ops/data.hpp index 43b7fa9000f9e922df3b2e481b6d6a0440192c07..99001022e2581bd604ca88498134e0d186c6305e 100644 --- a/include/daemon/ops/data.hpp +++ b/include/daemon/ops/data.hpp @@ -66,6 +66,14 @@ extern "C" { namespace gkfs::data { +std::pair +expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, + char* buf, size_t size, off64_t offset); + +int +expand_on_demand_materialize_for_partial_write(const std::string& path, + uint64_t chunk_id); + /** * @brief Internal Exception for all general chunk operations. */ diff --git a/include/daemon/util.hpp b/include/daemon/util.hpp index d557060cc853721914b7f85e0ee823a2d8ea8576..7f2787ebdca9d846df8e546e57d9c03958bc55a6 100644 --- a/include/daemon/util.hpp +++ b/include/daemon/util.hpp @@ -45,10 +45,11 @@ namespace gkfs::utils { /** * @brief Registers the daemon's RPC address to the shared hosts file. + * @param expand_mode If true, prepend '+' marker to the line * @throws std::runtime_error when file stream fails */ void -populate_hosts_file(); +populate_hosts_file(bool expand_mode = false); /** * @brief Attempts to remove the entire hosts file. diff --git a/scripts/run/gkfs b/scripts/run/gkfs index d59fd99a14d5ff93b289cde33525b46b94fc47f7..3e5aebd1f3876db15b33fbf517df3f9b7e8e624e 100755 --- a/scripts/run/gkfs +++ b/scripts/run/gkfs @@ -18,6 +18,58 @@ C_AST_RED="${C_BRED}*${C_NONE} [gkfs] " # Important const globals FS_INSTANCE_MARKER_CONST="#FS_INSTANCE_END" + +count_host_lines() { + local file="$1" + grep -v '^#' "${file}" 2>/dev/null | grep -cv '^[[:space:]]*$' +} + +count_before_mutate_host_lines() { + local file="$1" + grep -v '^#' "${file}" 2>/dev/null | grep -v '^[[:space:]]*+' | grep -cv '^[[:space:]]*$' +} + +ensure_malleability_bin() { + if [[ -n ${GKFS_MALLEABILITY_BIN_} ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "${GKFS_MALLEABILITY_BIN_}") + else + GKFS_MALLEABILITY_BIN_=$(command -v gkfs_malleability) + fi + if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then + if [[ -f $(dirname "${DAEMON_BIN}")/gkfs_malleability ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "${DAEMON_BIN}")/gkfs_malleability") + elif [[ -f $(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability ]]; then + GKFS_MALLEABILITY_BIN_=$(readlink -f "$(dirname "$(dirname "${DAEMON_BIN}")")/tools/gkfs_malleability") + else + echo -e "${C_AST_RED}ERROR: gkfs_malleability binary not found. Exiting ..." + exit 1 + fi + fi +} + +run_malleability_workflow() { + local label="$1" + local old_count="$2" + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + + ${GKFS_MALLEABILITY_BIN_} mutate start + echo -e "${C_AST_GREEN}${label} progress: " + until MALLEABILITY_STATUS=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate status 2>&1); [ $((${MALLEABILITY_STATUS})) -eq 0 ] + do + sleep 1 + show_expand_progress ${MALLEABILITY_STATUS} ${old_count} + done + show_expand_progress ${MALLEABILITY_STATUS} ${old_count} + echo + echo -e "${C_AST_GREEN}Redistribution process done. Finalizing ..." + MALLEABILITY_FINALIZE=$(${GKFS_MALLEABILITY_BIN_} --machine-readable mutate finalize 2>&1) + if [ $((${MALLEABILITY_FINALIZE})) -ne 0 ]; then + echo -e "${C_AST_RED}ERROR: ${label} finalize failed. This is not recoverable. Exiting ..." + exit 1 + fi + echo -e "${C_AST_GREEN}${label} done." +} ####################################### # Poll GekkoFS hostsfile until all daemons are started. # Exits with 1 if daemons cannot be started. @@ -38,11 +90,12 @@ wait_for_gkfs_daemons() { if [[ -n ${NODE_NUM} ]]; then nodes=${NODE_NUM} fi - # when expanding the total number of nodes is: initial nodelist + expand nodelist - if [[ ${COMMAND} == *"expand"* ]]; then + # when expanding/mutating with new daemons, the hostfile contains old entries + # plus '+' entries written by the added daemons + if [[ ${GKFS_DAEMON_EXPAND} == "ON" ]]; then nodes=${NODE_CNT_EXPAND} fi - until [ $(($(grep -cv '^#' "${HOSTSFILE}" 2> /dev/null | awk '{print $1}') + 0)) -eq "${nodes}" ] + until [ $(($(count_host_lines "${HOSTSFILE}" | awk '{print $1}') + 0)) -eq "${nodes}" ] do #echo "Waiting for all servers to report connection. Try $server_wait_cnt" sleep 2 @@ -362,70 +415,6 @@ stop_daemons() { fi } -####################################### -# Sets up expand progress for later operation -# Globals: -# RUN_FOREGROUND -# EXPAND_NODELIST -# HOSTSFILE -# DAEMON_NODELIST -# USE_PROXY -# GKFS_MALLEABILITY_BIN_ -# VERBOSE -# Outputs: -# sets GKFS_MALLEABILITY_BIN_ if not already given by config -####################################### -expand_setup() { - # sanity checks - if [[ ${RUN_FOREGROUND} == true ]]; then - echo -e "${C_AST_RED}ERROR: Cannot run in foreground for expansion. Exiting ..." - exit 1 - fi - if [[ -z ${EXPAND_NODELIST} ]]; then - echo -e "${C_AST_RED}ERROR: No expand host file given. We need to know which nodes should be used. Exiting ..." - exit 1 - fi - # if proxy is enabled error out - # to support proxy, all proxies need to be shutdown during expansion and started up after again - # to get the new configuration. - if [[ ${USE_PROXY} == true ]]; then - echo -e "${C_AST_RED}ERROR: Proxy not supported for expansion. Exiting ..." - exit 1 - fi - # check that gkfs host file exists - if [[ ! -f ${HOSTSFILE} ]]; then - echo -e "${C_AST_RED}ERROR: No GekkoFS hostfile for expansion found at ${HOSTSFILE}. Exiting ..." - exit 1 - fi - # check that daemon pid file exists - if [[ ! -f ${DAEMON_PID_FILE} ]]; then - echo -e "${C_AST_RED}ERROR: No daemon pid file found at ${DAEMON_PID_FILE}." - echo -e "${C_AST_RED} Existing daemon must run in background for extension. Exiting ..." - exit 1 - fi - # modify all necessary environment variables from the config file to fit expand - DAEMON_NODELIST_=${DAEMON_NODELIST} - # Set daemon node list based on given expand hostfile - DAEMON_NODELIST_=$(readlink -f ${EXPAND_NODELIST}) - # setup - # This must be equivalent to the line set in include/common/common_defs.hpp - echo "$FS_INSTANCE_MARKER_CONST" >> "${HOSTSFILE}" - # check that the gkfs_malleability binary exists in $PATH if not already set via config - if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then - GKFS_MALLEABILITY_BIN_=$(COMMAND -v gkfs_malleability) - fi - # if not found check if it exists in the parent directory of the daemon bin - if [[ -z ${GKFS_MALLEABILITY_BIN_} ]]; then - # check that the gkfs_malleability binary exists somewhere in the parent directory where daemon bin is located - if [[ -f $(dirname ${DAEMON_BIN})/gkfs_malleability ]]; then - GKFS_MALLEABILITY_BIN_=$(readlink -f $(dirname ${DAEMON_BIN})/gkfs_malleability) - else - echo -e "${C_AST_RED}ERROR: gkfs_malleability binary not found. Exiting ..." - exit 1 - fi - fi -} - ####################################### # Prints expansion progress # Input: @@ -458,43 +447,33 @@ show_expand_progress() { printf "] %d/%d left" "$current" "$total" } -####################################### -# Adds GekkoFS daemons to an existing GekkoFS instance -# Globals: -# DAEMON_PID_FILE -# PROXY_PID_FILE -# VERBOSE -# Outputs: -# Writes status to stdout -####################################### -add_daemons() { - expand_setup - # get old and new node configuration - local node_cnt_initial=$(grep -v '^#' "${HOSTSFILE}" | wc -l) - NODE_CNT_EXPAND=$((${node_cnt_initial}+$(cat ${EXPAND_NODELIST} | wc -l))) - # start new set of daemons - start_daemons - export LIBGKFS_HOSTS_FILE=${HOSTSFILE} - # start expansion which redistributes metadata and data - ${GKFS_MALLEABILITY_BIN_} expand start - echo -e "${C_AST_GREEN}Expansion progress: " - # wait for expansion to finish - until EXPAND_STATUS=$(${GKFS_MALLEABILITY_BIN_} -m expand status); [ $((${EXPAND_STATUS})) -eq 0 ] - do - sleep 1 - show_expand_progress ${EXPAND_STATUS} ${node_cnt_initial} - done - show_expand_progress ${EXPAND_STATUS} ${node_cnt_initial} - echo - # finalize and remove marker - echo -e "${C_AST_GREEN}Redistribution process done. Finalizing ..." - sed -i '/^#/d' ${HOSTSFILE} - EXPAND_FINALIZE=$(${GKFS_MALLEABILITY_BIN_} -m expand finalize) - if [ $((${EXPAND_FINALIZE})) -ne 0 ]; then - echo -e "${C_AST_RED}ERROR: Expansion finalized failed. This is not recoverable. Exiting ..." +mutate_daemons() { + if [[ ${RUN_FOREGROUND} == true ]]; then + echo -e "${C_AST_RED}ERROR: Cannot run in foreground for mutate. Exiting ..." + exit 1 + fi + if [[ ${USE_PROXY} == true ]]; then + echo -e "${C_AST_RED}ERROR: Proxy not supported for mutate. Exiting ..." exit 1 fi - echo -e "${C_AST_GREEN}Expansion done." + if [[ ! -f ${HOSTSFILE} ]]; then + echo -e "${C_AST_RED}ERROR: No GekkoFS hostfile found at ${HOSTSFILE}. Exiting ..." + exit 1 + fi + local node_cnt_initial=$(count_before_mutate_host_lines "${HOSTSFILE}") + run_malleability_workflow "Mutate" "${node_cnt_initial}" +} + +malleability_status() { + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + ${GKFS_MALLEABILITY_BIN_} mutate status +} + +malleability_finalize() { + ensure_malleability_bin + export LIBGKFS_HOSTS_FILE=${HOSTSFILE} + ${GKFS_MALLEABILITY_BIN_} mutate finalize } ####################################### @@ -504,9 +483,9 @@ add_daemons() { ####################################### usage_short() { echo " -usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-a/--args ] [--proxy ] [-f/--foreground ] - [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-v/--verbose ] - {start,expand,stop} +usage: gkfs [-h/--help] [-r/--rootdir ] [-m/--mountdir ] [-d/--daemon_args ] [--proxy ] [-f/--foreground ] + [--srun ] [-n/--numnodes ] [--cpuspertask <64>] [-H/--hostsfile ] [-v/--verbose ] + {start,mutate,status,finalize,stop} " } ####################################### @@ -522,7 +501,7 @@ help_msg() { additional permanent configurations can be set. positional arguments: - COMMAND Command to execute: 'start', 'stop', 'expand' + COMMAND Command to execute: 'start', 'stop', 'mutate', 'status', 'finalize' optional arguments: -h, --help Shows this help message and exits @@ -538,17 +517,19 @@ help_msg() { Nodelist is extracted from Slurm via the SLURM_JOB_ID env variable. --cpuspertask <#cores> Set the number of cores the daemons can use. Must use '--srun'. -c, --config Path to configuration file. By defaults looks for a 'gkfs.conf' in this directory. - -e, --expand_hostfile Path to the hostfile with new nodes where GekkoFS should be extended to (hostfile contains one line per node). + -H, --hostsfile Path to the GekkoFS hostfile. For mutate, this may already contain '+' and '-' markers. -v, --verbose Increase verbosity -t, --time Set a limit on the total run time of the slurm job allocation. -A, --account Account for the slurm job (only required for job allocation) -P, --partition Partition for the slurm job (only required for job allocation) + malleability examples: + gkfs mutate " } CONFIGPATH="" argv=("$@") # get config path first from argument list -for i in "${argv[@]}"; do +for i in "${!argv[@]}"; do if [[ "${argv[i]}" == "-c" || "${argv[i]}" == "--config" ]]; then CONFIGPATH=$(readlink -mn "${argv[i+1]}") break @@ -593,7 +574,6 @@ HOSTSFILE=$(readlink -f ${HOSTSFILE}) PROXY_LOCAL_PID_FILE=$(readlink -f ${PROXY_LOCAL_PID_FILE}) DAEMON_PID_FILE=$(readlink -f ${DAEMON_PID_FILE}) PROXY_PID_FILE=$(readlink -f ${PROXY_PID_FILE}) -EXPAND_NODELIST="" GKFS_MALLEABILITY_BIN_=${GKFS_MALLEABILITY_BIN} # parse input @@ -659,8 +639,8 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; - -e | --expand_hostfile) - EXPAND_NODELIST=$2 + -H | --hostsfile) + HOSTSFILE=$2 shift # past argument shift # past value ;; @@ -694,6 +674,7 @@ while [[ $# -gt 0 ]]; do esac done set -- "${POSITIONAL[@]}" # restore positional parameters +HOSTSFILE=$(readlink -f ${HOSTSFILE}) # positional arguments if [[ -z ${1+x} ]]; then @@ -703,7 +684,7 @@ if [[ -z ${1+x} ]]; then fi COMMAND="${1}" # checking input -if [[ ${COMMAND} != *"start"* ]] && [[ ${COMMAND} != *"stop"* ]] && [[ ${COMMAND} != *"expand"* ]]; then +if [[ ${COMMAND} != "start" ]] && [[ ${COMMAND} != "stop" ]] && [[ ${COMMAND} != "mutate" ]] && [[ ${COMMAND} != "status" ]] && [[ ${COMMAND} != "finalize" ]]; then echo -e "${C_AST_RED}ERROR: COMMAND ${COMMAND} not supported" usage_short exit 1 @@ -713,8 +694,12 @@ if [[ ${COMMAND} == "start" ]]; then start_daemons elif [[ ${COMMAND} == "stop" ]]; then stop_daemons -elif [[ ${COMMAND} == "expand" ]]; then - add_daemons +elif [[ ${COMMAND} == "mutate" ]]; then + mutate_daemons +elif [[ ${COMMAND} == "status" ]]; then + malleability_status +elif [[ ${COMMAND} == "finalize" ]]; then + malleability_finalize fi if [[ ${VERBOSE} == true ]]; then echo -e "${C_AST_GREEN}Nothing left to do. Exiting :)" diff --git a/scripts/run/gkfs.conf b/scripts/run/gkfs.conf index 3650e682e1f7eaf649906a2179afba0494c2b989..7c32449fdaf2043864b44372013bf9b589a5e307 100644 --- a/scripts/run/gkfs.conf +++ b/scripts/run/gkfs.conf @@ -8,6 +8,12 @@ PROXY_BIN=../../build/src/proxy/gkfs_proxy # client configuration (needs to be set for all clients) LIBGKFS_HOSTS_FILE=/home/XXX/workdir/gkfs_hosts.txt +# distribution configuration shared by clients, daemons, and tools +# supported examples: simple_hash, random_slicing +export GKFS_DISTRIBUTION_STRATEGY=simple_hash +# enable Random Slicing CutShift during expand-only malleability operations +export GKFS_RANDOM_SLICING_CUTSHIFT=OFF + # tools (if build) GKFS_MALLEABILITY_BIN=../../build/tools/gkfs_malleability diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 8485a590fc9204ab4961100ec63e2448234080b0..e6fb81b8ffee2d5ea6b342f753fa5739b37f6263 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -50,6 +50,7 @@ target_sources(gkfs_common rpc/forward_malleability.cpp cache.cpp syscalls/detail/syscall_info.c syscalls/util.S + ../common/malleability_markers.cpp ) target_link_libraries( diff --git a/src/client/fuse/fuse_client.cpp b/src/client/fuse/fuse_client.cpp index 87ff84e864b4db8ce0a447554c270d28671a0974..8571e4b402179b515b46a244f29df6d5777db580 100644 --- a/src/client/fuse/fuse_client.cpp +++ b/src/client/fuse/fuse_client.cpp @@ -38,6 +38,7 @@ */ #include +#include #ifdef GKFS_ENABLE_CLIENT_METRICS #include #endif @@ -1405,20 +1406,20 @@ main(int argc, char* argv[]) { ud.attr_timeout = ud.timeout; ud.negative_timeout = ud.timeout; - if(const char* env_e = std::getenv("GKFS_FUSE_ENTRY_TIMEOUT")) { + if(const char* env_e = std::getenv(gkfs::env::FUSE_ENTRY_TIMEOUT)) { ud.entry_timeout = std::stod(env_e); LOG(INFO, "FUSE entry_timeout set to {}", ud.entry_timeout); } - if(const char* env_a = std::getenv("GKFS_FUSE_ATTR_TIMEOUT")) { + if(const char* env_a = std::getenv(gkfs::env::FUSE_ATTR_TIMEOUT)) { ud.attr_timeout = std::stod(env_a); LOG(INFO, "FUSE attr_timeout set to {}", ud.attr_timeout); } - if(const char* env_n = std::getenv("GKFS_FUSE_NEGATIVE_TIMEOUT")) { + if(const char* env_n = std::getenv(gkfs::env::FUSE_NEGATIVE_TIMEOUT)) { ud.negative_timeout = std::stod(env_n); LOG(INFO, "FUSE negative_timeout set to {}", ud.negative_timeout); } - if(const char* env_w = std::getenv("GKFS_FUSE_WRITEBACK")) { + if(const char* env_w = std::getenv(gkfs::env::FUSE_WRITEBACK)) { ud.writeback = (std::string(env_w) == "ON" || std::string(env_w) == "1"); LOG(INFO, "FUSE writeback set to {}", ud.writeback); diff --git a/src/client/gkfs_data.cpp b/src/client/gkfs_data.cpp index 748c4df1fd452776af0e30a4263f1a08ec329925..88f85f0ced40f30d4caedb1519d4617fc31850ca 100644 --- a/src/client/gkfs_data.cpp +++ b/src/client/gkfs_data.cpp @@ -703,9 +703,6 @@ gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { } } - if(read == 0) { - return -1; - } return read; } @@ -727,7 +724,6 @@ gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { } auto pos = gkfs_fd->pos(); // retrieve the current offset auto ret = gkfs_preadv(fd, iov, iovcnt, pos); - assert(ret != 0); if(ret < 0) { return -1; } diff --git a/src/client/gkfs_metadata.cpp b/src/client/gkfs_metadata.cpp index c611fd09b24efd5eae065431a2b3454c09a6ce4f..ba31c629db3da2b3a72b8380454b96d4e32182ca 100644 --- a/src/client/gkfs_metadata.cpp +++ b/src/client/gkfs_metadata.cpp @@ -1259,7 +1259,8 @@ gkfs_opendir(const std::string& path) { } auto entries = gkfs::rpc::decompress_and_parse_entries( - out, buffers[buffer_id].data()); + out, buffers[buffer_id].data(), + buffers[buffer_id].size()); consume_entries(entries); if(!entries.empty()) { const auto& last_key = std::get<0>(entries.back()); diff --git a/src/client/malleability.cpp b/src/client/malleability.cpp index aaa0378c5ec9f1bd2578a0aefa035df7f4f96085..0894980aaa3cff3849a57493c8b32ab4643254c2 100644 --- a/src/client/malleability.cpp +++ b/src/client/malleability.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include @@ -49,81 +50,108 @@ using namespace std; namespace gkfs::malleable { int -expand_start(int old_server_conf, int new_server_conf) { - LOG(INFO, "{}() Expand operation enter", __func__); - // sanity checks - if(old_server_conf == new_server_conf) { - auto err_str = - "ERR: Old server configuration is the same as the new one"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; - } - if(CTX->hosts().size() != static_cast(old_server_conf)) { - auto err_str = - "ERR: Old server configuration does not match the number of hosts in hostsfile"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; - } - // TODO check that hostsfile contains endmarker - return gkfs::malleable::rpc::forward_expand_start(old_server_conf, - new_server_conf); -} - -int -expand_status() { - LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_expand_status(); - LOG(INFO, "{}() '{}' nodes working on extend operation.", __func__, res); - return res; -} - -int -expand_finalize() { - LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_expand_finalize(); - LOG(INFO, "{}() extend operation finalized. ", __func__); - return res; -} - -int -shrink_start(int old_server_conf, int new_server_conf, +mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file) { - LOG(INFO, "{}() Shrink operation enter", __func__); + LOG(INFO, "{}() Mutate operation enter", __func__); // sanity checks + auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); if(old_server_conf == new_server_conf) { - auto err_str = - "ERR: Old server configuration is the same as the new one"; - cerr << err_str << endl; - LOG(ERROR, "{}() {}", __func__, err_str); - return -1; + bool same_count_marker_mode = false; + if(hf) { + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + same_count_marker_mode = markers.has_markers() && + !markers.adding.empty() && + !markers.removing.empty(); + } catch(...) { + // Keep the old error below for malformed/non-marker hostfiles. + } + } + if(!same_count_marker_mode) { + auto err_str = + "ERR: Old server configuration is the same as the new one"; + cerr << err_str << endl; + LOG(ERROR, "{}() {}", __func__, err_str); + return -1; + } } - if(CTX->hosts().size() != static_cast(old_server_conf)) { + // For marker-based workflow: the CLI tool computes old_nodes = active + + // removing. But CTX->hosts() only contains active hosts (loaded at + // gkfs_init time from LIBGKFS_HOSTS_FILE). For shrink operations, the + // removing nodes are marked with '-' prefix and are not loaded into CTX. + // + // Key insight: We can't lookup removing hosts at the client side because + // they may be shutting down. Instead, the daemon handles data migration + // from its own state (which includes removing hosts parsed from markers). + // We just need to validate that the marker file has the right number of + // hosts. + uint64_t detected_hosts = CTX->hosts().size(); + bool marker_mode = false; + if(hf && detected_hosts != static_cast(old_server_conf)) { + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + uint64_t total_hosts = static_cast( + markers.active.size() + markers.removing.size()); + if(total_hosts == static_cast(old_server_conf)) { + // Marker file has the right count - this is a shrink operation + // The daemon will handle data migration using its own host list + marker_mode = true; + } + } catch(...) { + // Ignore parse errors + } + } + if(!marker_mode && + CTX->hosts().size() != static_cast(old_server_conf)) { auto err_str = "ERR: Old server configuration does not match the number of hosts in hostsfile"; cerr << err_str << endl; LOG(ERROR, "{}() {}", __func__, err_str); return -1; } - return gkfs::malleable::rpc::forward_shrink_start( + return gkfs::malleable::rpc::forward_mutate_start( old_server_conf, new_server_conf, new_hosts_file); } int -shrink_status() { +mutate_status() { LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_shrink_status(); - LOG(INFO, "{}() '{}' nodes working on shrink operation.", __func__, res); + auto res = gkfs::malleable::rpc::forward_mutate_status(); + LOG(INFO, "{}() '{}' nodes working on mutate operation.", __func__, res); return res; } int -shrink_finalize() { +mutate_finalize() { LOG(INFO, "{}() enter", __func__); - auto res = gkfs::malleable::rpc::forward_shrink_finalize(); - LOG(INFO, "{}() shrink operation finalized. ", __func__); + auto res = gkfs::malleable::rpc::forward_mutate_finalize(); + const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE"); + if(res == 0 && hf != nullptr) { + // Keep the marker file intact while removed-daemon endpoints are + // collected and shut down. Cleanup below removes those markers only + // after the old daemons have been told to exit. + auto shutdown_res = + gkfs::malleable::rpc::forward_mutate_shutdown_removed(hf); + if(shutdown_res != 0) { + LOG(ERROR, "{}() failed to gracefully shutdown removed daemons: {}", + __func__, shutdown_res); + return shutdown_res; + } + + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + auto rs_intervals = gkfs::malleable::parse_rs_interval_comments(hf); + gkfs::malleable::write_clean_hostfile(hf, markers, rs_intervals); + LOG(INFO, "{}() wrote final clean hosts file '{}'", __func__, hf); + res = gkfs::malleable::rpc::forward_mutate_reload(hf); + } catch(const std::exception& e) { + LOG(ERROR, "{}() failed to clean/reload final hosts file: {}", + __func__, e.what()); + res = EIO; + } + } + LOG(INFO, "{}() mutate operation finalized. ", __func__); return res; } -} // namespace gkfs::malleable \ No newline at end of file +} // namespace gkfs::malleable diff --git a/src/client/open_file_map.cpp b/src/client/open_file_map.cpp index 1e36b6b082894e67cd5b287b2bfa67e19b6583e3..52b403ae780977dbed4b7eb8ec49f97b782fe281 100644 --- a/src/client/open_file_map.cpp +++ b/src/client/open_file_map.cpp @@ -260,6 +260,16 @@ OpenFileMap::remove(const int fd) { shard.files.erase(fd); total_files_--; + if(fd >= 0 && fd < 3) { + // A GKFS file descriptor may intentionally be mapped onto stdin, + // stdout, or stderr via dup2()/dup3() (GNU dd does this for its input + // and output files). In that case the numeric descriptor can also name + // a real kernel stdio fd. The GKFS close path must only remove the + // virtual mapping; closing the kernel fd here may invalidate + // libc/logging descriptors during process teardown. + return true; + } + if(!CTX->protect_fds()) { if(!CTX->range_fd()) { // We close the dev null fd diff --git a/src/client/preload.cpp b/src/client/preload.cpp index fe3996a635b9043ed372d86b09c1f64db5ef6aa6..e658a858bd5778e94fa197e12ac6d11e25b63c71 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -43,14 +43,18 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include +#include #ifdef GKFS_ENABLE_CLIENT_METRICS #include #endif @@ -288,15 +292,44 @@ init_environment() { CTX->fwd_host_id(), CTX->hosts().size()); CTX->distributor(forwarder_dist); } else { - -#ifdef GKFS_USE_GUIDED_DISTRIBUTION - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#else - auto distributor = std::make_shared( - CTX->local_host_id(), CTX->hosts().size()); -#endif - CTX->distributor(distributor); + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + LOG(INFO, "{}() Distribution strategy: '{}'", __func__, + config.get_strategy_string()); + + auto distributor = + create_from_config(config, CTX->local_host_id(), + CTX->hosts().size(), CTX->fwd_host_id()); + if(!distributor) { + exit_error_msg(EXIT_FAILURE, "Failed to create distributor"); + } + if(config.is_random_slicing()) { + if(const auto* hostfile = std::getenv(gkfs::env::HOSTS_FILE)) { + auto rs = dynamic_cast( + distributor.get()); + auto comments = + gkfs::malleable::parse_rs_interval_comments(hostfile); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + if(rs && !intervals.empty() && rs->set_intervals(intervals)) { + LOG(INFO, + "{}() Loaded {} random-slicing intervals from hostfile", + __func__, intervals.size()); + } + } + } + CTX->distributor(std::move(distributor)); } auto use_dcache = gkfs::env::get_var(gkfs::env::cache::DENTRY, diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index 27733afb416090030a3d2bae862ef8cd9480a6ba..61efc9b08a07af945e62b6d661d9a4e259e01f34 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include // #include @@ -174,19 +175,14 @@ load_hostfile(const std::string& path) { path, strerror(errno))); } vector> hosts; + vector> removing_hosts; const regex line_re( "^(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)\\s+(\\S+)$", regex::ECMAScript | regex::optimize); - - string line; std::smatch match; - while(getline(lf, line)) { - // if line starts with #, it indicates the end of current FS instance - // Further hosts are not part of the file system instance yet and are - // therefore skipped The hostfile is ordered, so nothgin below this line - // can contain valid hosts - if(line.find(gkfs::client::hostsfile_end_str) != string::npos) - break; + + auto parse_host_line = [&](const string& line, + vector>& out) { if(!regex_match(line, match, line_re)) { LOG(ERROR, "Unrecognized line format: [path: '{}', line: '{}']", path, line); @@ -197,7 +193,7 @@ load_hostfile(const std::string& path) { string host = match[1]; string uri = match[2]; // match[3] that is the proxy (not used here) - hosts.emplace_back(host, uri); + out.emplace_back(host, uri); // info will be repeated line per line: CTX->mountdir(match[4]); @@ -212,6 +208,41 @@ load_hostfile(const std::string& path) { // convert match[11] and match[12] to unsigned integers. CTX->fs_conf()->uid = std::stoi(match[11]); CTX->fs_conf()->gid = std::stoi(match[12]); + }; + + string line; + while(getline(lf, line)) { + if(line.empty()) + continue; + // if line starts with #, it indicates the end of current FS instance + // Further hosts are not part of the file system instance yet and are + // therefore skipped. The hostfile is ordered, so nothing below this + // line can contain valid hosts + if(line.find(gkfs::client::hostsfile_end_str) != string::npos) + break; + // Skip comment lines (lines starting with #) + if(line[0] == '#') + continue; + + if(line[0] == '+') { + // Adding daemons are reachable, but they are not part of the old + // distribution at client bootstrap. mutate_start contacts them via + // marker parsing in forward_malleability.cpp. + continue; + } + + if(line[0] == '-') { + parse_host_line(gkfs::malleable::strip_marker(line), + removing_hosts); + continue; + } + + parse_host_line(line, hosts); + } + if(hosts.empty() && !removing_hosts.empty()) { + LOG(INFO, + "Hosts file contains no unmarked daemons; bootstrapping from '-' marked daemons for mutate"); + hosts = std::move(removing_hosts); } if(hosts.empty()) { throw runtime_error( diff --git a/src/client/rpc/forward_malleability.cpp b/src/client/rpc/forward_malleability.cpp index 0fedf4aa66f5163df5def1165cd378d902ff7d08..0854ebafc4e22b35ff8753834a06ef13fff8a5c0 100644 --- a/src/client/rpc/forward_malleability.cpp +++ b/src/client/rpc/forward_malleability.cpp @@ -9,7 +9,7 @@ ADA-FS project under the SPPEXA project funded by the DFG. This software was partially supported by the - the European Union’s Horizon 2020 JTI-EuroHPC research and + the European Union's Horizon 2020 JTI-EuroHPC research and innovation programme, by the project ADMIRE (Project ID: 956748, admire-eurohpc.eu) @@ -27,44 +27,217 @@ #include #include #include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +thallium::endpoint +lookup_mutate_endpoint(const std::string& uri) { + constexpr unsigned int max_attempts = 10; + for(unsigned int attempt = 0; attempt < max_attempts; ++attempt) { + try { + return CTX->rpc_engine()->lookup(uri); + } catch(const std::exception& ex) { + if(attempt + 1 == max_attempts) { + throw; + } + LOG(DEBUG, + "Lookup of mutate endpoint '{}' failed (attempt {}/{}): {}", + uri, attempt + 1, max_attempts, ex.what()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + throw std::runtime_error("unreachable endpoint lookup failure"); +} + +std::vector> +mutate_endpoints_from_loaded_hosts() { + std::vector> endpoints; + const auto& targets = CTX->distributor()->locate_directory_metadata(); + endpoints.reserve(targets.size()); + + for(auto target : targets) { + try { + endpoints.emplace_back(std::to_string(target), + CTX->hosts().at(target)); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to get mutate endpoint for host {}: {}", target, + ex.what()); + } + } + + return endpoints; +} + +std::vector> +mutate_endpoints_from_markers(const std::string& hostfile, + bool include_removing) { + std::vector> endpoints; + std::set seen_uris; + auto markers = gkfs::malleable::parse_hostfile_markers(hostfile); + + if(!markers.has_markers()) { + return endpoints; + } + + auto add_entry = [&](const gkfs::malleable::HostEntry& entry) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + return; + } + try { + LOG(DEBUG, "Looking up mutate endpoint '{}'", entry.uri); + endpoints.emplace_back(entry.uri, + lookup_mutate_endpoint(entry.uri)); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to lookup mutate endpoint '{}': {}", entry.uri, + ex.what()); + } + }; + + for(const auto& entry : markers.active) { + add_entry(entry); + } + for(const auto& entry : markers.adding) { + add_entry(entry); + } + if(include_removing) { + for(const auto& entry : markers.removing) { + add_entry(entry); + } + } + + return endpoints; +} + +std::vector> +mutate_removed_endpoints_from_markers(const std::string& hostfile) { + std::vector> endpoints; + std::set seen_uris; + auto markers = gkfs::malleable::parse_hostfile_markers(hostfile); + + for(const auto& entry : markers.removing) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + try { + LOG(DEBUG, "Looking up removed mutate endpoint '{}'", entry.uri); + endpoints.emplace_back(entry.uri, + lookup_mutate_endpoint(entry.uri)); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to lookup removed mutate endpoint '{}': {}", + entry.uri, ex.what()); + } + } + + return endpoints; +} + +std::vector> +mutate_endpoints(const std::string* hostfile, bool include_removing) { + if(hostfile && !hostfile->empty()) { + try { + auto endpoints = + mutate_endpoints_from_markers(*hostfile, include_removing); + if(!endpoints.empty()) { + return endpoints; + } + } catch(const std::exception& ex) { + LOG(DEBUG, "No marker mutate endpoint list from '{}': {}", + *hostfile, ex.what()); + } + } + + if(const auto* hf = std::getenv("LIBGKFS_HOSTS_FILE")) { + try { + auto endpoints = + mutate_endpoints_from_markers(hf, include_removing); + if(!endpoints.empty()) { + return endpoints; + } + } catch(const std::exception& ex) { + LOG(DEBUG, "No marker mutate endpoint list from '{}': {}", hf, + ex.what()); + } + } + + return mutate_endpoints_from_loaded_hosts(); +} + +} // namespace namespace gkfs::malleable::rpc { int -forward_expand_start(int old_server_conf, int new_server_conf) { +forward_mutate_start(int old_server_conf, int new_server_conf, + const std::string& new_hosts_file) { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(&new_hosts_file, true); + + std::size_t expected_targets = targets.size(); + try { + const auto markers = + gkfs::malleable::parse_hostfile_markers(new_hosts_file); + std::set expected_uris; + for(const auto& entry : markers.active) { + expected_uris.insert(entry.uri); + } + for(const auto& entry : markers.adding) { + expected_uris.insert(entry.uri); + } + for(const auto& entry : markers.removing) { + expected_uris.insert(entry.uri); + } + expected_targets = expected_uris.size(); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to validate mutate targets from '{}': {}", + new_hosts_file, ex.what()); + return EINVAL; + } + + if(targets.empty() || targets.size() != expected_targets) { + LOG(ERROR, "No mutate targets found for hosts file '{}'", + new_hosts_file); + LOG(ERROR, "Expected {} mutate targets, resolved {}", expected_targets, + targets.size()); + return EBUSY; + } auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); // define rpc - auto expand_start_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::expand_start); + auto mutate_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_start); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - gkfs::rpc::rpc_expand_start_in_t in; + gkfs::rpc::rpc_mutate_start_in_t in; in.old_server_conf = old_server_conf; in.new_server_conf = new_server_conf; + in.new_hosts_file = new_hosts_file; - waiters.push_back( - expand_start_rpc.on(CTX->hosts().at(target)).async(in)); + waiters.push_back(mutate_rpc.on(endpoint).async(in)); waiter_targets.push_back(target); } catch(const std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); - // Continue to try others? Or fail? - // Margo code continued. + err = EBUSY; } } - LOG(INFO, "{}() send expand_start rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_start rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses @@ -72,7 +245,11 @@ forward_expand_start(int old_server_conf, int new_server_conf) { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); if(out.err != 0) { - err = out.err; + LOG(ERROR, "Mutate start failed on target '{}' with error {}", + waiter_targets[i], out.err); + if(err == 0) { + err = out.err; + } } } catch(const std::exception& ex) { LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], @@ -84,32 +261,30 @@ forward_expand_start(int old_server_conf, int new_server_conf) { } int -forward_expand_status() { +forward_mutate_status() { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(nullptr, true); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto expand_status_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::expand_status); + auto mutate_status_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_status); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - expand_status_rpc.on(CTX->hosts().at(target)).async()); + waiters.push_back(mutate_status_rpc.on(endpoint).async()); waiter_targets.push_back(target); } catch(const std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); } } - LOG(INFO, "{}() send expand_status rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_status rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses @@ -123,8 +298,6 @@ forward_expand_status() { } else if(out.err < 0) { LOG(ERROR, "{}() Host '{}' error.", __func__, waiter_targets[i]); - // Margo logic didn't update global err? "err += - // mercury_out.err" only if > 0. But it logged error. } } catch(const std::exception& ex) { LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], @@ -136,32 +309,30 @@ forward_expand_status() { } int -forward_expand_finalize() { +forward_mutate_finalize() { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + auto targets = mutate_endpoints(nullptr, true); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto expand_finalize_rpc = CTX->rpc_engine()->define( - gkfs::malleable::rpc::tag::expand_finalize); + auto mutate_finalize_rpc = CTX->rpc_engine()->define( + gkfs::malleable::rpc::tag::mutate_finalize); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - expand_finalize_rpc.on(CTX->hosts().at(target)).async()); + waiters.push_back(mutate_finalize_rpc.on(endpoint).async()); waiter_targets.push_back(target); } catch(std::exception& ex) { LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); } } - LOG(INFO, "{}() send expand_finalize rpc to '{}' targets", __func__, + LOG(INFO, "{}() send mutate_finalize rpc to '{}' targets", __func__, waiters.size()); // wait for RPC responses @@ -182,52 +353,66 @@ forward_expand_finalize() { } int -forward_shrink_start(int old_server_conf, int new_server_conf, - const std::string& new_hosts_file) { +forward_mutate_reload(const std::string& hosts_file) { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); + std::vector> targets; + try { + auto markers = gkfs::malleable::parse_hostfile_markers(hosts_file); + std::set seen_uris; + for(const auto& entry : markers.active) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + targets.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); + } + for(const auto& entry : markers.adding) { + if(entry.uri.empty() || !seen_uris.insert(entry.uri).second) { + continue; + } + targets.emplace_back(entry.uri, + CTX->rpc_engine()->lookup(entry.uri)); + } + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to parse final hosts file '{}': {}", hosts_file, + ex.what()); + return EINVAL; + } + if(targets.empty()) { + LOG(ERROR, "Final hosts file '{}' contains no active hosts", + hosts_file); + return EINVAL; + } + auto reload_rpc = + CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::mutate_reload); - auto err = 0; + int err = 0; std::vector waiters; + std::vector waiter_targets; waiters.reserve(targets.size()); - std::vector waiter_targets; waiter_targets.reserve(targets.size()); - - // define rpc - auto shrink_start_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::shrink_start); - - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - - gkfs::rpc::rpc_shrink_start_in_t in; - in.old_server_conf = old_server_conf; - in.new_server_conf = new_server_conf; - in.new_hosts_file = new_hosts_file; - - waiters.push_back( - shrink_start_rpc.on(CTX->hosts().at(target)).async(in)); + gkfs::rpc::rpc_mutate_reload_in_t in; + in.hosts_file = hosts_file; + waiters.push_back(reload_rpc.on(endpoint).async(in)); waiter_targets.push_back(target); } catch(const std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); + LOG(ERROR, "Failed to send reload RPC to host {}: {}", target, + ex.what()); + err = EBUSY; } } - - LOG(INFO, "{}() send shrink_start rpc to '{}' targets", __func__, - waiters.size()); - - // wait for RPC responses - for(std::size_t i = 0; i < waiters.size(); ++i) { + for(size_t i = 0; i < waiters.size(); ++i) { try { - gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); + const auto out = waiters[i].wait().as(); if(out.err != 0) { + LOG(ERROR, "Failed reload on host '{}'", waiter_targets[i]); err = out.err; } } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], - ex.what()); + LOG(ERROR, "Reload RPC wait failed for host {}: {}", + waiter_targets[i], ex.what()); err = EBUSY; } } @@ -235,99 +420,69 @@ forward_shrink_start(int old_server_conf, int new_server_conf, } int -forward_shrink_status() { +forward_mutate_shutdown_removed(const std::string& hostfile) { LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); - - auto err = 0; - std::vector waiters; - waiters.reserve(targets.size()); - std::vector waiter_targets; - waiter_targets.reserve(targets.size()); - auto shrink_status_rpc = - CTX->rpc_engine()->define(gkfs::malleable::rpc::tag::shrink_status); - - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; - try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - shrink_status_rpc.on(CTX->hosts().at(target)).async()); - waiter_targets.push_back(target); - } catch(const std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); - } + std::vector> targets; + try { + targets = mutate_removed_endpoints_from_markers(hostfile); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to parse removed hosts from '{}': {}", hostfile, + ex.what()); + return EINVAL; } - LOG(INFO, "{}() send shrink_status rpc to '{}' targets", __func__, - waiters.size()); - - // wait for RPC responses - for(std::size_t i = 0; i < waiters.size(); ++i) { - try { - gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); - if(out.err > 0) { - LOG(DEBUG, "{}() Host '{}' not done yet.", __func__, - waiter_targets[i]); - err += out.err; - } else if(out.err < 0) { - LOG(ERROR, "{}() Host '{}' error.", __func__, - waiter_targets[i]); - } - } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], - ex.what()); - err = EBUSY; - } + if(targets.empty()) { + LOG(INFO, "{}() no removed daemons to shutdown", __func__); + return 0; } - return err; -} - -int -forward_shrink_finalize() { - LOG(INFO, "{}() enter", __func__); - const auto& targets = CTX->distributor()->locate_directory_metadata(); auto err = 0; std::vector waiters; waiters.reserve(targets.size()); - std::vector waiter_targets; + std::vector waiter_targets; waiter_targets.reserve(targets.size()); - auto shrink_finalize_rpc = CTX->rpc_engine()->define( - gkfs::malleable::rpc::tag::shrink_finalize); + auto mutate_shutdown_rpc = CTX->rpc_engine()->define( + gkfs::malleable::rpc::tag::mutate_shutdown); - for(std::size_t i = 0; i < targets.size(); ++i) { - auto target = targets[i]; + for(const auto& [target, endpoint] : targets) { try { - LOG(DEBUG, "{}() Sending RPC to host: '{}'", __func__, target); - waiters.push_back( - shrink_finalize_rpc.on(CTX->hosts().at(target)).async()); + LOG(DEBUG, "{}() Sending graceful shutdown RPC to host: '{}'", + __func__, target); + waiters.push_back(mutate_shutdown_rpc.on(endpoint).async()); waiter_targets.push_back(target); - } catch(std::exception& ex) { - LOG(ERROR, "Failed to send RPC to host {}: {}", target, ex.what()); + } catch(const std::exception& ex) { + LOG(ERROR, "Failed to send shutdown RPC to host {}: {}", target, + ex.what()); + err = EBUSY; } } - LOG(INFO, "{}() send shrink_finalize rpc to '{}' targets", __func__, - waiters.size()); + LOG(INFO, "{}() sent graceful shutdown RPC to '{}' removed targets", + __func__, waiters.size()); - // wait for RPC responses for(std::size_t i = 0; i < waiters.size(); ++i) { try { gkfs::rpc::rpc_err_out_t out = waiters[i].wait(); if(out.err != 0) { - LOG(ERROR, "Failed finalize on host '{}'", waiter_targets[i]); + LOG(ERROR, "Failed graceful shutdown on host '{}'", + waiter_targets[i]); err = out.err; } } catch(const std::exception& ex) { - LOG(ERROR, "RPC wait failed for target {}: {}", waiter_targets[i], - ex.what()); + LOG(ERROR, "Shutdown RPC wait failed for target {}: {}", + waiter_targets[i], ex.what()); err = EBUSY; } } + return err; } -} // namespace gkfs::malleable::rpc \ No newline at end of file +// Legacy forwarding functions (the headers already provide inline aliases +// above, so these are no-op stubs kept to avoid symbol conflicts on some +// linkers. They delegate to the new unified mutate functions. +// If the header already defines them as inline, these are no-ops. + +} // namespace gkfs::malleable::rpc diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index 7fe5460ac32e2099cb0cca2e47ce41290bf98faf..a18168474aad0f948a6c2ab131239d3b0d357661 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -334,6 +334,7 @@ forward_mk_symlink(const std::string& path, const std::string& target_path) { template std::pair decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, + std::size_t buffer_size, std::vector& decompressed_data) { if(out.err != 0) { throw std::runtime_error("Server returned an error: " + @@ -342,6 +343,12 @@ decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, if(out.dirents_size == 0) { return {nullptr, 0}; } + if(out.dirents_size > buffer_size) { + throw std::runtime_error( + "Server returned dirents payload larger than exposed client buffer: " + + std::to_string(out.dirents_size) + " > " + + std::to_string(buffer_size)); + } if(gkfs::config::rpc::use_dirents_compression) { const unsigned long long uncompressed_size = @@ -380,10 +387,10 @@ decompress_dirents_payload(const OutputType& out, const void* compressed_buffer, inline std::vector> decompress_and_parse_entries_standard( const gkfs::rpc::rpc_get_dirents_out_t& out, - const void* compressed_buffer) { + const void* compressed_buffer, std::size_t buffer_size) { std::vector decompressed_data; auto [payload, payload_size] = decompress_dirents_payload( - out, compressed_buffer, decompressed_data); + out, compressed_buffer, buffer_size, decompressed_data); if(payload_size == 0) { return {}; } @@ -503,8 +510,8 @@ forward_get_dirents(const string& path) { // Decompress and parse entries // The decompress function expects rpc_get_dirents_out_t // which matches the Thallium RPC output. - auto entries = - decompress_and_parse_entries_standard(out, base_ptr); + auto entries = decompress_and_parse_entries_standard( + out, base_ptr, per_host_buff_size); for(auto& e : entries) { auto type = get<1>(e); gkfs::filemap::FileType ftype = @@ -835,7 +842,7 @@ forward_get_dirents_single(const string& path, int server, } auto current_entries = gkfs::rpc::decompress_and_parse_entries( - out, large_buffer.data()); + out, large_buffer.data(), buffer_size); if(current_entries.empty()) { return make_pair(0, std::move(all_entries)); diff --git a/src/client/rpc/forward_metadata_proxy.cpp b/src/client/rpc/forward_metadata_proxy.cpp index b4148db723879d638090ca2d9865e319bd789db6..16900d0eab382b160690ce7d83eac1a1ada43740 100644 --- a/src/client/rpc/forward_metadata_proxy.cpp +++ b/src/client/rpc/forward_metadata_proxy.cpp @@ -225,7 +225,7 @@ forward_get_dirents_single_proxy_v2(const string& path, int server, try { // Here we still assume the buffer is populated by RMA auto entries_vector = gkfs::rpc::decompress_and_parse_entries( - out, large_buffer.data()); + out, large_buffer.data(), large_buffer.size()); if(entries_vector.empty()) { return make_pair(0, std::move(all_entries)); diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 9aa79fbd493e81d81935a0e90af1235913f30f32..4e9eae88f463363b311c69e7453974dcfe374160 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -44,8 +44,21 @@ set_property(TARGET distributor PROPERTY POSITION_INDEPENDENT_CODE ON) target_sources(distributor PUBLIC ${INCLUDE_DIR}/common/rpc/distributor.hpp + ${INCLUDE_DIR}/common/rpc/random_slicing_distributor.hpp + ${INCLUDE_DIR}/common/rpc/distributor_factory.hpp + ${INCLUDE_DIR}/common/rpc/data_migrator.hpp + ${INCLUDE_DIR}/common/rpc/cutshift_sorted.hpp + ${INCLUDE_DIR}/common/rpc/data_migration_executor.hpp + ${INCLUDE_DIR}/common/rpc/distribution_config.hpp + ${INCLUDE_DIR}/common/env.hpp PRIVATE ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/random_slicing_distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/distributor_factory.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/data_migrator.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/cutshift_sorted.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/data_migration_executor.cpp + ${CMAKE_CURRENT_LIST_DIR}/rpc/distribution_config.cpp ) add_library(statistics STATIC) diff --git a/src/common/malleability_markers.cpp b/src/common/malleability_markers.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7cadbf49b48b5d1fbfe25925ee45be05897e60d4 --- /dev/null +++ b/src/common/malleability_markers.cpp @@ -0,0 +1,302 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace gkfs { +namespace malleable { + +bool +is_marker_line(const std::string& line) { + if(line.empty()) + return false; + return line[0] == MARKER_ADD || line[0] == MARKER_REMOVE; +} + +char +get_marker(const std::string& line) { + if(line.empty()) + return '\0'; + if(line[0] == MARKER_ADD) + return MARKER_ADD; + if(line[0] == MARKER_REMOVE) + return MARKER_REMOVE; + return '\0'; +} + +std::string +strip_marker(const std::string& line) { + if(!is_marker_line(line)) + return line; + + auto first = line.find_first_not_of(" \t", 1); + if(first == std::string::npos) + return ""; + return line.substr(first); +} + +HostfileMarkers +parse_hostfile_markers(const std::string& path) { + HostfileMarkers result; + + std::ifstream file(path); + if(!file.is_open()) { + throw std::runtime_error( + fmt::format("Failed to open hostfile with markers: '{}': {}", + path, strerror(errno))); + } + + std::string line; + while(std::getline(file, line)) { + // Skip empty lines + if(line.empty()) + continue; + + // Trim trailing CR if present (Windows line endings) + if(!line.empty() && line.back() == '\r') { + line.pop_back(); + } + + // Skip comments + if(line[0] == '#') + continue; + + HostEntry entry; + char marker = get_marker(line); + + if(marker != '\0') { + // Has marker prefix + entry.marker = std::string(1, marker); + // Strip marker to get the actual content + std::string content = strip_marker(line); + if(content.empty()) + continue; // Skip empty entries after strip + + // Parse the content: hostname uri field1 ... field12 + std::istringstream iss(content); + if(!(iss >> entry.hostname >> entry.uri)) { + continue; // Skip malformed entries + } + + // Read remaining fields + std::string field; + while(std::getline(iss, field, ' ')) { + if(!field.empty()) { + entry.extra_fields.push_back(field); + } + } + } else { + // No marker - active entry + entry.marker = ""; + std::istringstream iss(line); + if(!(iss >> entry.hostname >> entry.uri)) { + continue; // Skip malformed entries + } + + std::string field; + while(std::getline(iss, field, ' ')) { + if(!field.empty()) { + entry.extra_fields.push_back(field); + } + } + } + + entry.raw_line = line; + + // Categorize by marker + if(marker == MARKER_ADD) { + result.adding.push_back(entry); + } else if(marker == MARKER_REMOVE) { + result.removing.push_back(entry); + } else { + result.active.push_back(entry); + } + } + + return result; +} + +void +write_clean_hostfile(const std::string& path, const HostfileMarkers& markers) { + write_clean_hostfile(path, markers, {}); +} + +std::vector +parse_rs_interval_comments(const std::string& path) { + std::vector intervals; + + std::ifstream file(path); + if(!file.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to open hostfile for RS intervals: '{}': {}", path, + strerror(errno))); + } + + std::string line; + while(std::getline(file, line)) { + if(line.rfind("# GKFS_RS_INTERVAL ", 0) != 0) { + continue; + } + + RandomSlicingIntervalComment interval; + std::istringstream iss( + line.substr(std::string("# GKFS_RS_INTERVAL ").size())); + std::string token; + bool have_host = false; + bool have_start = false; + bool have_end = false; + while(iss >> token) { + const auto pos = token.find('='); + if(pos == std::string::npos) { + continue; + } + const auto key = token.substr(0, pos); + const auto value = token.substr(pos + 1); + if(key == "host") { + interval.host_id = static_cast(std::stoull(value)); + have_host = true; + } else if(key == "start") { + interval.start = std::stod(value); + have_start = true; + } else if(key == "end") { + interval.end = std::stod(value); + have_end = true; + } + } + if(have_host && have_start && have_end && + interval.start < interval.end) { + intervals.push_back(interval); + } + } + + return intervals; +} + +void +write_clean_hostfile( + const std::string& path, const HostfileMarkers& markers, + const std::vector& rs_intervals) { + std::ofstream file(path, std::ios::trunc); + if(!file.is_open()) { + throw std::runtime_error( + fmt::format("Failed to write clean hostfile: '{}': {}", path, + strerror(errno))); + } + + // Write active entries (unchanged) + for(const auto& entry : markers.active) { + file << strip_marker(entry.raw_line) << "\n"; + } + + // Promote adding entries (remove marker prefix) + for(const auto& entry : markers.adding) { + file << strip_marker(entry.raw_line) << "\n"; + } + + // Removing entries are not written (they are dropped) + + for(const auto& interval : rs_intervals) { + file << "# GKFS_RS_INTERVAL host=" << interval.host_id + << " start=" << std::fixed << std::setprecision(9) + << interval.start << " end=" << interval.end << "\n"; + } + + file.close(); +} + +void +write_rs_interval_comments( + const std::string& path, + const std::vector& rs_intervals) { + std::ifstream in(path); + if(!in.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to read hostfile for RS intervals: '{}': {}", path, + strerror(errno))); + } + + std::vector lines; + std::string line; + while(std::getline(in, line)) { + if(line.rfind("# GKFS_RS_INTERVAL ", 0) == 0) { + continue; + } + lines.push_back(line); + } + in.close(); + + // Write to a sibling temporary file and rename it into place. Mutate + // start sends this shared hostfile to every daemon concurrently; opening + // the destination with std::ios::trunc leaves a window in which a peer can + // observe an empty or partially-written hostfile. + const auto temporary_path = + path + ".tmp." + std::to_string(static_cast(getpid())); + std::ofstream out(temporary_path, std::ios::trunc); + if(!out.is_open()) { + throw std::runtime_error(fmt::format( + "Failed to write temporary hostfile for RS intervals: '{}': {}", + temporary_path, strerror(errno))); + } + for(const auto& kept_line : lines) { + out << kept_line << "\n"; + } + for(const auto& interval : rs_intervals) { + out << "# GKFS_RS_INTERVAL host=" << interval.host_id + << " start=" << std::fixed << std::setprecision(9) << interval.start + << " end=" << interval.end << "\n"; + } + out.close(); + if(!out) { + std::remove(temporary_path.c_str()); + throw std::runtime_error(fmt::format( + "Failed to write temporary hostfile for RS intervals: '{}': {}", + temporary_path, strerror(errno))); + } + + if(std::rename(temporary_path.c_str(), path.c_str()) != 0) { + const auto error = errno; + std::remove(temporary_path.c_str()); + throw std::runtime_error(fmt::format( + "Failed to replace hostfile for RS intervals: '{}': {}", path, + strerror(error))); + } +} + +} // namespace malleable +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/cutshift_sorted.cpp b/src/common/rpc/cutshift_sorted.cpp new file mode 100644 index 0000000000000000000000000000000000000000..65cfd2829c146b9ca4f5ef3a61bc8b439aad4606 --- /dev/null +++ b/src/common/rpc/cutshift_sorted.cpp @@ -0,0 +1,496 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/cutshift_sorted.hpp" +#include +#include +#include + +namespace gkfs { +namespace rpc { + +namespace { + +constexpr float epsilon = 1.0e-6f; + +float +interval_size(const Interval& interval) { + return interval.end - interval.start; +} + +void +merge_adjacent_gaps(std::vector& gaps) { + if(gaps.empty()) { + return; + } + + std::sort(gaps.begin(), gaps.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + + std::vector merged; + merged.reserve(gaps.size()); + merged.push_back(gaps.front()); + + for(size_t i = 1; i < gaps.size(); ++i) { + auto& last = merged.back(); + const auto& current = gaps[i]; + if(current.start <= last.end + epsilon) { + last.end = std::max(last.end, current.end); + } else { + merged.push_back(current); + } + } + + gaps = std::move(merged); +} + +void +sort_gaps_largest_first(std::vector& gaps) { + std::sort(gaps.begin(), gaps.end(), + [](const Interval& a, const Interval& b) { + const auto size_a = interval_size(a); + const auto size_b = interval_size(b); + if(std::abs(size_a - size_b) > epsilon) { + return size_a > size_b; + } + return a.start < b.start; + }); +} + +void +sort_intervals_by_start(std::vector& intervals) { + std::sort(intervals.begin(), intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); +} + +void +merge_adjacent_same_owner(std::vector& intervals) { + if(intervals.empty()) { + return; + } + + sort_intervals_by_start(intervals); + + std::vector merged; + merged.reserve(intervals.size()); + merged.push_back(intervals.front()); + + for(size_t i = 1; i < intervals.size(); ++i) { + auto& last = merged.back(); + const auto& current = intervals[i]; + if(last.host_id == current.host_id && + current.start <= last.end + epsilon) { + last.end = std::max(last.end, current.end); + } else if(interval_size(current) > epsilon) { + merged.push_back(current); + } + } + + intervals = std::move(merged); +} + +std::vector +collect_gaps_cutshift_in_place( + std::vector& partitions, + const std::unordered_map& reductions) { + + std::vector gaps; + bool cut_from_beginning = true; + + std::vector partition_order(partitions.size()); + for(size_t i = 0; i < partition_order.size(); ++i) { + partition_order[i] = i; + } + std::sort(partition_order.begin(), partition_order.end(), + [&partitions](auto a, auto b) { + return partitions[a].host_id < partitions[b].host_id; + }); + + for(const auto partition_index : partition_order) { + auto& part = partitions[partition_index]; + auto reduction_it = reductions.find(part.host_id); + if(reduction_it == reductions.end()) { + continue; + } + + auto remaining_reduction = reduction_it->second; + if(remaining_reduction <= epsilon || part.intervals.empty()) { + continue; + } + + std::vector interval_order(part.intervals.size()); + for(size_t i = 0; i < interval_order.size(); ++i) { + interval_order[i] = i; + } + + // CutShift first tries to release complete intervals. Taking smaller + // intervals first maximizes the number of complete gaps before one + // partial split is required. + std::sort(interval_order.begin(), interval_order.end(), + [&part](auto a, auto b) { + const auto size_a = interval_size(part.intervals[a]); + const auto size_b = interval_size(part.intervals[b]); + if(std::abs(size_a - size_b) > epsilon) { + return size_a < size_b; + } + return part.intervals[a].start < part.intervals[b].start; + }); + + std::vector remove_interval(part.intervals.size(), false); + for(const auto interval_index : interval_order) { + if(remaining_reduction <= epsilon) { + break; + } + + const auto& interval = part.intervals[interval_index]; + const auto size = interval_size(interval); + if(size <= epsilon || size > remaining_reduction + epsilon) { + continue; + } + + gaps.push_back({interval.start, interval.end, 0}); + remove_interval[interval_index] = true; + remaining_reduction -= size; + } + + if(remaining_reduction > epsilon) { + auto split_it = std::find_if( + interval_order.rbegin(), interval_order.rend(), + [&part, &remove_interval](auto interval_index) { + return !remove_interval[interval_index] && + interval_size(part.intervals[interval_index]) > + epsilon; + }); + + if(split_it != interval_order.rend()) { + auto& interval = part.intervals[*split_it]; + const auto shrink_amount = + std::min(remaining_reduction, interval_size(interval)); + + if(cut_from_beginning) { + gaps.push_back({interval.start, + interval.start + shrink_amount, 0}); + interval.start += shrink_amount; + } else { + gaps.push_back( + {interval.end - shrink_amount, interval.end, 0}); + interval.end -= shrink_amount; + } + + cut_from_beginning = !cut_from_beginning; + remaining_reduction -= shrink_amount; + } + } + + std::vector kept; + kept.reserve(part.intervals.size()); + for(size_t i = 0; i < part.intervals.size(); ++i) { + if(remove_interval[i] || + interval_size(part.intervals[i]) <= epsilon) { + continue; + } + part.intervals[i].host_id = part.host_id; + kept.push_back(part.intervals[i]); + } + std::sort(kept.begin(), kept.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + part.intervals = std::move(kept); + part.total_capacity = part.coverage(); + } + + merge_adjacent_gaps(gaps); + sort_gaps_largest_first(gaps); + return gaps; +} + +} // namespace + +// CutShift+Sorted algorithm from thesis Chapter 4 +// Simplification: uniform capacity per node (heterogeneous weights are Phase 4) + +std::vector +collect_gaps_cutshift(const std::vector& old_partitions, + const std::unordered_map& reductions) { + auto scratch_partitions = old_partitions; + return collect_gaps_cutshift_in_place(scratch_partitions, reductions); +} + +std::vector +assign_gaps_to_new_nodes(std::vector gaps, + const std::vector& new_hosts, + const std::vector& old_partitions) { + + float new_node_capacity = 0.0f; + for(const auto& gap : gaps) { + new_node_capacity += (gap.end - gap.start); + } + if(!new_hosts.empty()) { + new_node_capacity /= static_cast(new_hosts.size()); + } + + std::vector new_partitions; + size_t gap_idx = 0; + + for(host_t host : new_hosts) { + Partition p; + p.host_id = host; + p.total_capacity = new_node_capacity; + + float needed = new_node_capacity; + while(needed > epsilon && gap_idx < gaps.size()) { + Interval g = gaps[gap_idx]; + float gap_size = interval_size(g); + + if(gap_size <= epsilon) { + gap_idx++; + continue; + } + + if(gap_size <= needed + epsilon) { + // Take the entire gap + g.host_id = host; + p.intervals.push_back(g); + needed -= gap_size; + gap_idx++; + } else { + // Take part of the gap + Interval partial = g; + partial.end = partial.start + needed; + partial.host_id = host; + if(interval_size(partial) > epsilon) { + p.intervals.push_back(partial); + } + // Leave remaining gap for next node + g.start = partial.end; + gaps[gap_idx] = g; + needed = 0.0f; + } + } + + new_partitions.push_back(p); + } + + // Add remaining gaps back (they'll be distributed among old nodes) + // for Phase 1, we discard remaining gaps — they represent + // capacity that was removed from the cluster + + return new_partitions; +} + +std::vector +expand_with_cutshift(std::vector current_partitions, + const std::vector& new_hosts, + float old_total_capacity, float new_total_capacity) { + + if(new_hosts.empty()) + return current_partitions; + + // Compute old node count + unsigned int old_count = + static_cast(current_partitions.size()); + float old_per_node = old_total_capacity > 0.0f + ? 1.0f / static_cast(old_count) + : 0.0f; + float new_per_node = + 1.0f / static_cast(old_count + new_hosts.size()); + + // Compute reductions for each old node + std::unordered_map reductions; + for(const auto& p : current_partitions) { + float reduction = old_per_node - new_per_node; + if(reduction > 0.0f) { + reductions[p.host_id] = reduction; + } + } + + // Shrink old partitions and collect the released intervals. This is the + // CutShift phase: complete intervals are assimilated first; when a split is + // required, split side alternates between beginning and end. + auto gaps = collect_gaps_cutshift_in_place(current_partitions, reductions); + + // Create new partitions for new nodes from gaps + auto new_partitions = + assign_gaps_to_new_nodes(gaps, new_hosts, current_partitions); + + // Finalize old partition capacities after the in-place CutShift shrink. + for(auto& p : current_partitions) { + auto it = reductions.find(p.host_id); + if(it != reductions.end()) { + p.total_capacity = new_per_node; + } + } + + // Append new partitions + current_partitions.insert(current_partitions.end(), new_partitions.begin(), + new_partitions.end()); + + return current_partitions; +} + +std::vector +shrink_with_cutshift(std::vector current_partitions, + const std::vector& removed_hosts) { + + if(removed_hosts.empty()) { + return current_partitions; + } + + std::unordered_set removed(removed_hosts.begin(), + removed_hosts.end()); + + std::vector survivors_old_ids; + survivors_old_ids.reserve(current_partitions.size()); + std::vector evacuated; + float evacuated_capacity = 0.0f; + + for(const auto& partition : current_partitions) { + if(removed.find(partition.host_id) != removed.end()) { + for(auto interval : partition.intervals) { + if(interval_size(interval) <= epsilon) { + continue; + } + interval.host_id = 0; + evacuated_capacity += interval_size(interval); + evacuated.push_back(interval); + } + } else { + survivors_old_ids.push_back(partition.host_id); + } + } + + if(survivors_old_ids.empty()) { + return {}; + } + + std::sort(survivors_old_ids.begin(), survivors_old_ids.end()); + + std::unordered_map old_to_new; + old_to_new.reserve(survivors_old_ids.size()); + for(size_t i = 0; i < survivors_old_ids.size(); ++i) { + old_to_new.emplace(survivors_old_ids[i], static_cast(i)); + } + + std::vector survivors(survivors_old_ids.size()); + for(size_t i = 0; i < survivors_old_ids.size(); ++i) { + survivors[i].host_id = static_cast(i); + } + + for(const auto& partition : current_partitions) { + const auto map_it = old_to_new.find(partition.host_id); + if(map_it == old_to_new.end()) { + continue; + } + + auto& survivor = survivors[map_it->second]; + for(auto interval : partition.intervals) { + if(interval_size(interval) <= epsilon) { + continue; + } + interval.host_id = survivor.host_id; + survivor.intervals.push_back(interval); + } + survivor.total_capacity = survivor.coverage(); + } + + if(evacuated.empty()) { + for(auto& survivor : survivors) { + survivor.total_capacity = survivor.coverage(); + merge_adjacent_same_owner(survivor.intervals); + } + return survivors; + } + + merge_adjacent_gaps(evacuated); + sort_gaps_largest_first(evacuated); + + const auto target_capacity = 1.0f / static_cast(survivors.size()); + std::vector deficits(survivors.size(), 0.0f); + float total_deficit = 0.0f; + for(size_t i = 0; i < survivors.size(); ++i) { + deficits[i] = std::max(0.0f, target_capacity - survivors[i].coverage()); + total_deficit += deficits[i]; + } + + // Floating point/comments may not add up exactly. Scale demands to match + // evacuated capacity, so every removed interval is assigned before removed + // daemons disappear at mutate-finalize time. + if(total_deficit > epsilon && evacuated_capacity > epsilon) { + const auto scale = evacuated_capacity / total_deficit; + for(auto& deficit : deficits) { + deficit *= scale; + } + } + + size_t gap_idx = 0; + for(size_t survivor_idx = 0; + survivor_idx < survivors.size() && gap_idx < evacuated.size(); + ++survivor_idx) { + auto needed = deficits[survivor_idx]; + while(needed > epsilon && gap_idx < evacuated.size()) { + auto gap = evacuated[gap_idx]; + const auto gap_size = interval_size(gap); + if(gap_size <= epsilon) { + ++gap_idx; + continue; + } + + if(gap_size <= needed + epsilon) { + gap.host_id = survivors[survivor_idx].host_id; + survivors[survivor_idx].intervals.push_back(gap); + needed -= gap_size; + ++gap_idx; + } else { + Interval piece = gap; + piece.end = piece.start + needed; + piece.host_id = survivors[survivor_idx].host_id; + survivors[survivor_idx].intervals.push_back(piece); + evacuated[gap_idx].start = piece.end; + needed = 0.0f; + } + } + } + + // If rounding left bytes of evacuated ownership, append them to + // least-loaded survivors. This keeps full coverage and avoids dangling + // ownership on nodes that will disappear after mutate completes. + while(gap_idx < evacuated.size()) { + auto gap = evacuated[gap_idx++]; + if(interval_size(gap) <= epsilon) { + continue; + } + auto target = std::min_element(survivors.begin(), survivors.end(), + [](const auto& a, const auto& b) { + return a.coverage() < b.coverage(); + }); + gap.host_id = target->host_id; + target->intervals.push_back(gap); + } + + for(auto& survivor : survivors) { + merge_adjacent_same_owner(survivor.intervals); + survivor.total_capacity = survivor.coverage(); + } + + return survivors; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/data_migration_executor.cpp b/src/common/rpc/data_migration_executor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..3f1417ab491d9031dc1048d46e62beebafff6aa5 --- /dev/null +++ b/src/common/rpc/data_migration_executor.cpp @@ -0,0 +1,110 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/data_migration_executor.hpp" + +namespace gkfs { +namespace rpc { + +MigrationStatus +DataMigrationExecutor::execute(std::vector& jobs, + MigrationProgressCallback progress, + JobHandler handler) { + size_t total = jobs.size(); + size_t done = 0; + size_t succeeded = 0; + size_t failed = 0; + + if(handler) { + for(size_t i = 0; i < jobs.size(); ++i) { + int ret = handler(jobs[i]); + if(ret == 0) { + ++succeeded; + } else { + ++failed; + } + ++done; + + if(progress) { + progress(done, total); + } + } + } else { + for(size_t i = 0; i < jobs.size(); ++i) { + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); + ++succeeded; + ++done; + + if(progress) { + progress(done, total); + } + } + } + + if(succeeded == total) + return MigrationStatus::Success; + if(succeeded == 0) + return MigrationStatus::AllFailed; + return MigrationStatus::PartialFailure; +} + +MigrationStatus +DataMigrationExecutor::execute_batched(std::vector& jobs, + size_t batch_size, + MigrationProgressCallback progress, + JobHandler handler) { + size_t total = jobs.size(); + size_t done = 0; + size_t succeeded = 0; + size_t failed = 0; + + for(size_t i = 0; i < total; i += batch_size) { + size_t batch_end = std::min(i + batch_size, total); + for(size_t j = i; j < batch_end; ++j) { + if(handler) { + int ret = handler(jobs[j]); + if(ret == 0) { + ++succeeded; + } else { + ++failed; + } + } else { + success_count_.fetch_add(1); + total_bytes_.fetch_add(4096); + ++succeeded; + } + ++done; + + if(progress) { + progress(done, total); + } + } + } + + if(succeeded == total) + return MigrationStatus::Success; + if(succeeded == 0) + return MigrationStatus::AllFailed; + return MigrationStatus::PartialFailure; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/data_migrator.cpp b/src/common/rpc/data_migrator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b750a333c9b5988296a850d52c5329730bb3a33a --- /dev/null +++ b/src/common/rpc/data_migrator.cpp @@ -0,0 +1,185 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/data_migrator.hpp" +#include +#include +#include + +namespace gkfs { +namespace rpc { + +// reuse RandomSlicingDistributor's hashing for consistent host +// lookups +host_t +find_host_for(const std::vector& partitions, const std::string& path, + chunkid_t chnk_id) { + // Build temporary interval index from partitions + std::vector intervals; + for(const auto& p : partitions) { + for(const auto& iv : p.intervals) { + intervals.push_back(iv); + } + } + std::sort(intervals.begin(), intervals.end(), + [](const Interval& a, const Interval& b) { + return a.start < b.start; + }); + + // Hash and PRNG (same as RandomSlicingDistributor::locate_data) + uint64_t h = 14695981039346656037ULL; + for(char c : path) { + h ^= static_cast(static_cast(c)); + h *= 1099511628211ULL; + } + h ^= static_cast(chnk_id) + 0x9e3779b9 + (h << 6) + (h >> 2); + h ^= h >> 33; + h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; + + std::mt19937_64 prng(static_cast(h ^ (h >> 16))); + std::uniform_real_distribution dist(0.0f, 1.0f); + float x = dist(prng); + + auto it = std::upper_bound( + intervals.begin(), intervals.end(), x, + [](float val, const Interval& iv) { return val < iv.start; }); + if(it != intervals.begin()) { + --it; + if(x >= it->start && x < it->end) { + return it->host_id; + } + } + return 0; +} + +std::vector +DataMigrator::compute_migrations(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { + + std::vector jobs; + // sample a fixed set of chunk IDs per file path + // This is an estimation — in practice the MDS knows exact chunk mappings + for(int chunk_id = 0; chunk_id < chunk_sample_size; ++chunk_id) { + // Use a synthetic path for estimation + std::string path = "/sample_file_" + std::to_string(chunk_id); + host_t old_host = find_host_for(old_partitions, path, chunk_id); + host_t new_host = find_host_for(new_partitions, path, chunk_id); + + if(old_host != new_host && old_host != 0) { + jobs.push_back({path, static_cast(chunk_id), old_host, + new_host}); + } + } + return jobs; +} + +DataMigrator::MigrationStats +DataMigrator::get_migration_stats(const std::vector& old_partitions, + const std::vector& new_partitions, + int chunk_sample_size) { + + MigrationStats stats; + stats.jobs = compute_migrations(old_partitions, new_partitions, + chunk_sample_size); + stats.total_chunks = static_cast(chunk_sample_size); + stats.migrating_chunks = stats.jobs.size(); + + for(const auto& job : stats.jobs) { + stats.from_counts[job.source_node]++; + stats.to_counts[job.target_node]++; + } + + stats.migration_ratio = + (stats.total_chunks > 0) + ? static_cast(stats.migrating_chunks) / + static_cast(stats.total_chunks) + : 0.0; + + return stats; +} + +void +DataMigrator::print_migration_summary(const MigrationStats& stats) { + std::cout << "=== Migration Summary ===" << std::endl; + std::cout << "Total chunks sampled: " << stats.total_chunks << std::endl; + std::cout << "Chunks to migrate: " << stats.migrating_chunks + << std::endl; + std::cout << "Migration ratio: " << std::fixed << std::setprecision(2) + << (stats.migration_ratio * 100.0) << "%" << std::endl; + + if(!stats.from_counts.empty()) { + std::cout << "\nHosts losing data:" << std::endl; + for(const auto& [host, count] : stats.from_counts) { + std::cout << " Host " << host << ": " << count << " chunks" + << std::endl; + } + } + + if(!stats.to_counts.empty()) { + std::cout << "\nHosts gaining data:" << std::endl; + for(const auto& [host, count] : stats.to_counts) { + std::cout << " Host " << host << ": " << count << " chunks" + << std::endl; + } + } + std::cout << "=========================" << std::endl; +} + +bool +DataMigrator::needs_migration(const std::vector& old_partitions, + const std::vector& new_partitions) { + + if(old_partitions.size() != new_partitions.size()) + return true; + + for(const auto& old_p : old_partitions) { + // Find corresponding new partition + bool found = false; + for(const auto& new_p : new_partitions) { + if(old_p.host_id == new_p.host_id) { + if(old_p.intervals.size() != new_p.intervals.size()) + return true; + // Check each interval + for(const auto& old_iv : old_p.intervals) { + bool iv_found = false; + for(const auto& new_iv : new_p.intervals) { + if(old_iv.start == new_iv.start && + old_iv.end == new_iv.end) { + iv_found = true; + break; + } + } + if(!iv_found) + return true; + } + found = true; + break; + } + } + if(!found) + return true; // host removed + } + return false; +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/distribution_config.cpp b/src/common/rpc/distribution_config.cpp new file mode 100644 index 0000000000000000000000000000000000000000..520eb90c547b42a832ce89318fbb68f3b0a97aee --- /dev/null +++ b/src/common/rpc/distribution_config.cpp @@ -0,0 +1,45 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/distribution_config.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +std::unique_ptr +create_from_config(const DistributionConfig& config, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { + return create_distributor(config.get_strategy(), localhost, hosts_size, + fwd_host); +} + +DistributionStrategy +read_strategy_from_env() { + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val == nullptr || env_val[0] == '\0') { + return DistributionConfig::default_strategy(); + } + return string_to_strategy(std::string(env_val)); +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/distributor.cpp b/src/common/rpc/distributor.cpp index 8fd771284dc622291f49e91fe3c7561c06b066f9..9d9c0265ad45f83d1b0fca55a33f23c82728f483 100644 --- a/src/common/rpc/distributor.cpp +++ b/src/common/rpc/distributor.cpp @@ -65,6 +65,8 @@ SimpleHashDistributor::hosts_size() const { void SimpleHashDistributor::hosts_size(unsigned int size) { hosts_size_ = size; + all_hosts_ = std::vector(size); + ::iota(all_hosts_.begin(), all_hosts_.end(), 0); } host_t diff --git a/src/common/rpc/distributor_factory.cpp b/src/common/rpc/distributor_factory.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6551962f01e5fcb4ae4976078965311f6001fd3c --- /dev/null +++ b/src/common/rpc/distributor_factory.cpp @@ -0,0 +1,89 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/distributor_factory.hpp" +#include "common/rpc/random_slicing_distributor.hpp" +#include +#include + +namespace gkfs { +namespace rpc { + +const char* +strategy_to_string(DistributionStrategy strategy) { + switch(strategy) { + case DistributionStrategy::SimpleHash: + return "simple_hash"; + case DistributionStrategy::RandomSlicing: + return "random_slicing"; + case DistributionStrategy::LocalOnly: + return "local_only"; + case DistributionStrategy::Forwarder: + return "forwarder"; + } + return "unknown"; // unreachable default +} + +DistributionStrategy +string_to_strategy(const std::string& str) { + // case-insensitive comparison + std::string lower = str; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return std::tolower(c); }); + + if(lower == "simple_hash") + return DistributionStrategy::SimpleHash; + if(lower == "random_slicing") + return DistributionStrategy::RandomSlicing; + if(lower == "local_only") + return DistributionStrategy::LocalOnly; + if(lower == "forwarder") + return DistributionStrategy::Forwarder; + return DistributionStrategy::SimpleHash; // default: fallback to simple hash +} + +std::unique_ptr +create_distributor(DistributionStrategy strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { + + switch(strategy) { + case DistributionStrategy::SimpleHash: + return std::make_unique(localhost, + hosts_size); + case DistributionStrategy::RandomSlicing: + return std::make_unique(localhost, + hosts_size); + case DistributionStrategy::LocalOnly: + return std::make_unique(localhost); + case DistributionStrategy::Forwarder: + return std::make_unique(localhost, fwd_host); + } + return nullptr; // should never happen +} + +std::unique_ptr +create_distributor_from_string(const std::string& strategy, host_t localhost, + unsigned int hosts_size, host_t fwd_host) { + return create_distributor(string_to_strategy(strategy), localhost, + hosts_size, fwd_host); +} + +} // namespace rpc +} // namespace gkfs \ No newline at end of file diff --git a/src/common/rpc/random_slicing_distributor.cpp b/src/common/rpc/random_slicing_distributor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bc2207f77aac77b40ce4c3832a74392c4b17ed21 --- /dev/null +++ b/src/common/rpc/random_slicing_distributor.cpp @@ -0,0 +1,363 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + */ + +#include "common/rpc/random_slicing_distributor.hpp" +#include "common/env.hpp" +#include "common/rpc/cutshift_sorted.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace gkfs { +namespace rpc { + +namespace { + +bool +random_slicing_cutshift_enabled() { + const auto* env_value = std::getenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + if(env_value == nullptr || env_value[0] == '\0') { + return false; + } + + const std::string value(env_value); + return value == "1" || value == "ON" || value == "on" || value == "TRUE" || + value == "true" || value == "YES" || value == "yes"; +} + +} // namespace + +// ===== IntervalIndex implementation ===== + +void +IntervalIndex::build(const std::vector& partitions) { + intervals_.clear(); + intervals_.reserve(partitions.size() * 3); // small per-node average + for(const auto& p : partitions) { + for(const auto& iv : p.intervals) { + intervals_.push_back(iv); + } + } + // Sort by start position + std::sort(intervals_.begin(), intervals_.end(), + [](const Interval& a, const Interval& b) { + return a.start < b.start; + }); +} + +int +IntervalIndex::find_partition(float x) const { + // upper_bound returns first element with start > x; back off one + auto it = std::upper_bound( + intervals_.begin(), intervals_.end(), x, + [](float val, const Interval& iv) { return val < iv.start; }); + if(it == intervals_.begin()) + return -1; + --it; + // Check x is within this interval + if(x >= it->start && x < it->end) { + return static_cast(it - intervals_.begin()); + } + return -1; +} + +host_t +IntervalIndex::find_host(float x) const { + int idx = find_partition(x); + if(idx >= 0) + return intervals_[idx].host_id; + return 0; // fallback +} + +// ===== RandomSlicingDistributor implementation ===== + +uint64_t +RandomSlicingDistributor::hash_seed(const std::string& path, + chunkid_t chnk_id) const { + // Combine path and chunk_id into a 64-bit seed using FNV-1a hash + // FNV-1a is faster than SHA1 and provides adequate distribution + // for random slicing with minstd_rand + uint64_t hash = 14695981039346656037ULL; // FNV offset basis + for(char c : path) { + hash ^= static_cast(static_cast(c)); + hash *= 1099511628211ULL; // FNV prime + } + hash ^= static_cast(chnk_id) + 0x9e3779b9 + (hash << 6) + + (hash >> 2); + hash ^= hash >> 33; + hash *= 0xff51afd7ed558ccdULL; + hash ^= hash >> 33; + return hash; +} + +void +RandomSlicingDistributor::init_partitions_from_hosts() { + partitions_.clear(); + if(all_hosts_.empty()) + return; + + // assume uniform capacity (weight=1) for now. + // Future: read weight from host metadata in RPCData + unsigned int n = static_cast(all_hosts_.size()); + float capacity_per_host = 1.0f / static_cast(n); + + float current_pos = 0.0f; + for(unsigned int i = 0; i < n; ++i) { + Partition p; + p.host_id = all_hosts_[i]; + p.total_capacity = capacity_per_host; + Interval iv; + iv.start = current_pos; + iv.end = current_pos + capacity_per_host; + iv.host_id = all_hosts_[i]; + p.intervals.push_back(iv); + partitions_.push_back(p); + current_pos = iv.end; + } + + // Build interval index for lookup + interval_idx_.build(partitions_); +} + +RandomSlicingDistributor::RandomSlicingDistributor(host_t localhost, + unsigned int hosts_size) + : localhost_(localhost), hosts_size_(hosts_size), + prng_(42) { // fixed seed for determinism + all_hosts_.resize(hosts_size); + for(unsigned int i = 0; i < hosts_size; ++i) { + all_hosts_[i] = i; + } + init_partitions_from_hosts(); +} + +// ===== Distributor interface ===== + +host_t +RandomSlicingDistributor::localhost() const { + return localhost_; +} + +unsigned int +RandomSlicingDistributor::hosts_size() const { + return hosts_size_; +} + +void +RandomSlicingDistributor::hosts_size(unsigned int size) { + hosts_size_ = size; + // hosts_size() setter doesn't auto-reconfigure. + // Caller must call reconfigure() or add_nodes()/remove_nodes() explicitly. +} + +host_t +RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + const int num_copy) const { + // Simplification: return primary host only, same as current + // SimpleHashDistributor. The replication is handled at a higher level by + // the MDS. + uint64_t seed = hash_seed(path, chnk_id); + + // Use mt19937 for better distribution quality (FISHER-YATES recommendation + // #4 from thesis) + std::mt19937_64 prng(static_cast(seed ^ (seed >> 16))); + std::uniform_real_distribution dist(0.0f, 1.0f); + float x = dist(prng); + // x is in [0, 1) + + host_t host = interval_idx_.find_host(x); + // Fallback should never happen if intervals cover [0, 1) + return host; +} + +host_t +RandomSlicingDistributor::locate_data(const std::string& path, + const chunkid_t& chnk_id, + unsigned int hosts_size, + const int num_copy) { + return locate_data(path, chnk_id, num_copy); +} + +host_t +RandomSlicingDistributor::locate_file_metadata(const std::string& path, + const int num_copy) const { + // use same algorithm as locate_data for consistency + return locate_data(path, 0, num_copy); +} + +std::vector +RandomSlicingDistributor::locate_directory_metadata() const { + // distribute directory metadata across all nodes + std::vector result; + result.reserve(all_hosts_.size()); + for(auto h : all_hosts_) { + result.push_back(h); + } + return result; +} + +// ===== Random Slicing-specific methods ===== + +void +RandomSlicingDistributor::reconfigure() { + init_partitions_from_hosts(); +} + +bool +RandomSlicingDistributor::set_intervals( + const std::vector& intervals) { + if(intervals.empty()) { + return false; + } + + std::vector partitions; + partitions.reserve(hosts_size_); + for(auto host : all_hosts_) { + Partition partition; + partition.host_id = host; + partitions.push_back(partition); + } + + for(auto interval : intervals) { + if(interval.start < 0.0f || interval.end > 1.0f || + interval.start >= interval.end || interval.host_id >= hosts_size_) { + return false; + } + interval.host_id = static_cast(interval.host_id); + partitions[interval.host_id].intervals.push_back(interval); + } + + for(auto& partition : partitions) { + std::sort( + partition.intervals.begin(), partition.intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + partition.total_capacity = partition.coverage(); + } + + auto sorted = intervals; + std::sort(sorted.begin(), sorted.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + constexpr float epsilon = 1.0e-5f; + if(std::abs(sorted.front().start) > epsilon || + std::abs(sorted.back().end - 1.0f) > epsilon) { + return false; + } + for(size_t i = 1; i < sorted.size(); ++i) { + if(std::abs(sorted[i].start - sorted[i - 1].end) > epsilon) { + return false; + } + } + + partitions_ = std::move(partitions); + interval_idx_.build(partitions_); + return true; +} + +std::vector +RandomSlicingDistributor::get_intervals() const { + return interval_idx_.intervals(); +} + +void +RandomSlicingDistributor::add_nodes(std::vector new_nodes) { + if(new_nodes.empty()) + return; + + const auto old_host_count = all_hosts_.size(); + + for(auto h : new_nodes) { + all_hosts_.push_back(h); + } + hosts_size_ = static_cast(all_hosts_.size()); + + if(random_slicing_cutshift_enabled() && old_host_count > 0 && + !partitions_.empty()) { + auto updated_partitions = + expand_with_cutshift(partitions_, new_nodes, 1.0f, 1.0f); + if(updated_partitions.size() == all_hosts_.size()) { + partitions_ = std::move(updated_partitions); + interval_idx_.build(partitions_); + return; + } + } + + // Default: full reconfiguration on add. + init_partitions_from_hosts(); +} + +void +RandomSlicingDistributor::remove_nodes(std::vector old_nodes) { + if(old_nodes.empty()) { + return; + } + + if(random_slicing_cutshift_enabled() && + old_nodes.size() < all_hosts_.size() && !partitions_.empty()) { + auto updated_partitions = shrink_with_cutshift(partitions_, old_nodes); + if(!updated_partitions.empty()) { + hosts_size_ = static_cast(updated_partitions.size()); + all_hosts_.clear(); + all_hosts_.reserve(hosts_size_); + for(host_t h = 0; h < hosts_size_; ++h) { + all_hosts_.push_back(h); + } + partitions_ = std::move(updated_partitions); + interval_idx_.build(partitions_); + return; + } + } + + // Remove hosts from all_hosts_ and compact ids: removed daemons disappear + // once mutate is finalized, so the post-shrink layout uses new ranks + // 0..N-1. + for(auto h : old_nodes) { + auto it = std::find(all_hosts_.begin(), all_hosts_.end(), h); + if(it != all_hosts_.end()) { + all_hosts_.erase(it); + } + } + + hosts_size_ = static_cast(all_hosts_.size()); + all_hosts_.clear(); + all_hosts_.reserve(hosts_size_); + for(host_t h = 0; h < hosts_size_; ++h) { + all_hosts_.push_back(h); + } + // Recompute partitions + init_partitions_from_hosts(); + + // chunks that were on removed nodes are now mapped to remaining + // nodes automatically via the new interval table. Migration is handled by + // DataMigrator (Phase 3). +} + +// interval table persistence stubs — intervals are rebuilt from hosts +// on each daemon/proxy startup. This is intentional: random slicing depends on +// the live host list and capacities, so stale interval files would be +// incorrect. +// TODO: implement persistence if needed for Phase 4 (warm restarts) + +} // namespace rpc +} // namespace gkfs diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 1ba881612482b031dac3b62e86c887860d55dcf8..998c67b5f57d6a8df1b8eb984a355948751e4419 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -41,6 +41,7 @@ target_sources( PRIVATE daemon.cpp handler/srv_data.cpp ../common/rpc/rpc_util.cpp + ../common/malleability_markers.cpp util.cpp ops/metadentry.cpp ops/data.cpp diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 17b5a598967570d6a8f5e06d61c417c0e20fb6ac..f9883ee69627ff29454d43c2e6dcbeeb75d5f9b7 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -155,7 +156,7 @@ RocksDBBackend::RocksDBBackend(const std::string& path) { // Enable WAL if requested via environment bool use_wal = gkfs::config::rocksdb::use_write_ahead_log; - char* env_wal = std::getenv("GKFS_DAEMON_ENABLE_WAL"); + const auto* env_wal = std::getenv(gkfs::env::ENABLE_WAL); if(env_wal != nullptr) { use_wal = (std::string(env_wal) == "ON"); } diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index d7802898721d16a5923e8d3ef802151065502573..6680ad76e478fa37d472470f25d93fe0e7315078 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -38,6 +38,7 @@ #include #include +#include #include @@ -53,6 +54,58 @@ FsData::~FsData() { // getter/setter +bool +FsData::expand_on_demand_active() const { + return expand_on_demand_active_; +} + +void +FsData::expand_on_demand_active(bool active) { + expand_on_demand_active_ = active; +} + +unsigned int +FsData::expand_on_demand_old_hosts_size() const { + return expand_on_demand_old_hosts_size_; +} + +void +FsData::expand_on_demand_old_hosts_size(unsigned int hosts_size) { + expand_on_demand_old_hosts_size_ = hosts_size; +} + +uint64_t +FsData::expand_on_demand_old_local_host_id() const { + return expand_on_demand_old_local_host_id_; +} + +void +FsData::expand_on_demand_old_local_host_id(uint64_t host_id) { + expand_on_demand_old_local_host_id_ = host_id; +} + +std::shared_ptr +FsData::expand_on_demand_old_distributor() const { + return expand_on_demand_old_distributor_; +} + +void +FsData::expand_on_demand_old_distributor( + std::shared_ptr distributor) { + expand_on_demand_old_distributor_ = std::move(distributor); +} + +const std::map& +FsData::expand_on_demand_old_rpc_endpoints() const { + return expand_on_demand_old_rpc_endpoints_; +} + +void +FsData::expand_on_demand_old_rpc_endpoints( + std::map endpoints) { + expand_on_demand_old_rpc_endpoints_ = std::move(endpoints); +} + const std::shared_ptr& FsData::spdlogger() const { return spdlogger_; @@ -344,20 +397,52 @@ FsData::maintenance_mode(bool maintenance_mode) { auto err_str = "Critical error: Maintenance mode enabled twice, e.g., due to multiple expand requests. This is not a allowed and should not happen."; spdlogger()->error(err_str); + ABT_mutex_unlock(maintenance_mode_mutex_); throw std::runtime_error(err_str); } maintenance_mode_ = maintenance_mode; ABT_mutex_unlock(maintenance_mode_mutex_); } +bool +FsData::begin_mutate_start(int old_server_conf, int new_server_conf, + const std::string& hosts_file) { + ABT_mutex_lock(maintenance_mode_mutex_); + if(mutate_start_active_) { + const bool duplicate = mutate_old_server_conf_ == old_server_conf && + mutate_new_server_conf_ == new_server_conf && + mutate_hosts_file_ == hosts_file; + ABT_mutex_unlock(maintenance_mode_mutex_); + if(duplicate) { + return true; + } + throw std::runtime_error( + "A different mutate operation is already active"); + } + mutate_start_active_ = true; + mutate_old_server_conf_ = old_server_conf; + mutate_new_server_conf_ = new_server_conf; + mutate_hosts_file_ = hosts_file; + ABT_mutex_unlock(maintenance_mode_mutex_); + return false; +} + +void +FsData::end_mutate_start() { + ABT_mutex_lock(maintenance_mode_mutex_); + mutate_start_active_ = false; + mutate_hosts_file_.clear(); + ABT_mutex_unlock(maintenance_mode_mutex_); +} + bool FsData::redist_running() const { - return redist_running_; + return redist_running_.load(); } void FsData::redist_running(bool redist_running) { - redist_running_ = redist_running; + redist_running_.store(redist_running); } const std::shared_ptr& @@ -382,4 +467,14 @@ FsData::keep_hosts_file(bool keep) { keep_hosts_file_ = keep; } +bool +FsData::expand_mode() const { + return expand_mode_; +} + +void +FsData::expand_mode(bool expand_mode) { + expand_mode_ = expand_mode; +} + } // namespace gkfs::daemon diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index a2fd49a35b90ce509d2e92e358bdceb881ebc3f6..07f0b7801755f94c189e1a25c7d5c88dfc8a1376 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -46,6 +46,7 @@ #include #include #include +#include #include #include #include @@ -60,6 +61,9 @@ #include #include #include +#include +#include +#include #ifdef GKFS_ENABLE_AGIOS #include @@ -91,6 +95,14 @@ using namespace std; namespace fs = std::filesystem; namespace tl = thallium; +// Forward declarations for malleability RPC handlers (from +// srv_malleability.cpp) +namespace gkfs::rpc { +struct rpc_mutate_start_in_t; +struct rpc_migrate_metadata_in_t; +struct rpc_err_out_t; +} // namespace gkfs::rpc + static condition_variable shutdown_please; // handler for shutdown signaling static mutex mtx; // mutex to wait on shutdown conditional variable static bool keep_rootdir = true; @@ -324,18 +336,16 @@ register_server_rpcs(std::shared_ptr engine) { rpc_srv_create_write_inline); // Malleability RPCs - engine->define(gkfs::malleable::rpc::tag::expand_start, - rpc_srv_expand_start); - engine->define(gkfs::malleable::rpc::tag::expand_status, - rpc_srv_expand_status); - engine->define(gkfs::malleable::rpc::tag::expand_finalize, - rpc_srv_expand_finalize); - engine->define(gkfs::malleable::rpc::tag::shrink_start, - rpc_srv_shrink_start); - engine->define(gkfs::malleable::rpc::tag::shrink_status, - rpc_srv_shrink_status); - engine->define(gkfs::malleable::rpc::tag::shrink_finalize, - rpc_srv_shrink_finalize); + engine->define(gkfs::malleable::rpc::tag::mutate_start, + rpc_srv_mutate_start); + engine->define(gkfs::malleable::rpc::tag::mutate_status, + rpc_srv_mutate_status); + engine->define(gkfs::malleable::rpc::tag::mutate_finalize, + rpc_srv_mutate_finalize); + engine->define(gkfs::malleable::rpc::tag::mutate_reload, + rpc_srv_mutate_reload); + engine->define(gkfs::malleable::rpc::tag::mutate_shutdown, + rpc_srv_mutate_shutdown); engine->define(gkfs::malleable::rpc::tag::migrate_metadata, rpc_srv_migrate_metadata); } @@ -382,16 +392,7 @@ init_rpc_server() { */ void register_client_rpcs(std::shared_ptr engine) { - // Commented out - /* - RPC_DATA->rpc_client_ids().migrate_metadata_id = - MARGO_REGISTER(mid, gkfs::malleable::rpc::tag::migrate_metadata, - rpc_migrate_metadata_in_t, rpc_err_out_t, NULL); - // this is just a write - RPC_DATA->rpc_client_ids().migrate_data_id = - MARGO_REGISTER(mid, gkfs::rpc::tag::write, rpc_write_data_in_t, - rpc_data_out_t, NULL); - */ + (void) engine; } /** @@ -482,7 +483,7 @@ init_environment() { // Check for environment variables for configuration gkfs::config::metadata::use_inline_data = - gkfs::env::get_var(gkfs::env::DAEMON_USE_INLINE_DATA, + gkfs::env::get_var(gkfs::env::USE_INLINE_DATA, gkfs::config::metadata::use_inline_data ? "ON" : "OFF") == "ON"; @@ -492,7 +493,7 @@ init_environment() { ? "ON" : "OFF") == "ON"; gkfs::config::metadata::create_check_parents = - gkfs::env::get_var(gkfs::env::DAEMON_CREATE_CHECK_PARENTS, + gkfs::env::get_var(gkfs::env::CREATE_CHECK_PARENTS, gkfs::config::metadata::create_check_parents ? "ON" : "OFF") == "ON"; @@ -502,18 +503,22 @@ init_environment() { ? "ON" : "OFF") == "ON"; gkfs::config::metadata::symlink_support = - gkfs::env::get_var(gkfs::env::DAEMON_SYMLINK_SUPPORT, + gkfs::env::get_var(gkfs::env::SYMLINK_SUPPORT, gkfs::config::metadata::symlink_support ? "ON" : "OFF") == "ON"; gkfs::config::metadata::rename_support = - gkfs::env::get_var(gkfs::env::DAEMON_RENAME_SUPPORT, + gkfs::env::get_var(gkfs::env::RENAME_SUPPORT, gkfs::config::metadata::rename_support ? "ON" : "OFF") == "ON"; GKFS_DATA->keep_hosts_file( gkfs::env::get_var(gkfs::env::KEEP_HOSTS_FILE, "OFF") == "ON"); + // Read GKFS_DAEMON_EXPAND env var for marker-based hostfile + GKFS_DATA->expand_mode( + gkfs::env::get_var(gkfs::env::DAEMON_EXPAND_MODE, "OFF") == "ON"); + GKFS_DATA->spdlogger()->info( "{}() Inline data: {} / Dirents compression: {} / Create check parents: {} / Create exist check: {} / Symlink support: {} / Rename support: {} / Keep hosts file: {}", __func__, gkfs::config::metadata::use_inline_data, @@ -617,9 +622,12 @@ init_environment() { throw runtime_error("Failed to write root metadentry to KV store: "s + e.what()); } - // setup hostfile to let clients know that a daemon is running on this host + + // Publish the hostfile entry before distributor initialization. The + // distributor may read random-slicing metadata from this file, and fresh + // workspaces do not contain it yet. if(!GKFS_DATA->hosts_file().empty()) { - gkfs::utils::populate_hosts_file(); + gkfs::utils::populate_hosts_file(GKFS_DATA->expand_mode()); } // Init margo client @@ -638,8 +646,43 @@ init_environment() { GKFS_DATA->spdlogger()->debug("{}() Initializing Distributor ... ", __func__); try { - auto distributor = std::make_shared(); - RPC_DATA->distributor(distributor); + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + GKFS_DATA->spdlogger()->info("{}() Distribution strategy: '{}'", + __func__, config.get_strategy_string()); + + // Use the factory to create the distributor + auto distributor = create_from_config( + config, RPC_DATA->local_host_id(), RPC_DATA->hosts_size(), + 0); // fwd_host not used for daemon + if(!distributor) { + throw std::runtime_error("Failed to create distributor"); + } + if(config.is_random_slicing() && !GKFS_DATA->hosts_file().empty()) { + auto rs = dynamic_cast( + distributor.get()); + auto comments = gkfs::malleable::parse_rs_interval_comments( + GKFS_DATA->hosts_file()); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + if(rs && !intervals.empty() && rs->set_intervals(intervals)) { + GKFS_DATA->spdlogger()->info( + "{}() Loaded {} random-slicing intervals from hostfile", + __func__, intervals.size()); + } + } + RPC_DATA->distributor(std::move(distributor)); } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( "{}() Failed to initialize Distributor: {}", __func__, @@ -751,6 +794,15 @@ shutdown_handler(int dummy) { shutdown_please.notify_all(); } +namespace gkfs::daemon { + +void +request_shutdown() { + shutdown_please.notify_all(); +} + +} // namespace gkfs::daemon + /** * @brief Initializes the daemon logging environment. * @internal diff --git a/src/daemon/handler/srv_data.cpp b/src/daemon/handler/srv_data.cpp index aebe546bfec75a293784ee6c0388cf16b90752d6..536ae4a0a5dd092ecdf2f5b44df1a5320ad6b612 100644 --- a/src/daemon/handler/srv_data.cpp +++ b/src/daemon/handler/srv_data.cpp @@ -103,581 +103,695 @@ using namespace std; void rpc_srv_write(const std::shared_ptr& engine, const tl::request& req, const gkfs::rpc::rpc_write_data_in_t& in) { - gkfs::rpc:: - run_rpc_handler(req, in, - [&engine, &req](const gkfs::rpc:: - rpc_write_data_in_t& - in, - gkfs::rpc::rpc_data_out_t& out) { - /* - * 1. Setup - */ - out.err = EIO; - out.io_size = 0; - - size_t bulk_size = - in.bulk_handle - .size(); - - GKFS_DATA - ->spdlogger() - ->debug("{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", - __func__, - in.path, - in.chunk_start, - in.chunk_end, - in.chunk_n, - in.total_chunk_size, - bulk_size, - in.offset); - - std::vector write_ops_vect = - gkfs::rpc::decompress_bitset( - in.wbitset); - - // Calculate the - // number of chunks - // hashing to this - // host - uint64_t - host_chunk_n = + gkfs:: + rpc:: + run_rpc_handler(req, in, + [&engine, + &req](const gkfs::rpc:: + rpc_write_data_in_t& + in, + gkfs::rpc::rpc_data_out_t& + out) { + /* + * 1. Setup + */ + out.err = + EIO; + out.io_size = 0; - for(uint64_t chnk_id_file = - in.chunk_start; - chnk_id_file <= - in.chunk_end; - chnk_id_file++) { - if(gkfs::rpc::get_bitset( - write_ops_vect, - chnk_id_file - - in.chunk_start)) { - host_chunk_n++; - } - } - - GKFS_DATA - ->spdlogger() - ->debug("{}() host_chunk_n {}", - __func__, - host_chunk_n); -#ifdef GKFS_ENABLE_AGIOS - int* data; - ABT_eventual eventual = - ABT_EVENTUAL_NULL; - - /* creating eventual - */ - ABT_eventual_create( - sizeof(int64_t), - &eventual); - - unsigned long long int request_id = - generate_unique_id(); - char* agios_path = const_cast< - char*>( - in.path.c_str()); - - // We should call - // AGIOS before - // chunking (as that - // is an internal way - // to handle the - // requests) - if(!agios_add_request( - agios_path, - AGIOS_WRITE, - in.offset, - in.total_chunk_size, - request_id, - AGIOS_SERVER_ID_IGNORE, - agios_eventual_callback, - eventual)) { - GKFS_DATA - ->spdlogger() - ->error("{}() Failed to send request to AGIOS", - __func__); - } else { - GKFS_DATA - ->spdlogger() - ->debug("{}() request {} was sent to AGIOS", - __func__, - request_id); - } - - /* Block until the - * eventual is - * signaled */ - ABT_eventual_wait( - eventual, - reinterpret_cast< - void**>( - &data)); - - unsigned long long int - result = - *data; - GKFS_DATA - ->spdlogger() - ->debug("{}() request {} was unblocked (offset = {})!", - __func__, - result, - in.offset); - - ABT_eventual_free( - &eventual); - - // Let AGIOS knows it - // can release the - // request, as it is - // completed - if(!agios_release_request( - agios_path, - AGIOS_WRITE, - in.total_chunk_size, - in.offset)) { - GKFS_DATA - ->spdlogger() - ->error("{}() Failed to release request from AGIOS", - __func__); - } -#endif + size_t bulk_size = + in.bulk_handle + .size(); - /* - * 2. Set up buffers - * for pull bulk - * transfers - */ - - // Allocate memory - // for bulk transfer - // using vector - std::vector bulk_buf( - in.total_chunk_size); - - // Expose the local - // buffer - std::vector> - segments; - segments.emplace_back( - bulk_buf.data(), - bulk_buf.size()); - - tl::bulk local_bulk = engine->expose( - segments, - tl::bulk_mode:: - write_only); - - auto const host_id = - in.host_id; - [[maybe_unused]] auto const - host_size = - in.host_size; - - // Use string - // directly chnk_ids - // used by this host - vector chnk_ids_host( - host_chunk_n); - // counter to track - // how many chunks - // have been assigned - auto chnk_id_curr = - static_cast< - uint64_t>( - 0); - // chnk sizes per - // chunk for this - // host - vector chnk_sizes( - host_chunk_n); - // how much size is - // left to assign - // chunks for writing - auto chnk_size_left_host = - in.total_chunk_size; - // temporary - // traveling pointer - char* chnk_ptr = - bulk_buf.data(); - - // temporary - // variables - auto transfer_size = - (bulk_size <= - gkfs::config:: - rpc::chunksize) - ? bulk_size - : gkfs::config:: - rpc::chunksize; - uint64_t - origin_offset; - uint64_t - local_offset; - // object for - // asynchronous disk - // IO - gkfs::data::ChunkWriteOperation - chunk_op{ - in.path, - host_chunk_n}; - - /* - * 3. Calculate chunk - * sizes that - * correspond to this - * host, transfer - * data, and start - * tasks to write to - * disk - */ - // Start to look for - // a chunk that - // hashes to this - // host with the - // first chunk in the - // buffer - for(auto chnk_id_file = - in.chunk_start; - chnk_id_file <= - in.chunk_end && - chnk_id_curr < - host_chunk_n; - chnk_id_file++) { - // Continue if - // chunk does not - // hash to this - // host - - if(!(gkfs::rpc::get_bitset( - write_ops_vect, - chnk_id_file - - in.chunk_start))) { GKFS_DATA ->spdlogger() - ->trace("{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", + ->debug("{}() path: '{}' chunk_start '{}' chunk_end '{}' chunk_n '{}' total_chunk_size '{}' bulk_size: '{}' offset: '{}'", __func__, - chnk_id_file, - host_id, - chnk_id_curr); - continue; - } - - if(GKFS_DATA - ->enable_chunkstats()) { - GKFS_DATA - ->stats() - ->add_write( in.path, - chnk_id_file); - } - - chnk_ids_host[chnk_id_curr] = - chnk_id_file; // save this id to host chunk list - // offset case. - // Only relevant - // in the first - // iteration of - // the loop and - // if the chunk - // hashes to this - // host - if(chnk_id_file == - in.chunk_start && - in.offset > - 0) { - // if only 1 - // destination - // and 1 - // chunk - // (small - // write) the - // transfer_size - // == - // bulk_size - size_t offset_transfer_size = + in.chunk_start, + in.chunk_end, + in.chunk_n, + in.total_chunk_size, + bulk_size, + in.offset); + + std::vector write_ops_vect = + gkfs::rpc::decompress_bitset( + in.wbitset); + + // Calculate + // the number + // of chunks + // hashing to + // this host + uint64_t host_chunk_n = 0; - if(in.offset + - bulk_size <= - gkfs::config:: - rpc::chunksize) - offset_transfer_size = - bulk_size; - else - offset_transfer_size = static_cast< - size_t>( - gkfs::config:: - rpc::chunksize - - in.offset); - - // PULL - // transfer - try { - size_t current_buf_offset = - chnk_ptr - - bulk_buf.data(); - - local_bulk( - current_buf_offset, - offset_transfer_size) - << in.bulk_handle - .on(req.get_endpoint())( - 0, - offset_transfer_size); - - } catch(const std::exception& - e) { + for(uint64_t chnk_id_file = + in.chunk_start; + chnk_id_file <= + in.chunk_end; + chnk_id_file++) { + if(gkfs::rpc::get_bitset( + write_ops_vect, + chnk_id_file - + in.chunk_start)) { + host_chunk_n++; + } + } + + GKFS_DATA + ->spdlogger() + ->debug("{}() host_chunk_n {}", + __func__, + host_chunk_n); + +#ifdef GKFS_ENABLE_AGIOS + int* data; + ABT_eventual eventual = + ABT_EVENTUAL_NULL; + + /* creating + * eventual + */ + ABT_eventual_create( + sizeof(int64_t), + &eventual); + + unsigned long long int request_id = + generate_unique_id(); + char* agios_path = const_cast< + char*>( + in.path.c_str()); + + // We should + // call AGIOS + // before + // chunking + // (as that + // is an + // internal + // way to + // handle the + // requests) + if(!agios_add_request( + agios_path, + AGIOS_WRITE, + in.offset, + in.total_chunk_size, + request_id, + AGIOS_SERVER_ID_IGNORE, + agios_eventual_callback, + eventual)) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to send request to AGIOS", + __func__); + } else { GKFS_DATA ->spdlogger() - ->error("{}() Failed to pull data from client for chunk {} (startchunk {}; endchunk {})", + ->debug("{}() request {} was sent to AGIOS", __func__, - chnk_id_file, - in.chunk_start, - in.chunk_end - - 1); - out.err = - EBUSY; - return; + request_id); } - chnk_sizes[chnk_id_curr] = - offset_transfer_size; - chnk_ptr += - offset_transfer_size; - chnk_size_left_host -= - offset_transfer_size; - } else { - local_offset = - in.total_chunk_size - - chnk_size_left_host; - // origin - // offset of - // a chunk is - // dependent - // on a given - // offset in - // a write - // operation - if(in.offset > - 0) - origin_offset = - (gkfs::config:: - rpc::chunksize - - in.offset) + - ((chnk_id_file - - in.chunk_start) - - 1) * gkfs::config::rpc:: - chunksize; - else - origin_offset = - (chnk_id_file - - in.chunk_start) * - gkfs::config:: - rpc::chunksize; - // last chunk - // might have - // different - // transfer_size - if(chnk_id_curr == - in.chunk_n - - 1) - transfer_size = - chnk_size_left_host; + /* Block + * until the + * eventual + * is + * signaled + */ + ABT_eventual_wait( + eventual, + reinterpret_cast< + void**>( + &data)); + + unsigned long long int + result = + *data; GKFS_DATA ->spdlogger() - ->trace("{}() BULK_TRANSFER_PULL hostid {} file {} chnkid {} total_Csize {} Csize_left {} origin offset {} local offset {} transfersize {}", + ->debug("{}() request {} was unblocked (offset = {})!", __func__, - host_id, - in.path, - chnk_id_file, - in.total_chunk_size, - chnk_size_left_host, - origin_offset, - local_offset, - transfer_size); - - // RDMA PULL - try { - // margo: - // PULL, - // in.bulk, - // origin_offset, - // bulk_handle, - // local_offset, - // transfer_size - local_bulk( - local_offset, - transfer_size) - << in.bulk_handle - .on(req.get_endpoint())( - origin_offset, - transfer_size); - } catch(const std::exception& - e) { + result, + in.offset); + + ABT_eventual_free( + &eventual); + + // Let AGIOS + // knows it + // can + // release + // the + // request, + // as it is + // completed + if(!agios_release_request( + agios_path, + AGIOS_WRITE, + in.total_chunk_size, + in.offset)) { GKFS_DATA ->spdlogger() - ->error("{}() Failed to pull data from client. file {} chunk {} (startchunk {}; endchunk {}). Error: {}", - __func__, - in.path, - chnk_id_file, - in.chunk_start, - (in.chunk_end - - 1), - e.what()); - out.err = - EBUSY; - return; + ->error("{}() Failed to release request from AGIOS", + __func__); } +#endif - chnk_sizes[chnk_id_curr] = - transfer_size; - chnk_ptr += - transfer_size; - chnk_size_left_host -= - transfer_size; - } - try { - // start - // tasklet - // for + /* + * 2. Set up + * buffers + * for pull + * bulk + * transfers + */ + + // Allocate + // memory for + // bulk + // transfer + // using + // vector + std::vector bulk_buf( + in.total_chunk_size); + + // Expose the + // local + // buffer + std::vector> + segments; + segments.emplace_back( + bulk_buf.data(), + bulk_buf.size()); + + tl::bulk local_bulk = engine->expose( + segments, + tl::bulk_mode:: + write_only); + + auto const host_id = + in.host_id; + [[maybe_unused]] auto const + host_size = + in.host_size; + + // Use string + // directly + // chnk_ids + // used by + // this host + vector chnk_ids_host( + host_chunk_n); + // counter to + // track how + // many + // chunks + // have been + // assigned + auto chnk_id_curr = static_cast< + uint64_t>( + 0); + // chnk sizes + // per chunk + // for this + // host + vector chnk_sizes( + host_chunk_n); + // how much + // size is + // left to + // assign + // chunks for // writing - // chunk + auto chnk_size_left_host = + in.total_chunk_size; + // temporary + // traveling + // pointer + char* chnk_ptr = + bulk_buf.data(); + + // temporary + // variables + auto transfer_size = + (bulk_size <= + gkfs::config:: + rpc::chunksize) + ? bulk_size + : gkfs::config:: + rpc::chunksize; + uint64_t + origin_offset; + uint64_t + local_offset; + // object for + // asynchronous + // disk IO + gkfs::data::ChunkWriteOperation chunk_op{ + in.path, + host_chunk_n}; - if(chnk_id_file == - in.chunk_start && - in.offset > - 0) { - chunk_op.write_nonblock( - chnk_id_curr, - chnk_ids_host - [chnk_id_curr], - bulk_buf.data(), - chnk_sizes - [chnk_id_curr], - in.offset); - } else { - size_t computed_offset = - in.total_chunk_size - - (chnk_size_left_host + - chnk_sizes - [chnk_id_curr]); - // chnk_size_left_host - // was - // just - // decremented. - // so - // previous - // left - // was - // chnk_size_left_host - // + - // size. + /* + * 3. + * Calculate + * chunk + * sizes that + * correspond + * to this + * host, + * transfer + * data, and + * start + * tasks to + * write to + * disk + */ + // Start to + // look for + // a chunk + // that + // hashes to + // this host + // with the + // first + // chunk in + // the buffer + for(auto chnk_id_file = + in.chunk_start; + chnk_id_file <= + in.chunk_end && + chnk_id_curr < + host_chunk_n; + chnk_id_file++) { + // Continue + // if + // chunk + // does + // not + // hash + // to + // this + // host + + if(!(gkfs::rpc::get_bitset( + write_ops_vect, + chnk_id_file - + in.chunk_start))) { + GKFS_DATA + ->spdlogger() + ->trace("{}() chunkid '{}' ignored as it does not match to this host with id '{}'. chnk_id_curr '{}'", + __func__, + chnk_id_file, + host_id, + chnk_id_curr); + continue; + } + + if(GKFS_DATA + ->enable_chunkstats()) { + GKFS_DATA + ->stats() + ->add_write( + in.path, + chnk_id_file); + } + + chnk_ids_host[chnk_id_curr] = + chnk_id_file; // save this id to host chunk list // offset - // = - // total - // - - // (prev_left) - // = - // total - // - - // (left - // + - // size) - // This - // matches - // local_offset. - - chunk_op.write_nonblock( - chnk_id_curr, - chnk_ids_host - [chnk_id_curr], - bulk_buf.data() + - computed_offset, - chnk_sizes - [chnk_id_curr], - 0); + // case. + // Only + // relevant + // in the + // first + // iteration + // of the + // loop + // and if + // the + // chunk + // hashes + // to + // this + // host + if(chnk_id_file == + in.chunk_start && + in.offset > + 0) { + // if + // only + // 1 + // destination + // and + // 1 + // chunk + // (small + // write) + // the + // transfer_size + // == + // bulk_size + size_t offset_transfer_size = + 0; + if(in.offset + + bulk_size <= + gkfs::config:: + rpc::chunksize) + offset_transfer_size = + bulk_size; + else + offset_transfer_size = static_cast< + size_t>( + gkfs::config:: + rpc::chunksize - + in.offset); + + // PULL + // transfer + try { + size_t current_buf_offset = + chnk_ptr - + bulk_buf.data(); + + local_bulk( + current_buf_offset, + offset_transfer_size) + << in.bulk_handle + .on(req.get_endpoint())( + 0, + offset_transfer_size); + + } catch(const std::exception& + e) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to pull data from client for chunk {} (startchunk {}; endchunk {})", + __func__, + chnk_id_file, + in.chunk_start, + in.chunk_end - + 1); + out.err = + EBUSY; + return; + } + + chnk_sizes[chnk_id_curr] = + offset_transfer_size; + chnk_ptr += + offset_transfer_size; + chnk_size_left_host -= + offset_transfer_size; + } else { + local_offset = + in.total_chunk_size - + chnk_size_left_host; + // origin + // offset + // of + // a + // chunk + // is + // dependent + // on + // a + // given + // offset + // in + // a + // write + // operation + if(in.offset > + 0) + origin_offset = + (gkfs::config:: + rpc::chunksize - + in.offset) + + ((chnk_id_file - + in.chunk_start) - + 1) * gkfs::config::rpc:: + chunksize; + else + origin_offset = + (chnk_id_file - + in.chunk_start) * + gkfs::config:: + rpc::chunksize; + // last + // chunk + // might + // have + // different + // transfer_size + if(chnk_id_curr == + in.chunk_n - + 1) + transfer_size = + chnk_size_left_host; + GKFS_DATA + ->spdlogger() + ->trace("{}() BULK_TRANSFER_PULL hostid {} file {} chnkid {} total_Csize {} Csize_left {} origin offset {} local offset {} transfersize {}", + __func__, + host_id, + in.path, + chnk_id_file, + in.total_chunk_size, + chnk_size_left_host, + origin_offset, + local_offset, + transfer_size); + + // RDMA + // PULL + try { + // margo: + // PULL, + // in.bulk, + // origin_offset, + // bulk_handle, + // local_offset, + // transfer_size + local_bulk( + local_offset, + transfer_size) + << in.bulk_handle + .on(req.get_endpoint())( + origin_offset, + transfer_size); + } catch(const std::exception& + e) { + GKFS_DATA + ->spdlogger() + ->error("{}() Failed to pull data from client. file {} chunk {} (startchunk {}; endchunk {}). Error: {}", + __func__, + in.path, + chnk_id_file, + in.chunk_start, + (in.chunk_end - + 1), + e.what()); + out.err = + EBUSY; + return; + } + + chnk_sizes[chnk_id_curr] = + transfer_size; + chnk_ptr += + transfer_size; + chnk_size_left_host -= + transfer_size; + } + const bool partial_chunk_write = + (chnk_id_file == + in.chunk_start && + in.offset > + 0) || + chnk_sizes[chnk_id_curr] < + gkfs::config:: + rpc::chunksize; + if(partial_chunk_write) { + auto mat_err = gkfs::data::expand_on_demand_materialize_for_partial_write( + in.path, + chnk_ids_host + [chnk_id_curr]); + if(mat_err != + 0) { + GKFS_DATA + ->spdlogger() + ->error("{}() expand-on-demand partial-write materialization failed for '{}' chunk '{}': err '{}'", + __func__, + in.path, + chnk_ids_host + [chnk_id_curr], + mat_err); + out.err = + mat_err; + return; + } + } + + try { + // start + // tasklet + // for + // writing + // chunk + + if(chnk_id_file == + in.chunk_start && + in.offset > + 0) { + chunk_op.write_nonblock( + chnk_id_curr, + chnk_ids_host + [chnk_id_curr], + bulk_buf.data(), + chnk_sizes + [chnk_id_curr], + in.offset); + } else { + size_t computed_offset = + in.total_chunk_size - + (chnk_size_left_host + + chnk_sizes + [chnk_id_curr]); + // chnk_size_left_host + // was + // just + // decremented. + // so + // previous + // left + // was + // chnk_size_left_host + // + + // size. + // offset + // = + // total + // - + // (prev_left) + // = + // total + // - + // (left + // + + // size) + // This + // matches + // local_offset. + + chunk_op.write_nonblock( + chnk_id_curr, + chnk_ids_host + [chnk_id_curr], + bulk_buf.data() + + computed_offset, + chnk_sizes + [chnk_id_curr], + 0); + } + + } catch(const gkfs::data::ChunkWriteOpException& + e) { + // This + // exception + // is + // caused + // by + // setup + // of + // Argobots + // variables. + // If + // this + // fails, + // something + // is + // really + // wrong + GKFS_DATA + ->spdlogger() + ->error("{}() while write_nonblock err '{}'", + __func__, + e.what()); + out.err = + EIO; + return; + } + // next + // chunk + chnk_id_curr++; } - - } catch(const gkfs::data::ChunkWriteOpException& - e) { - // This - // exception - // is caused - // by setup - // of - // Argobots - // variables. - // If this - // fails, - // something - // is really - // wrong - GKFS_DATA - ->spdlogger() - ->error("{}() while write_nonblock err '{}'", - __func__, - e.what()); + // Sanity + // check that + // all chunks + // where + // detected + // in + // previous + // loop + // TODO don't + // proceed if + // that + // happens. + if(chnk_size_left_host != + 0) + GKFS_DATA + ->spdlogger() + ->warn("{}() Not all chunks were detected!!! Size left {}", + __func__, + chnk_size_left_host); + /* + * 4. Read + * task + * results + * and + * accumulate + * in + * out.io_size + */ + auto write_result = + chunk_op.wait_for_tasks(); out.err = - EIO; - return; - } - // next chunk - chnk_id_curr++; - } - // Sanity check that - // all chunks where - // detected in - // previous loop - // TODO don't proceed - // if that happens. - if(chnk_size_left_host != - 0) - GKFS_DATA - ->spdlogger() - ->warn("{}() Not all chunks were detected!!! Size left {}", - __func__, - chnk_size_left_host); - /* - * 4. Read task - * results and - * accumulate in - * out.io_size - */ - auto write_result = - chunk_op.wait_for_tasks(); - out.err = - write_result - .first; - out.io_size = - write_result - .second; - - // Sanity check to - // see if all data - // has been written - if(in.total_chunk_size != - out.io_size) { - GKFS_DATA - ->spdlogger() - ->warn("{}() total chunk size {} and out.io_size {} mismatch!", - __func__, - in.total_chunk_size, - out.io_size); - } - - if(GKFS_DATA - ->enable_stats()) { - GKFS_DATA - ->stats() - ->add_value_size( - gkfs::utils::Stats:: - SizeOp::write_size, - bulk_size); - } - }); + write_result + .first; + out.io_size = + write_result + .second; + + // Sanity + // check to + // see if all + // data has + // been + // written + if(in.total_chunk_size != + out.io_size) { + GKFS_DATA + ->spdlogger() + ->warn("{}() total chunk size {} and out.io_size {} mismatch!", + __func__, + in.total_chunk_size, + out.io_size); + } + + if(GKFS_DATA + ->enable_stats()) { + GKFS_DATA + ->stats() + ->add_value_size( + gkfs::utils::Stats:: + SizeOp::write_size, + bulk_size); + } + }); } /** diff --git a/src/daemon/handler/srv_malleability.cpp b/src/daemon/handler/srv_malleability.cpp index cf61ada0afa7fbf7e9412854ce1d53629051ed25..65bff96abf5729dc986532f8487fa8427bb3a6e6 100644 --- a/src/daemon/handler/srv_malleability.cpp +++ b/src/daemon/handler/srv_malleability.cpp @@ -43,6 +43,9 @@ #include #include +#include +#include + extern "C" { #include } @@ -52,23 +55,40 @@ using namespace std; // namespace { void -rpc_srv_expand_start(const tl::request& req, - const gkfs::rpc::rpc_expand_start_in_t& in) { +rpc_srv_mutate_start(const tl::request& req, + const gkfs::rpc::rpc_mutate_start_in_t& in) { gkfs::rpc::rpc_err_out_t out; + bool entered_maintenance = false; + bool duplicate_request = false; + bool registered_mutate = false; GKFS_DATA->spdlogger()->debug( - "{}() Got RPC with old conf '{}' new conf '{}'", __func__, - in.old_server_conf, in.new_server_conf); + "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", + __func__, in.old_server_conf, in.new_server_conf, + in.new_hosts_file); try { - // if maintenance mode is already set, error is thrown -- not allowed + duplicate_request = GKFS_DATA->begin_mutate_start( + in.old_server_conf, in.new_server_conf, in.new_hosts_file); + if(duplicate_request) { + out.err = 0; + gkfs::utils::safe_respond(req, out); + return; + } + registered_mutate = true; GKFS_DATA->maintenance_mode(true); - GKFS_DATA->malleable_manager()->expand_start(in.old_server_conf, - in.new_server_conf); + entered_maintenance = true; + GKFS_DATA->malleable_manager()->mutate_start( + in.old_server_conf, in.new_server_conf, in.new_hosts_file); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to start expansion: '{}' ", + GKFS_DATA->spdlogger()->error("{}() Failed to start mutate: '{}' ", __func__, e.what()); - GKFS_DATA->maintenance_mode(false); + if(entered_maintenance) { + GKFS_DATA->maintenance_mode(false); + } + if(registered_mutate) { + GKFS_DATA->end_mutate_start(); + } out.err = -1; } @@ -78,15 +98,14 @@ rpc_srv_expand_start(const tl::request& req, } void -rpc_srv_expand_status(const tl::request& req) { +rpc_srv_mutate_status(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); try { - // return 1 if redistribution is running, 0 otherwise. out.err = GKFS_DATA->redist_running() ? 1 : 0; } catch(const std::exception& e) { GKFS_DATA->spdlogger()->error( - "{}() Failed to check status for expansion: '{}'", __func__, + "{}() Failed to check status for mutate: '{}'", __func__, e.what()); out.err = -1; } @@ -96,14 +115,16 @@ rpc_srv_expand_status(const tl::request& req) { } void -rpc_srv_expand_finalize(const tl::request& req) { +rpc_srv_mutate_finalize(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); try { GKFS_DATA->maintenance_mode(false); + GKFS_DATA->keep_hosts_file(true); + GKFS_DATA->end_mutate_start(); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to finalize expansion: '{}'", + GKFS_DATA->spdlogger()->error("{}() Failed to finalize mutate: '{}'", __func__, e.what()); out.err = -1; } @@ -114,94 +135,61 @@ rpc_srv_expand_finalize(const tl::request& req) { } void -rpc_srv_shrink_start(const tl::request& req, - const gkfs::rpc::rpc_shrink_start_in_t& in) { +rpc_srv_mutate_reload(const tl::request& req, + const gkfs::rpc::rpc_mutate_reload_in_t& in) { gkfs::rpc::rpc_err_out_t out; - - GKFS_DATA->spdlogger()->debug( - "{}() Got RPC with old conf '{}' new conf '{}' new_hosts_file '{}'", - __func__, in.old_server_conf, in.new_server_conf, - in.new_hosts_file); + GKFS_DATA->spdlogger()->debug("{}() Got RPC for hosts file '{}'", __func__, + in.hosts_file); try { - GKFS_DATA->maintenance_mode(true); - GKFS_DATA->malleable_manager()->shrink_start( - in.old_server_conf, in.new_server_conf, in.new_hosts_file); + GKFS_DATA->malleable_manager()->reload_hosts_file(in.hosts_file); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to start shrink: '{}' ", + GKFS_DATA->spdlogger()->error("{}() Failed to reload hosts file: '{}'", __func__, e.what()); - GKFS_DATA->maintenance_mode(false); out.err = -1; } - - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); gkfs::utils::safe_respond(req, out); } void -rpc_srv_shrink_status(const tl::request& req) { +rpc_srv_mutate_shutdown(const tl::request& req) { gkfs::rpc::rpc_err_out_t out; - GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); - try { - out.err = GKFS_DATA->redist_running() ? 1 : 0; - } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to check status for shrink: '{}'", __func__, - e.what()); - out.err = -1; - } - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); - gkfs::utils::safe_respond(req, out); -} + GKFS_DATA->spdlogger()->info( + "{}() Got graceful shutdown RPC after malleability finalize", + __func__); -void -rpc_srv_shrink_finalize(const tl::request& req) { - gkfs::rpc::rpc_err_out_t out; - GKFS_DATA->spdlogger()->debug("{}() Got RPC ", __func__); try { GKFS_DATA->maintenance_mode(false); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to finalize shrink: '{}'", + GKFS_DATA->spdlogger()->error("{}() Failed to prepare shutdown: '{}'", __func__, e.what()); out.err = -1; } - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); gkfs::utils::safe_respond(req, out); + + if(out.err == 0) { + std::thread([]() { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + gkfs::daemon::request_shutdown(); + }).detach(); + } } void rpc_srv_migrate_metadata(const tl::request& req, const gkfs::rpc::rpc_migrate_metadata_in_t& in) { - gkfs::rpc::rpc_err_out_t out{}; - - GKFS_DATA->spdlogger()->debug("{}() Got RPC with key '{}' value '{}'", - __func__, in.key, in.value); + gkfs::rpc::rpc_err_out_t out; try { - // create metadentry GKFS_DATA->mdb()->put(in.key, in.value); out.err = 0; } catch(const std::exception& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to create KV entry: '{}'", + GKFS_DATA->spdlogger()->error("{}() Failed to write metadata: '{}'", __func__, e.what()); out.err = -1; } - - GKFS_DATA->spdlogger()->debug("{}() Sending output err '{}'", __func__, - out.err); gkfs::utils::safe_respond(req, out); } // } // namespace - -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_start) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_status) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_expand_finalize) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_start) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_status) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_shrink_finalize) -// DEFINE_MARGO_RPC_HANDLER(rpc_srv_migrate_metadata) diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index e7522cc272e601219fa4cc66da36d01d688a755e..6e6891dcab6f6dda6c44d5b3f435a45827856500 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -747,6 +747,8 @@ rpc_srv_get_dirents(const std::shared_ptr& engine, GKFS_DATA->spdlogger()->debug("{}() Got RPC: path '{}' ", __func__, in.path); + const auto client_bulk_size = in.bulk_handle.size(); + std::vector> entries{}; try { entries = gkfs::metadata::get_dirents(in.path); @@ -761,7 +763,8 @@ rpc_srv_get_dirents(const std::shared_ptr& engine, "{}() path '{}' Read database with '{}' entries", __func__, in.path, entries.size()); - get_dirents_helper(engine, req, entries, 0, in.bulk_handle, out); + get_dirents_helper(engine, req, entries, client_bulk_size, in.bulk_handle, + out); if(GKFS_DATA->enable_stats()) { GKFS_DATA->stats()->add_value_iops( diff --git a/src/daemon/malleability/malleable_manager.cpp b/src/daemon/malleability/malleable_manager.cpp index 48b18b0391d0979e19a6514525bda1db7009770e..466812d2f3e127e66a78d6124881b3158e55ab55 100644 --- a/src/daemon/malleability/malleable_manager.cpp +++ b/src/daemon/malleability/malleable_manager.cpp @@ -42,12 +42,21 @@ #include #include +#include +#include +#include +#include +#include #include #include #include #include #include +#include +#include +#include +#include extern "C" { #include @@ -61,6 +70,255 @@ namespace fs = std::filesystem; namespace gkfs::malleable { +static string +host_key(pair host) { + if(auto idx = host.first.rfind("#"); idx != string::npos) { + host.first.erase(idx); + } + return host.first + "\n" + host.second; +} + +static bool +env_truthy(const char* value) { + if(value == nullptr || value[0] == '\0') { + return false; + } + string v(value); + transform(v.begin(), v.end(), v.begin(), + [](unsigned char c) { return static_cast(tolower(c)); }); + return v == "1" || v == "on" || v == "true" || v == "yes"; +} + +static vector +to_interval_comments(const vector& intervals) { + vector comments; + comments.reserve(intervals.size()); + for(const auto& interval : intervals) { + comments.push_back({static_cast(interval.host_id), + static_cast(interval.start), + static_cast(interval.end)}); + } + return comments; +} + +static vector +make_equal_rs_partitions_for_before_hosts( + const vector>& before_hosts, + const vector>& after_hosts) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + vector partitions; + partitions.reserve(before_hosts.size()); + const auto n = static_cast(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + const auto it = after_ids.find(host_key(before_hosts[old_id])); + if(it == after_ids.end()) { + continue; + } + const auto start = static_cast(old_id) / n; + const auto end = static_cast(old_id + 1) / n; + gkfs::rpc::Partition partition; + partition.host_id = it->second; + partition.total_capacity = end - start; + partition.intervals.push_back({start, end, it->second}); + partitions.push_back(partition); + } + return partitions; +} + +static vector +make_equal_rs_partitions_for_before_hosts( + const vector>& before_hosts) { + vector partitions; + partitions.reserve(before_hosts.size()); + const auto n = static_cast(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + const auto start = static_cast(old_id) / n; + const auto end = static_cast(old_id + 1) / n; + gkfs::rpc::Partition partition; + partition.host_id = static_cast(old_id); + partition.total_capacity = end - start; + partition.intervals.push_back( + {start, end, static_cast(old_id)}); + partitions.push_back(partition); + } + return partitions; +} + +static vector +rs_comments_to_old_id_intervals( + const vector& comments, + const vector>& before_hosts) { + vector intervals; + intervals.reserve(comments.size()); + unordered_map seen_hosts; + for(const auto& comment : comments) { + if(comment.host_id >= before_hosts.size()) { + return {}; + } + if(!std::isfinite(comment.start) || !std::isfinite(comment.end) || + comment.start < 0.0 || comment.end > 1.0 || + comment.start >= comment.end) { + return {}; + } + seen_hosts.emplace(static_cast(comment.host_id), + true); + intervals.push_back({static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + + // A raced hostfile may already contain the post-shrink table. Such a table + // can still cover [0, 1), but it is not a valid old layout when it lacks + // one or more old host IDs. Reject it so the caller rebuilds the complete + // pre-shrink layout instead of mixing post-shrink intervals with old IDs. + if(seen_hosts.size() != before_hosts.size()) { + return {}; + } + + auto sorted = intervals; + sort(sorted.begin(), sorted.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + constexpr double epsilon = 1.0e-5; + if(sorted.empty() || std::abs(sorted.front().start) > epsilon || + std::abs(sorted.back().end - 1.0) > epsilon) { + return {}; + } + for(size_t i = 1; i < sorted.size(); ++i) { + if(std::abs(sorted[i].start - sorted[i - 1].end) > epsilon) { + return {}; + } + } + + return intervals; +} + +static vector +remap_compact_survivor_partitions_to_after_ids( + vector partitions, + const vector>& before_hosts, + const vector>& after_hosts, + const vector& removed_old_ids) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + unordered_map removed; + for(auto old_id : removed_old_ids) { + removed.emplace(old_id, true); + } + + vector survivor_old_ids; + survivor_old_ids.reserve(before_hosts.size()); + for(size_t old_id = 0; old_id < before_hosts.size(); ++old_id) { + if(removed.find(static_cast(old_id)) == + removed.end()) { + survivor_old_ids.push_back(static_cast(old_id)); + } + } + + for(auto& partition : partitions) { + if(partition.host_id >= survivor_old_ids.size()) { + return {}; + } + const auto old_id = survivor_old_ids[partition.host_id]; + const auto after_it = after_ids.find(host_key(before_hosts[old_id])); + if(after_it == after_ids.end()) { + return {}; + } + partition.host_id = after_it->second; + for(auto& interval : partition.intervals) { + interval.host_id = partition.host_id; + } + } + return partitions; +} + +static string +rs_intervals_to_string(const vector& intervals) { + ostringstream os; + os << "["; + for(size_t i = 0; i < intervals.size(); ++i) { + if(i != 0) { + os << ", "; + } + os << "{host=" << intervals[i].host_id << ", start=" << fixed + << setprecision(9) << intervals[i].start + << ", end=" << intervals[i].end << "}"; + } + os << "]"; + return os.str(); +} + +static string +rs_partitions_to_string(const vector& partitions) { + ostringstream os; + os << "["; + for(size_t i = 0; i < partitions.size(); ++i) { + if(i != 0) { + os << ", "; + } + os << "{host=" << partitions[i].host_id + << ", intervals=" << rs_intervals_to_string(partitions[i].intervals) + << "}"; + } + os << "]"; + return os.str(); +} + +static vector +remap_existing_rs_comments( + const vector& comments, + const vector>& before_hosts, + const vector>& after_hosts) { + unordered_map after_ids; + for(size_t i = 0; i < after_hosts.size(); ++i) { + after_ids.emplace(host_key(after_hosts[i]), + static_cast(i)); + } + + vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + if(comment.host_id >= before_hosts.size()) { + return {}; + } + const auto it = after_ids.find(host_key(before_hosts[comment.host_id])); + if(it == after_ids.end()) { + return {}; + } + intervals.push_back({static_cast(comment.start), + static_cast(comment.end), it->second}); + } + return intervals; +} + +static vector +intervals_to_partitions(const vector& intervals) { + map grouped; + for(const auto& interval : intervals) { + auto& partition = grouped[interval.host_id]; + partition.host_id = interval.host_id; + partition.intervals.push_back(interval); + } + + vector partitions; + partitions.reserve(grouped.size()); + for(auto& [_, partition] : grouped) { + sort(partition.intervals.begin(), partition.intervals.end(), + [](const auto& a, const auto& b) { return a.start < b.start; }); + partition.total_capacity = partition.coverage(); + partitions.push_back(std::move(partition)); + } + return partitions; +} + // TODO The following three functions are almost identical to the proxy code // They should be moved to a common and shared between the proxy and the daemon vector> @@ -88,6 +346,9 @@ MalleableManager::load_hostfile(const std::string& path) { // It is therefore skipped if(line[0] == '#') continue; + // Skip marker lines (for backward compatibility) + if(gkfs::malleable::is_marker_line(line)) + continue; if(!regex_match(line, match, line_re)) { GKFS_DATA->spdlogger()->error( "{}() Unrecognized line format: [path: '{}', line: '{}']", @@ -111,6 +372,7 @@ MalleableManager::load_hostfile(const std::string& path) { if(idx != string::npos) h.first.erase(idx, h.first.length()); } + return hosts; } @@ -146,7 +408,9 @@ MalleableManager::connect_to_hosts( const auto& local_uri = RPC_DATA->self_addr_str(); bool local_host_found = false; - RPC_DATA->hosts_size(hosts.size()); + std::map new_rpc_endpoints; + std::map new_rpc_endpoints_str; + uint64_t new_local_host_id = std::numeric_limits::max(); vector host_ids(hosts.size()); // populate vector with [0, ..., host_size - 1] ::iota(::begin(host_ids), ::end(host_ids), 0); @@ -168,7 +432,8 @@ MalleableManager::connect_to_hosts( for(uint32_t i = 0; i < 4; i++) { try { auto svr_addr = RPC_DATA->client_rpc_engine()->lookup(uri); - RPC_DATA->rpc_endpoints().insert(make_pair(id, svr_addr)); + new_rpc_endpoints.insert(make_pair(id, svr_addr)); + new_rpc_endpoints_str.insert(make_pair(id, uri)); break; } catch(const std::exception& e) { // still not working after 5 tries. @@ -194,7 +459,7 @@ MalleableManager::connect_to_hosts( : (hostname == local_hostname))) { GKFS_DATA->spdlogger()->debug("{}() Found local host: {} (uri: {})", __func__, hostname, uri); - RPC_DATA->local_host_id(id); + new_local_host_id = id; local_host_found = true; } GKFS_DATA->spdlogger()->debug("{}() Found daemon: id '{}' uri '{}'", @@ -205,7 +470,7 @@ MalleableManager::connect_to_hosts( GKFS_DATA->spdlogger()->info( "{}() Local host '{}' (uri: '{}') not found in new hosts file. This node is being removed.", __func__, local_hostname, local_uri); - RPC_DATA->local_host_id(std::numeric_limits::max()); + new_local_host_id = std::numeric_limits::max(); } else { auto err_msg = fmt::format( "{}() Local host '{}' (uri: '{}') not found in hosts file. This should not happen.", @@ -213,6 +478,11 @@ MalleableManager::connect_to_hosts( throw runtime_error(err_msg); } } + + RPC_DATA->hosts_size(hosts.size()); + RPC_DATA->rpc_endpoints(std::move(new_rpc_endpoints)); + RPC_DATA->rpc_endpoints_str(std::move(new_rpc_endpoints_str)); + RPC_DATA->local_host_id(new_local_host_id); } int @@ -233,22 +503,40 @@ MalleableManager::redistribute_metadata() { if(key == "/") { continue; } - auto dest_id = RPC_DATA->distributor()->locate_file_metadata(key, 0); + const auto dest_id = + RPC_DATA->distributor()->locate_file_metadata(key, 0); + const auto local_id = RPC_DATA->local_host_id(); GKFS_DATA->spdlogger()->trace( - "{}() Migration: key {} and value {}. From host {} to host {}", - __func__, key, value, RPC_DATA->local_host_id(), dest_id); - if(dest_id == RPC_DATA->local_host_id()) { - GKFS_DATA->spdlogger()->trace("{}() SKIP", __func__); - continue; + "{}() Migration: key {} and value {}. From host {} primary target {}", + __func__, key, value, local_id, dest_id); + + bool copied_to_primary = (dest_id == local_id); + for(const auto& [target_id, _] : RPC_DATA->rpc_endpoints()) { + if(target_id == local_id) { + continue; + } + + auto err = gkfs::malleable::rpc::forward_metadata(key, value, + target_id); + if(err != 0) { + GKFS_DATA->spdlogger()->error( + "{}() Failed to copy metadata for key '{}' to host '{}'", + __func__, key, target_id); + migration_err++; + } else if(target_id == dest_id) { + copied_to_primary = true; + } } - auto err = gkfs::malleable::rpc::forward_metadata(key, value, dest_id); - if(err != 0) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to migrate metadata for key '{}'", __func__, - key); - migration_err++; + + // Keep the source metadata. This avoids metadata loss if source and + // target overlap in single-machine tests and keeps mutate conservative: + // after finalize, clients route to the new primary, while stale source + // copies are harmless. + if(!copied_to_primary) { + GKFS_DATA->spdlogger()->warn( + "{}() Primary metadata target '{}' for key '{}' was not copied", + __func__, dest_id, key); } - GKFS_DATA->mdb()->remove(key); count++; if(percent_interval > 0 && count % percent_interval == 0) { GKFS_DATA->spdlogger()->info( @@ -261,127 +549,293 @@ MalleableManager::redistribute_metadata() { return migration_err; } +int +MalleableManager::do_migration(gkfs::rpc::MigrationJob& job) { + // Read chunk data from local storage + std::string chunk_path; + try { + std::string internal_path = job.path; + std::replace(internal_path.begin(), internal_path.end(), '/', ':'); + if(!internal_path.empty() && internal_path.front() == ':') { + internal_path.erase(0, 1); + } + chunk_path = fmt::format("{}/{}/{}", + GKFS_DATA->storage()->get_chunk_directory(), + internal_path, job.chunk_id); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() Failed to build chunk path for {} chnk {}: {}", __func__, + job.path, job.chunk_id, e.what()); + return -1; + } + + // Open and read the chunk file + int fd = open(chunk_path.c_str(), O_RDONLY); + if(fd < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Chunk file not found, skipping: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + struct stat st; + if(fstat(fd, &st) < 0) { + close(fd); + GKFS_DATA->spdlogger()->warn( + "{}() Failed to stat chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + std::vector buf(st.st_size); + ssize_t bytes_read = read(fd, buf.data(), st.st_size); + close(fd); + + if(bytes_read < 0) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to read chunk file: {} (err: {})", __func__, + chunk_path, strerror(errno)); + return -1; + } + + // Forward data to target node using existing RPC infrastructure + auto err = gkfs::malleable::rpc::forward_data( + job.path, buf.data(), static_cast(bytes_read), + static_cast(job.chunk_id), + static_cast(job.target_node)); + + if(err == 0) { + GKFS_DATA->spdlogger()->trace( + "{}() Migrated: {} chnk {} {} bytes to host {}", __func__, + job.path, job.chunk_id, bytes_read, job.target_node); + // Keep the source chunk after a successful copy. This preserves data + // in single-machine/test deployments where several daemons + // intentionally share the same backend root and source/target chunk + // paths alias. Reads after finalize use the new distributor and + // therefore use the migrated target copy; the old copy is harmless + // stale storage. + } else { + GKFS_DATA->spdlogger()->error( + "{}() Failed to migrate data for {} chnk {} to host {}: err {}", + __func__, job.path, job.chunk_id, job.target_node, err); + } + + return err == 0 ? 0 : -1; +} + void -MalleableManager::redistribute_data() { - GKFS_DATA->spdlogger()->info("{}() Starting data redistribution...", - __func__); +MalleableManager::redistribute_data_v2() { + GKFS_DATA->spdlogger()->info( + "{}() Starting data redistribution from local chunk inventory...", + __func__); + + auto distributor = RPC_DATA->distributor(); + if(!distributor) { + GKFS_DATA->spdlogger()->error("{}() No distributor available", + __func__); + return; + } - auto chunk_dir = fs::path(GKFS_DATA->storage()->get_chunk_directory()); - auto dir_iterator = GKFS_DATA->storage()->get_all_chunk_files(); + std::vector jobs; + const auto local_host = RPC_DATA->local_host_id(); + const auto chunk_root = + fs::path(GKFS_DATA->storage()->get_chunk_directory()); - // TODO this can be parallelized, e.g., async chunk I/O - for(const auto& entry : dir_iterator) { + if(!fs::exists(chunk_root)) { + GKFS_DATA->spdlogger()->info( + "{}() Chunk directory '{}' does not exist.", __func__, + chunk_root.native()); + return; + } + + for(const auto& entry : fs::recursive_directory_iterator(chunk_root)) { if(!entry.is_regular_file()) { continue; } - // path under chunkdir as placed in the rootdir - auto rel_chunk_dir = fs::relative(entry, chunk_dir); - // chunk id from this entry used for determining destination - uint64_t chunk_id = stoul(rel_chunk_dir.filename().string()); - // mountdir gekkofs path used for determining destination - auto gkfs_path = rel_chunk_dir.parent_path().string(); - ::replace(gkfs_path.begin(), gkfs_path.end(), ':', '/'); - gkfs_path = "/" + gkfs_path; - auto dest_id = - RPC_DATA->distributor()->locate_data(gkfs_path, chunk_id, 0); - GKFS_DATA->spdlogger()->trace( - "{}() Migrating chunkfile: {} for gkfs file {} chnkid {} destid {}", - __func__, rel_chunk_dir.string(), gkfs_path, chunk_id, dest_id); - if(dest_id == RPC_DATA->local_host_id()) { - GKFS_DATA->spdlogger()->trace("{}() SKIPPERS", __func__); - continue; - } - auto fd = open(entry.path().c_str(), O_RDONLY); - if(fd < 0) { - GKFS_DATA->spdlogger()->error("{}() Failed to open chunkfile: {}", - __func__, entry.path().c_str()); + + const auto chunk_name = entry.path().filename().string(); + if(chunk_name.empty() || + !std::all_of(chunk_name.begin(), chunk_name.end(), ::isdigit)) { continue; } - std::vector buf(entry.file_size()); - auto bytes_read = read(fd, buf.data(), entry.file_size()); - if(bytes_read < 0) { - GKFS_DATA->spdlogger()->error("{}() Failed to read chunkfile: {}", - __func__, entry.path().c_str()); + + auto chunk_id = + static_cast(std::stoul(chunk_name)); + auto rel_parent = + fs::relative(entry.path().parent_path(), chunk_root).string(); + std::replace(rel_parent.begin(), rel_parent.end(), ':', '/'); + auto file_path = fmt::format("/{}", rel_parent); + + auto target = distributor->locate_data(file_path, chunk_id, 0); + if(target == local_host) { continue; } - auto err = gkfs::malleable::rpc::forward_data( - gkfs_path, buf.data(), bytes_read, chunk_id, dest_id); - if(err != 0) { - GKFS_DATA->spdlogger()->error( - "{}() Failed to migrate data for chunkfile: {}", __func__, - entry.path().c_str()); - } - close(fd); - GKFS_DATA->spdlogger()->trace( - "{}() Data migration completed for chunkfile: {}. Removing ...", - __func__, entry.path().c_str()); - // remove file after migration - auto entry_dir = entry.path().parent_path(); - try { - fs::remove(entry); - if(fs::is_empty(entry_dir)) { - fs::remove(entry_dir); - } - } catch(const fs::filesystem_error& e) { - GKFS_DATA->spdlogger()->error("{}() Failed to remove chunkfile: {}", - __func__, entry.path().c_str()); - } - GKFS_DATA->spdlogger()->trace("{}() Done for chunkfile: {}", __func__, - entry.path().c_str()); + + jobs.push_back(gkfs::rpc::MigrationJob{ + file_path, chunk_id, static_cast(local_host), + target}); } - GKFS_DATA->spdlogger()->info("{}() Data redistribution completed.", - __func__); + if(jobs.empty()) { + GKFS_DATA->spdlogger()->info("{}() No local chunks need migration.", + __func__); + return; + } + + GKFS_DATA->spdlogger()->info("{}() Computed {} local chunk migration jobs", + __func__, jobs.size()); + + gkfs::rpc::DataMigrationExecutor executor; + auto status = executor.execute( + jobs, + // Progress callback + [](size_t done, size_t total) { + // Could log progress here if needed + }, + // Job handler + [this](gkfs::rpc::MigrationJob& job) -> int { + return do_migration(job); + }); + + auto stats = executor.get_stats(); + switch(status) { + case gkfs::rpc::MigrationStatus::Success: + GKFS_DATA->spdlogger()->info( + "{}() Data migration completed: all {} jobs succeeded, " + "{} bytes transferred", + __func__, stats.success_count, stats.total_bytes); + break; + case gkfs::rpc::MigrationStatus::PartialFailure: + GKFS_DATA->spdlogger()->warn( + "{}() Data migration completed with failures: {} succeeded, {} " + "failed, {} bytes transferred", + __func__, stats.success_count, stats.fail_count, + stats.total_bytes); + break; + case gkfs::rpc::MigrationStatus::AllFailed: + GKFS_DATA->spdlogger()->error( + "{}() Data migration failed: all {} jobs failed", __func__, + stats.fail_count); + break; + case gkfs::rpc::MigrationStatus::Cancelled: + GKFS_DATA->spdlogger()->warn("{}() Data migration was cancelled", + __func__); + break; + } } void -MalleableManager::expand_abt(void* _arg) { - GKFS_DATA->spdlogger()->info("{}() Starting expansion process...", +MalleableManager::expand_abt_v2(void* _arg) { + auto self = static_cast(_arg); + if(!self) { + GKFS_DATA->spdlogger()->error( + "{}() No MalleableManager pointer available", __func__); + return; + } + GKFS_DATA->spdlogger()->info("{}() Starting v2 expansion process...", __func__); GKFS_DATA->redist_running(true); GKFS_DATA->malleable_manager()->redistribute_metadata(); try { - GKFS_DATA->malleable_manager()->redistribute_data(); + GKFS_DATA->malleable_manager()->redistribute_data_v2(); } catch(const gkfs::data::ChunkStorageException& e) { GKFS_DATA->spdlogger()->error("{}() Failed to redistribute data: '{}'", __func__, e.what()); } GKFS_DATA->redist_running(false); GKFS_DATA->spdlogger()->info( - "{}() Expansion process successfully finished.", __func__); + "{}() V2 expansion process successfully finished.", __func__); } -// PUBLIC - void -MalleableManager::expand_start(int old_server_conf, int new_server_conf) { - auto hosts = read_hosts_file(); - if(hosts.size() != static_cast(new_server_conf)) { - throw runtime_error( - fmt::format("MalleableManager::{}() Something is wrong. " - "Number of hosts in hosts file ({}) " - "does not match new server configuration ({})", - __func__, hosts.size(), new_server_conf)); - } - connect_to_hosts(hosts); - RPC_DATA->distributor()->hosts_size(hosts.size()); - auto abt_err = - ABT_thread_create(RPC_DATA->io_pool(), expand_abt, - ABT_THREAD_ATTR_NULL, nullptr, &redist_thread_); - if(abt_err != ABT_SUCCESS) { - auto err_str = fmt::format( - "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", - __func__, abt_err); - throw runtime_error(err_str); +MalleableManager::expand_on_demand_abt(void* _arg) { + auto self = static_cast(_arg); + if(!self) { + GKFS_DATA->spdlogger()->error( + "{}() No MalleableManager pointer available", __func__); + return; } + GKFS_DATA->spdlogger()->info( + "{}() Starting expand-on-demand process: metadata eager, data lazy.", + __func__); + GKFS_DATA->redist_running(true); + GKFS_DATA->malleable_manager()->redistribute_metadata(); + GKFS_DATA->redist_running(false); + GKFS_DATA->spdlogger()->info( + "{}() Expand-on-demand metadata redistribution finished. Data chunks remain lazy.", + __func__); } +// PUBLIC + void -MalleableManager::shrink_start(int old_server_conf, int new_server_conf, +MalleableManager::mutate_start(int old_server_conf, int new_server_conf, const std::string& new_hosts_file) { - GKFS_DATA->spdlogger()->info("{}() Loading new hosts file '{}' for shrink", - __func__, new_hosts_file); - auto hosts = load_hostfile(new_hosts_file); + // Use the control-plane old server count as old topology size. Added + // daemons start with an empty local distributor, so RPC_DATA alone is not a + // valid old-layout snapshot on those nodes. + old_hosts_size_ = static_cast(old_server_conf); + + GKFS_DATA->spdlogger()->info("{}() Loading new hosts file '{}' for mutate " + "(old={} nodes, new={} nodes)", + __func__, new_hosts_file, old_server_conf, + new_server_conf); + + // MARKER-BASED WORKFLOW: Parse markers to get the "after" state (active + + // adding). load_hostfile() strips all markers, which loses '+' daemons for + // expand. We must use parse_hostfile_markers to include adding nodes in the + // after state. + auto markers = gkfs::malleable::parse_hostfile_markers(new_hosts_file); + vector> hosts; + for(const auto& e : markers.active) { + hosts.emplace_back(e.hostname, e.uri); + } + for(const auto& e : markers.adding) { + hosts.emplace_back(e.hostname, e.uri); + } + sort(hosts.begin(), hosts.end()); + // Remove rootdir suffix + for(auto& h : hosts) { + auto idx = h.first.rfind("#"); + if(idx != string::npos) + h.first.erase(idx, h.first.length()); + } + + // Save the actual pre-mutate map before connect_to_hosts() replaces it + // with the final topology. IDs in this map belong to the old distributor. + auto old_rpc_endpoints = RPC_DATA->rpc_endpoints(); + auto old_local_host_id = RPC_DATA->local_host_id(); + if(old_server_conf < new_server_conf && markers.removing.empty()) { + if(old_rpc_endpoints.size() < old_hosts_size_) { + vector> old_hosts; + old_hosts.reserve(markers.active.size()); + for(const auto& e : markers.active) { + old_hosts.emplace_back(e.hostname, e.uri); + } + sort(old_hosts.begin(), old_hosts.end()); + old_rpc_endpoints.clear(); + old_local_host_id = std::numeric_limits::max(); + for(size_t id = 0; id < old_hosts.size(); ++id) { + if(old_hosts[id].second == RPC_DATA->self_addr_str()) { + old_local_host_id = static_cast(id); + } + try { + old_rpc_endpoints.emplace( + static_cast(id), + RPC_DATA->client_rpc_engine()->lookup( + old_hosts[id].second)); + } catch(const std::exception& e) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to lookup old host '{}': {}", + __func__, old_hosts[id].second, e.what())); + } + } + } + } else { + old_rpc_endpoints.clear(); + old_local_host_id = std::numeric_limits::max(); + } + if(hosts.size() != static_cast(new_server_conf)) { throw runtime_error( fmt::format("MalleableManager::{}() Something is wrong. " @@ -389,12 +843,216 @@ MalleableManager::shrink_start(int old_server_conf, int new_server_conf, "does not match new server configuration ({})", __func__, hosts.size(), new_server_conf)); } + // On mutate, any old node not in the new set will be removed (pass true to + // connect_to_hosts so it handles missing local host gracefully) connect_to_hosts(hosts, true); - RPC_DATA->distributor()->hosts_size(hosts.size()); - auto abt_err = - ABT_thread_create(RPC_DATA->io_pool(), expand_abt, - ABT_THREAD_ATTR_NULL, nullptr, &redist_thread_); + + const bool expand_on_demand = + env_truthy(std::getenv(gkfs::env::EXPAND_ON_DEMAND)); + const bool pure_expand = + old_server_conf < new_server_conf && markers.removing.empty(); + if(expand_on_demand && !pure_expand) { + GKFS_DATA->spdlogger()->warn( + "{}() {} requested but disabled: only pure expand is supported (old={}, new={}, removing={}). Falling back to eager data migration.", + __func__, gkfs::env::EXPAND_ON_DEMAND, old_server_conf, + new_server_conf, markers.removing.size()); + } + + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + + auto distributor = gkfs::rpc::create_from_config( + config, static_cast(RPC_DATA->local_host_id()), + static_cast(hosts.size()), 0); + if(!distributor) { + throw runtime_error("Failed to recreate distributor for mutate"); + } + + auto old_distributor = gkfs::rpc::create_from_config( + config, static_cast(old_local_host_id), + old_hosts_size_, 0); + if(!old_distributor) { + throw runtime_error("Failed to recreate old distributor for mutate"); + } + + if(config.is_random_slicing()) { + vector> before_hosts; + before_hosts.reserve(markers.active.size() + markers.removing.size()); + for(const auto& e : markers.active) { + before_hosts.emplace_back(e.hostname, e.uri); + } + for(const auto& e : markers.removing) { + before_hosts.emplace_back(e.hostname, e.uri); + } + sort(before_hosts.begin(), before_hosts.end()); + for(auto& h : before_hosts) { + auto idx = h.first.rfind("#"); + if(idx != string::npos) { + h.first.erase(idx, h.first.length()); + } + } + + unordered_map before_keys; + for(const auto& host : before_hosts) { + before_keys.emplace(host_key(host), true); + } + + unordered_map after_keys; + for(const auto& host : hosts) { + after_keys.emplace(host_key(host), true); + } + + vector added_ids; + for(size_t i = 0; i < hosts.size(); ++i) { + if(before_keys.find(host_key(hosts[i])) == before_keys.end()) { + added_ids.push_back(static_cast(i)); + } + } + + vector removed_old_ids; + for(size_t i = 0; i < before_hosts.size(); ++i) { + if(after_keys.find(host_key(before_hosts[i])) == after_keys.end()) { + removed_old_ids.push_back(static_cast(i)); + } + } + + auto* rs = dynamic_cast( + distributor.get()); + vector final_intervals; + const bool cutshift = + env_truthy(std::getenv(gkfs::env::RANDOM_SLICING_CUTSHIFT)); + + if(rs && cutshift && (!added_ids.empty() || !removed_old_ids.empty()) && + !before_hosts.empty()) { + auto comments = + gkfs::malleable::parse_rs_interval_comments(new_hosts_file); + vector updated_partitions; + + if(removed_old_ids.empty()) { + auto old_intervals = + comments.empty() + ? vector{} + : remap_existing_rs_comments( + comments, before_hosts, hosts); + updated_partitions = + old_intervals.empty() + ? make_equal_rs_partitions_for_before_hosts( + before_hosts, hosts) + : intervals_to_partitions(old_intervals); + } else { + auto old_intervals = comments.empty() + ? vector{} + : rs_comments_to_old_id_intervals( + comments, before_hosts); + if(!comments.empty() && old_intervals.empty()) { + GKFS_DATA->spdlogger()->warn( + "{}() No usable pre-shrink RS interval comments found; rebuilding equal old layout", + __func__); + } + auto old_partitions = + old_intervals.empty() + ? make_equal_rs_partitions_for_before_hosts( + before_hosts) + : intervals_to_partitions(old_intervals); + + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink old RS intervals: {}", __func__, + rs_intervals_to_string(old_intervals)); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink old partitions: {}", __func__, + rs_partitions_to_string(old_partitions)); + + updated_partitions = gkfs::rpc::shrink_with_cutshift( + old_partitions, removed_old_ids); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink compact partitions before after-id remap: {}", + __func__, rs_partitions_to_string(updated_partitions)); + updated_partitions = + remap_compact_survivor_partitions_to_after_ids( + std::move(updated_partitions), before_hosts, + hosts, removed_old_ids); + GKFS_DATA->spdlogger()->debug( + "{}() CutShift shrink final after-id partitions: {}", + __func__, rs_partitions_to_string(updated_partitions)); + } + + if(!added_ids.empty() && !updated_partitions.empty()) { + updated_partitions = gkfs::rpc::expand_with_cutshift( + updated_partitions, added_ids, 1.0f, 1.0f); + } + + for(const auto& partition : updated_partitions) { + for(const auto& interval : partition.intervals) { + final_intervals.push_back(interval); + } + } + } + + if(rs) { + if(final_intervals.empty()) { + final_intervals = rs->get_intervals(); + } else if(!rs->set_intervals(final_intervals)) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to install CutShift RS intervals; falling back to rebuilt RS layout", + __func__); + final_intervals = rs->get_intervals(); + } + + if(RPC_DATA->local_host_id() == 0) { + try { + gkfs::malleable::write_rs_interval_comments( + new_hosts_file, + to_interval_comments(final_intervals)); + GKFS_DATA->spdlogger()->info( + "{}() Wrote {} random-slicing intervals to hostfile", + __func__, final_intervals.size()); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->warn( + "{}() Failed to write random-slicing intervals to hostfile: {}", + __func__, e.what()); + } + } + } + } + + GKFS_DATA->spdlogger()->info( + "{}() Recreated '{}' distributor for new topology with {} hosts", + __func__, config.get_strategy_string(), hosts.size()); + RPC_DATA->distributor(std::move(distributor)); + + if(expand_on_demand && pure_expand) { + GKFS_DATA->expand_on_demand_active(true); + GKFS_DATA->expand_on_demand_old_hosts_size(old_hosts_size_); + GKFS_DATA->expand_on_demand_old_local_host_id(old_local_host_id); + GKFS_DATA->expand_on_demand_old_distributor(std::move(old_distributor)); + GKFS_DATA->expand_on_demand_old_rpc_endpoints( + std::move(old_rpc_endpoints)); + GKFS_DATA->spdlogger()->info( + "{}() {} active: old_hosts={}, new_hosts={}. Skipping eager data migration.", + __func__, gkfs::env::EXPAND_ON_DEMAND, old_hosts_size_, + hosts.size()); + GKFS_DATA->redist_running(true); + auto abt_err = + ABT_thread_create(RPC_DATA->io_pool(), expand_on_demand_abt, + this, ABT_THREAD_ATTR_NULL, &redist_thread_); + if(abt_err != ABT_SUCCESS) { + GKFS_DATA->redist_running(false); + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", + __func__, abt_err)); + } + return; + } + + // Use v2 pipeline for mutate + GKFS_DATA->redist_running(true); + auto abt_err = ABT_thread_create(RPC_DATA->io_pool(), expand_abt_v2, this, + ABT_THREAD_ATTR_NULL, &redist_thread_); if(abt_err != ABT_SUCCESS) { + GKFS_DATA->redist_running(false); auto err_str = fmt::format( "MalleableManager::{}() Failed to create ABT thread with abt_err '{}'", __func__, abt_err); @@ -402,4 +1060,64 @@ MalleableManager::shrink_start(int old_server_conf, int new_server_conf, } } +void +MalleableManager::reload_hosts_file(const std::string& hosts_file) { + const auto hosts = [&]() { + try { + return load_hostfile(hosts_file); + } catch(const std::exception& e) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to load final hosts file '{}': {}", + __func__, hosts_file, e.what())); + } + }(); + + if(hosts.empty()) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Final hosts file '{}' is empty", + __func__, hosts_file)); + } + + connect_to_hosts(hosts, false); + + gkfs::rpc::DistributionConfig config; + if(const auto* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + + auto distributor = gkfs::rpc::create_from_config( + config, static_cast(RPC_DATA->local_host_id()), + static_cast(hosts.size()), 0); + if(!distributor) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Failed to recreate distributor", + __func__)); + } + + if(config.is_random_slicing()) { + auto comments = gkfs::malleable::parse_rs_interval_comments(hosts_file); + std::vector intervals; + intervals.reserve(comments.size()); + for(const auto& comment : comments) { + intervals.push_back( + {static_cast(comment.start), + static_cast(comment.end), + static_cast(comment.host_id)}); + } + auto* rs = dynamic_cast( + distributor.get()); + if(rs && !intervals.empty() && !rs->set_intervals(intervals)) { + throw runtime_error(fmt::format( + "MalleableManager::{}() Invalid final random-slicing intervals", + __func__)); + } + } + + RPC_DATA->distributor(std::move(distributor)); + GKFS_DATA->spdlogger()->info( + "{}() Reloaded final hosts file '{}' with {} hosts", __func__, + hosts_file, hosts.size()); +} + } // namespace gkfs::malleable diff --git a/src/daemon/ops/data.cpp b/src/daemon/ops/data.cpp index a2397f8e06cd31130b4ee3055795839b62ec1423..d760ffe05d316860f761e0cf8f4ee8a85c72e889 100644 --- a/src/daemon/ops/data.cpp +++ b/src/daemon/ops/data.cpp @@ -42,6 +42,9 @@ #include #include #include +#include +#include +#include #include extern "C" { @@ -52,6 +55,158 @@ using namespace std; namespace gkfs::data { +namespace { + +bool +expand_on_demand_owns_final_missing_candidate(const std::string& path, + uint64_t chunk_id, + uint64_t& old_owner) { + if(!GKFS_DATA->expand_on_demand_active()) { + return false; + } + auto old_dist = GKFS_DATA->expand_on_demand_old_distributor(); + auto final_dist = RPC_DATA->distributor(); + if(!old_dist || !final_dist) { + return false; + } + + const auto local = RPC_DATA->local_host_id(); + const auto final_owner = final_dist->locate_data( + path, static_cast(chunk_id), 0); + if(final_owner != local) { + return false; + } + + old_owner = old_dist->locate_data( + path, static_cast(chunk_id), 0); + return old_owner != GKFS_DATA->expand_on_demand_old_local_host_id() && + old_owner < GKFS_DATA->expand_on_demand_old_hosts_size() && + GKFS_DATA->expand_on_demand_old_rpc_endpoints().find(old_owner) != + GKFS_DATA->expand_on_demand_old_rpc_endpoints().end(); +} + +} // namespace + +std::pair +expand_on_demand_read_remote(const std::string& path, uint64_t chunk_id, + char* buf, size_t size, off64_t offset) { + uint64_t old_owner = 0; + if(!expand_on_demand_owns_final_missing_candidate(path, chunk_id, + old_owner)) { + return {ENOENT, 0}; + } + + std::vector full_chunk(gkfs::config::rpc::chunksize); + std::vector> segments = { + std::make_pair(full_chunk.data(), full_chunk.size())}; + tl::bulk bulk_handle; + try { + bulk_handle = RPC_DATA->client_rpc_engine()->expose( + segments, tl::bulk_mode::write_only); + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() failed to expose fallback read bulk for '{}' chunk '{}': {}", + __func__, path, chunk_id, e.what()); + return {EBUSY, 0}; + } + + gkfs::rpc::rpc_read_data_in_t in{}; + in.path = path; + in.offset = 0; + in.host_id = old_owner; + in.host_size = GKFS_DATA->expand_on_demand_old_hosts_size(); + std::vector bitset(1, 1); + in.wbitset = gkfs::rpc::compress_bitset(bitset); + in.chunk_n = 1; + in.chunk_start = chunk_id; + in.chunk_end = chunk_id; + in.total_chunk_size = full_chunk.size(); + in.bulk_handle = bulk_handle; + + try { + auto read_data = + RPC_DATA->client_rpc_engine()->define(gkfs::rpc::tag::read); + auto out = + read_data + .on(GKFS_DATA->expand_on_demand_old_rpc_endpoints().at( + old_owner))(in) + .as(); + if(out.err != 0 || out.io_size == 0) { + GKFS_DATA->spdlogger()->debug( + "{}() expand-on-demand fallback read '{}' chunk '{}' from old owner '{}': err={} size={}", + __func__, path, chunk_id, old_owner, out.err, out.io_size); + return {out.err, static_cast(out.io_size)}; + } + + try { + GKFS_DATA->storage()->write_chunk( + path, static_cast(chunk_id), + full_chunk.data(), static_cast(out.io_size), 0); + } catch(const gkfs::data::ChunkStorageException& e) { + GKFS_DATA->spdlogger()->error( + "{}() failed to materialize fallback chunk '{}' for '{}': {}", + __func__, chunk_id, path, e.what()); + return {e.code().value(), 0}; + } + const auto available = + offset >= 0 && static_cast(offset) < out.io_size + ? out.io_size - static_cast(offset) + : 0; + const auto copied = std::min(size, available); + if(copied > 0) { + std::memcpy(buf, full_chunk.data() + static_cast(offset), + copied); + } + GKFS_DATA->spdlogger()->debug( + "{}() expand-on-demand fallback read '{}' chunk '{}' from old owner '{}': err={} size={}", + __func__, path, chunk_id, old_owner, out.err, out.io_size); + return {0, static_cast(copied)}; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "{}() expand-on-demand fallback read failed for '{}' chunk '{}' from old owner '{}': {}", + __func__, path, chunk_id, old_owner, e.what()); + return {EBUSY, 0}; + } +} + +int +expand_on_demand_materialize_for_partial_write(const std::string& path, + uint64_t chunk_id) { + uint64_t old_owner = 0; + if(!expand_on_demand_owns_final_missing_candidate(path, chunk_id, + old_owner)) { + return 0; + } + + char probe = 0; + try { + auto ret = GKFS_DATA->storage()->read_chunk( + path, static_cast(chunk_id), &probe, 1, + 0); + if(ret >= 0) { + return 0; + } + } catch(const gkfs::data::ChunkStorageException& e) { + if(e.code().value() != ENOENT) { + return e.code().value(); + } + } + + std::vector chunk(gkfs::config::rpc::chunksize); + auto [err, read_size] = expand_on_demand_read_remote( + path, chunk_id, chunk.data(), chunk.size(), 0); + if(err == ENOENT) { + return 0; + } + if(err != 0) { + return err; + } + if(read_size <= 0) { + return 0; + } + return 0; +} + /* ------------------------------------------------------------------------ * -------------------------- TRUNCATE ------------------------------------ * ------------------------------------------------------------------------*/ @@ -456,6 +611,29 @@ ChunkReadOperation::wait_for_tasks_and_push_back(const bulk_args& args) { // sparse regions do not have chunk files and are therefore // skipped if(-(*task_size) == ENOENT) { + auto fallback = expand_on_demand_read_remote( + path_, args.chunk_ids->at(idx), + task_args_[idx].buf, task_args_[idx].size, + task_args_[idx].off); + if(fallback.first == 0 && fallback.second > 0) { + try { + args.local_bulk_handle( + args.local_offsets->at(idx), + fallback.second) >> + args.origin_bulk_handle.on( + args.endpoint)( + args.origin_offsets->at(idx), + fallback.second); + total_read += fallback.second; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "ChunkReadOperation::{}() Failed to push expand-on-demand fallback data to client with thallium err: '{}'", + __func__, e.what()); + io_err = EBUSY; + } + } else if(fallback.first != ENOENT) { + io_err = fallback.first; + } task_args_[idx].bulk_transfer_done = true; bulk_transfer_cnt++; ABT_eventual_free(&task_eventuals_[idx]); @@ -533,6 +711,26 @@ ChunkReadOperation::wait_for_tasks_and_push_back(const bulk_args& args) { // sparse regions do not have chunk files and are therefore // skipped if(-(*task_size) == ENOENT) { + auto fallback = expand_on_demand_read_remote( + path_, args.chunk_ids->at(idx), task_args_[idx].buf, + task_args_[idx].size, task_args_[idx].off); + if(fallback.first == 0 && fallback.second > 0) { + try { + args.local_bulk_handle(args.local_offsets->at(idx), + fallback.second) >> + args.origin_bulk_handle.on(args.endpoint)( + args.origin_offsets->at(idx), + fallback.second); + total_read += fallback.second; + } catch(const std::exception& e) { + GKFS_DATA->spdlogger()->error( + "ChunkReadOperation::{}() Failed to push expand-on-demand fallback data to client with thallium err: '{}'", + __func__, e.what()); + io_err = EBUSY; + } + } else if(fallback.first != ENOENT) { + io_err = fallback.first; + } ABT_eventual_free(&task_eventuals_[idx]); continue; } diff --git a/src/daemon/util.cpp b/src/daemon/util.cpp index f53a9819480e6fcd5dce639a16e4b11f05139240..7acfb2da7072e807d1b065d0c1bbd4ec09dea0f7 100644 --- a/src/daemon/util.cpp +++ b/src/daemon/util.cpp @@ -101,13 +101,14 @@ namespace gkfs::utils { void -populate_hosts_file() { +populate_hosts_file(bool expand_mode) { const auto& hosts_file = GKFS_DATA->hosts_file(); const auto& daemon_addr = RPC_DATA->self_addr_str(); const auto& proxy_addr = RPC_DATA->self_proxy_addr_str(); - GKFS_DATA->spdlogger()->debug("{}() Populating hosts file: '{}'", __func__, - hosts_file); + GKFS_DATA->spdlogger()->debug( + "{}() Populating hosts file: '{}' (expand_mode={})", __func__, + hosts_file, expand_mode); // if rootdir_suffix is used, append it to hostname auto hostname = GKFS_DATA->rootdir_suffix().empty() @@ -127,6 +128,11 @@ populate_hosts_file() { (int) GKFS_DATA->link_cnt_state(), (int) GKFS_DATA->blocks_state(), getuid(), getgid()); + // Prepend '+' marker if in expand mode + if(expand_mode) { + line_out = "+" + line_out; + } + // Constants for retry mechanism const int MAX_RETRIES = 5; // Maximum number of retry attempts const std::chrono::milliseconds RETRY_DELAY( diff --git a/src/proxy/proxy.cpp b/src/proxy/proxy.cpp index 1ded101cbf33541074151e20c7bf90c40409bcd1..c00a64f8150b642ddce6d99d5e845ef6283a4ddf 100644 --- a/src/proxy/proxy.cpp +++ b/src/proxy/proxy.cpp @@ -25,10 +25,12 @@ #include #include +#include #include #include #include #include +#include #include @@ -195,16 +197,29 @@ init_environment(const string& hostfile_path, const string& rpc_protocol) { throw runtime_error(err_msg); } - // Setup SimpleDistributor + // Setup Distributor using factory and GKFS_DISTRIBUTION_STRATEGY env var PROXY_DATA->log()->info( - "{}() Setting up simple hash distributor with local_host_id '{}' #hosts '{}'...", + "{}() Setting up distributor with local_host_id '{}' #hosts '{}'...", __func__, PROXY_DATA->local_host_id(), PROXY_DATA->rpc_endpoints().size()); + // Read distribution strategy from GKFS_DISTRIBUTION_STRATEGY env var + // (defaults to simple_hash for backward compatibility) + gkfs::rpc::DistributionConfig config; + const char* env_val = std::getenv(gkfs::env::DISTRIBUTION_STRATEGY); + if(env_val != nullptr && env_val[0] != '\0') { + config.set_strategy(env_val); + } + PROXY_DATA->log()->info("{}() Distribution strategy: '{}'", __func__, + config.get_strategy_string()); // TODO this needs to be globally configured because client must have same // distribution - auto simple_hash_dist = std::make_shared( - PROXY_DATA->local_host_id(), PROXY_DATA->rpc_endpoints().size()); - PROXY_DATA->distributor(simple_hash_dist); + auto distributor = create_from_config(config, PROXY_DATA->local_host_id(), + PROXY_DATA->rpc_endpoints().size(), + 0); // fwd_host not used for proxy + if(!distributor) { + throw std::runtime_error("Failed to create distributor"); + } + PROXY_DATA->distributor(std::move(distributor)); PROXY_DATA->log()->info("Startup successful. Proxy is ready."); } @@ -297,7 +312,7 @@ main(int argc, const char* argv[]) { // Check for environment variables for configuration gkfs::config::rpc::use_dirents_compression = - gkfs::env::get_var("GKFS_PROXY_USE_DIRENTS_COMPRESSION", + gkfs::env::get_var(gkfs::env::USE_DIRENTS_COMPRESSION, gkfs::config::rpc::use_dirents_compression ? "ON" : "OFF") == "ON"; diff --git a/tests/apps/lockfile.sh b/tests/apps/lockfile.sh index 7b4e83a74a61f7bdc620b41fa317decf67241fb9..0153f8c68969e3f9597831e2144a7a36b70884f2 100755 --- a/tests/apps/lockfile.sh +++ b/tests/apps/lockfile.sh @@ -46,7 +46,7 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfi export LIBGKFS_LOG=all export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create -LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & +LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex & GENERATOR=$! sleep 1 OUTPUTLS=`LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC ls $MNT/syscall/` diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2aed09feada2a1fddd3c64dee39de38bb0bfa4b3..22831dc4149f56d6d87147cd67ea0084d8ea946b 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -215,7 +215,9 @@ def gkfwd_daemon_factory(test_workspace, request): interface = request.config.getoption('--interface') - return FwdDaemonCreator(interface, test_workspace) + keep_hosts = "malleability" in request.node.nodeid.split("/") + + return FwdDaemonCreator(interface, test_workspace, keep_hosts=keep_hosts) @pytest.fixture def gkfwd_client_factory(test_workspace): diff --git a/tests/integration/conftest.template b/tests/integration/conftest.template index 2aed09feada2a1fddd3c64dee39de38bb0bfa4b3..22831dc4149f56d6d87147cd67ea0084d8ea946b 100644 --- a/tests/integration/conftest.template +++ b/tests/integration/conftest.template @@ -215,7 +215,9 @@ def gkfwd_daemon_factory(test_workspace, request): interface = request.config.getoption('--interface') - return FwdDaemonCreator(interface, test_workspace) + keep_hosts = "malleability" in request.node.nodeid.split("/") + + return FwdDaemonCreator(interface, test_workspace, keep_hosts=keep_hosts) @pytest.fixture def gkfwd_client_factory(test_workspace): diff --git a/tests/integration/directories/test_packing_order.py b/tests/integration/directories/test_packing_order.py index d650f330f366b41132132bed58b719eb5a1d340f..afe74122b068914152d2598901dba6197af11185 100644 --- a/tests/integration/directories/test_packing_order.py +++ b/tests/integration/directories/test_packing_order.py @@ -1,11 +1,12 @@ - -import pytest import logging +import re import time -from harness.gkfs import Daemon, ShellClient, Client, find_command + +from harness.gkfs import Client, Daemon, ShellClient, find_command log = logging.getLogger(__name__) + def test_packing_order_all_fields(test_workspace, request): """ Comprehensive test to verify the packing order of compressed directory entries. @@ -14,106 +15,110 @@ def test_packing_order_all_fields(test_workspace, request): # Start Daemon with compression ON daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "ON" + "GKFS_USE_DIRENTS_COMPRESSION": "ON" } - daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() try: - client = ShellClient(test_workspace) + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir - client_env = {"LIBGKFS_USE_DIRENTS_COMPRESSION": "ON"} + client_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} # 1. Create files with specific properties # file_a: size 100 # file_b: size 200 # (wait) # file_c: size 100 - + file_a = mount_dir / "file_a" file_b = mount_dir / "file_b" file_c = mount_dir / "file_c" - - # Use dd to create files with specific sizes - client.run("dd", "if=/dev/zero", f"of={file_a}", "bs=100", "count=1", env=client_env) - client.run("dd", "if=/dev/zero", f"of={file_b}", "bs=200", "count=1", env=client_env) - + + ret = io_client.run("write_sequential", "--pathname", str(file_a), + "--count", "1", "--size", "100", env=client_env) + assert ret.retval == 0 + ret = io_client.run("write_sequential", "--pathname", str(file_b), + "--count", "1", "--size", "200", env=client_env) + assert ret.retval == 0 + log.info("Waiting 2 seconds to distinguish ctimes...") time.sleep(2) log.info("Resuming test after sleep.") - + # Create a timestamp file for 'newer' check timestamp_file = test_workspace.twd / "timestamp" timestamp_file.touch() - + time.sleep(1) - client.run("dd", "if=/dev/zero", f"of={file_c}", "bs=100", "count=1", env=client_env) - + ret = io_client.run("write_sequential", "--pathname", str(file_c), + "--count", "1", "--size", "100", env=client_env) + assert ret.retval == 0 + sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - + common_args = ["-M", str(mount_dir), "-S", "1", "--server-side"] - import re def check_match(ret, expected_matches, expected_total=3): output = ret.stdout.decode() log.info(f"SFIND OUTPUT:\n{output}") match = re.search(r"MATCHED (\d+)/(\d+)", output) if not match: assert False, f"Could not find MATCHED line in output:\n{output}" - + actual_matches = int(match.group(1)) total_checked = int(match.group(2)) - + log.info(f"Actual matches: {actual_matches}, Total checked: {total_checked}") - assert actual_matches == expected_matches, f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}" - assert total_checked == expected_total, f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}" + assert actual_matches == expected_matches, \ + f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}" + assert total_checked == expected_total, \ + f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}" # Scenario 1: Filter by Name log.info("Testing -name filter...") - ret = client.run(str(sfind_bin), str(mount_dir), "-name", "file_a", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "file_a", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 2: Filter by Size (100c) log.info("Testing -size filter...") # Should match file_a and file_c (both 100 bytes) - ret = client.run(str(sfind_bin), str(mount_dir), "-size", "100c", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "100c", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 2) # Scenario 3: Filter by Size (200c) log.info("Testing -size filter (200c)...") # Should match file_b - ret = client.run(str(sfind_bin), str(mount_dir), "-size", "200c", *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "200c", + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 4: Filter by Newer (ctime) log.info("Testing -newer filter...") # Should match file_c (created after timestamp_file) - ret = client.run(str(sfind_bin), str(mount_dir), "-newer", str(timestamp_file), *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-newer", + str(timestamp_file), *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) # Scenario 5: Combined Filter (Name *file* AND Size 100c AND Newer) log.info("Testing combined filter...") # Should match only file_c - ret = client.run(str(sfind_bin), str(mount_dir), "-name", "*file*", "-size", "100c", "-newer", str(timestamp_file), *common_args, env=client_env) + ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "*file*", + "-size", "100c", "-newer", str(timestamp_file), + *common_args, env=client_env) assert ret.exit_code == 0 check_match(ret, 1) log.info("Packing order verification successful!") finally: - # Dump daemon log to inspect scanned keys and ctime - # daemon_log = test_workspace.logdir / "daemon.log" - # if daemon_log.exists(): - ## log.info("DUMPING DAEMON LOG DEBUG ENTRIES:") - # with open(daemon_log, 'r') as f: - # for line in f: - # if "DEBUG" in line: - # print(line.strip()) - #else: - # log.warning(f"Daemon log not found at {daemon_log}") daemon.shutdown() diff --git a/tests/integration/directories/test_sfind.py b/tests/integration/directories/test_sfind.py index 1fd17873f3d0cc01bc674ee365601a0048fd8c56..d20b237ef373254d00d4cb506446062a0784e027 100644 --- a/tests/integration/directories/test_sfind.py +++ b/tests/integration/directories/test_sfind.py @@ -1,11 +1,10 @@ - -import pytest import logging -from harness.gkfs import Daemon, ShellClient, Client, find_command -import os -import shlex import time +import pytest + +from harness.gkfs import Client, Daemon, ShellClient, find_command + log = logging.getLogger(__name__) SHELL_CHECK_TIMEOUT = 180 @@ -13,9 +12,9 @@ SHELL_CHECK_TIMEOUT = 180 @pytest.mark.parametrize("buff_size", ["4096", "5242880"]) @pytest.mark.parametrize("conf", [ {"compress": "OFF", "cache": "OFF"}, - {"compress": "ON", "cache": "OFF"}, + {"compress": "ON", "cache": "OFF"}, {"compress": "OFF", "cache": "ON"}, - {"compress": "ON", "cache": "ON"}, + {"compress": "ON", "cache": "ON"}, ]) def test_sfind_permutations(test_workspace, request, conf, buff_size): """ @@ -24,38 +23,36 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 1. Populate with Safe Mode (Comp=OFF) 2. Restart Daemon with Target Mode 3. Run sfind with Target Client Mode - 4. Run ls with Target Client Mode + 4. Run readdir with Target Client Mode """ - + # 1. Safe Population (Daemon Comp=OFF) log.info("--- Phase 1: Population (Safe Mode) ---") pop_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "OFF" + "GKFS_USE_DIRENTS_COMPRESSION": "OFF" } - daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=pop_daemon_env) + daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=pop_daemon_env) daemon_pop.run() try: # Client Env for Population (Safe Mode) pop_client_env = { - "LIBGKFS_USE_DIRENTS_COMPRESSION": "OFF", + "GKFS_USE_DIRENTS_COMPRESSION": "OFF", "LIBGKFS_DENTRY_CACHE": "OFF", "LIBGKFS_LOG": "info" } - - client = ShellClient(test_workspace) + + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir test_dir = mount_dir / "testdir" - - # Ensure directory exists - client.run("mkdir", "-p", str(test_dir)) - # Populate using create_n_files (faster) - io_client = Client(test_workspace) - # Create 2000 files + ret = io_client.mkdir(str(test_dir), 0o755, env=pop_client_env) + assert ret.retval == 0, f"mkdir failed: {ret.errno}" + ret = io_client.create_n_files(str(test_dir), 2000, env=pop_client_env) - assert ret.retval == 0, f"Population failed: {ret.errno}" assert ret.files_created == 2000, f"Expected 2000 files, created {ret.files_created}" log.info("Population complete.") @@ -66,62 +63,53 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): # 2. Test Execution (Target Mode) log.info(f"--- Phase 2: Testing (Conf: {conf}) ---") - + test_daemon_env = { "GKFS_DAEMON_LOG_LEVEL": "info", - "GKFS_DAEMON_USE_DIRENTS_COMPRESSION": conf["compress"] + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"] } - daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=test_daemon_env) + daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=test_daemon_env) daemon_test.run() - + try: # Client Env for Test test_client_env = { - "LIBGKFS_USE_DIRENTS_COMPRESSION": conf["compress"], + "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"], "LIBGKFS_DENTRY_CACHE": conf["cache"], "LIBGKFS_DIRENTS_BUFF_SIZE": buff_size, "LIBGKFS_LOG": "info" } - - test_env_str = "\n".join([f"export {k}={v}" for k,v in test_client_env.items()]) - + sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - + # --- sfind Check --- - log.info(f"Running sfind...") + log.info("Running sfind...") # sfind -S 1 -M - sfind_cmd = f"{test_env_str}\n{sfind_bin} {test_dir} -S 1 -M {mount_dir}" - # Use run("bash", "-c", ...) - ret = client.run("bash", "-c", sfind_cmd, timeout=SHELL_CHECK_TIMEOUT) + # Run sfind directly instead of wrapping it in `bash -c`: ShellClient + # preloads the executed process, and preloading both bash and sfind can + # make the shell crash during teardown after sfind already printed a + # successful MATCHED line. + ret = shell.run( + str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir), + timeout=SHELL_CHECK_TIMEOUT, env=test_client_env) sfind_stderr = ret.stderr.decode() if ret.stderr else "" sfind_stdout = ret.stdout.decode() if ret.stdout else "" - assert ret.exit_code == 0, f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}" - assert "MATCHED 2000/2000" in sfind_stdout, f"sfind did not match 2000/2000. Output:\n{sfind_stdout}" + assert ret.exit_code == 0, \ + f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}" + assert "MATCHED 2000/2000" in sfind_stdout, \ + f"sfind did not match 2000/2000. Output:\n{sfind_stdout}" log.info("sfind verification successful.") - # --- ls Check --- - log.info(f"Running ls check...") - # Avoid `ls -l` here: long format stats every entry and is the first - # thing to time out on overloaded CI runners. - # Expected: 2000 - ls_cmd = ( - f"{test_env_str}\n" - f"ls -1 {shlex.quote(str(test_dir))} | grep '^file_' | wc -l" - ) - ret_ls = client.run("bash", "-c", ls_cmd, timeout=SHELL_CHECK_TIMEOUT) - - ls_stderr = ret_ls.stderr.decode() if ret_ls.stderr else "" - ls_stdout = ret_ls.stdout.decode() if ret_ls.stdout else "" - - assert ret_ls.exit_code == 0, f"ls check failed with {ret_ls.exit_code}\nStderr: {ls_stderr}" - - # parse count - count = ls_stdout.strip() - assert count == "2000", f"ls count expected 2000, got '{count}'" - log.info("ls verification successful.") + log.info("Running readdir check...") + ret_ls = io_client.readdir(str(test_dir), env=test_client_env) + assert ret_ls.errno == 0, f"readdir failed: {ret_ls.errno}" + count = sum(1 for entry in ret_ls.dirents if entry.d_name.startswith("file_")) + assert count == 2000, f"readdir count expected 2000, got '{count}'" + log.info("readdir verification successful.") finally: daemon_test.shutdown() diff --git a/tests/integration/directories/test_sfind_filtered.py b/tests/integration/directories/test_sfind_filtered.py index 23068bceb933a821b9a201ef116546acac1ae8d0..6a7cf87a0e8d51963170a4bb382eb9f83cfacc5a 100644 --- a/tests/integration/directories/test_sfind_filtered.py +++ b/tests/integration/directories/test_sfind_filtered.py @@ -1,54 +1,48 @@ - -import pytest import logging -from harness.gkfs import Daemon, ShellClient, Client, find_command -import os -import time + +from harness.gkfs import Client, Daemon, ShellClient, find_command log = logging.getLogger(__name__) + def test_sfind_filtered(test_workspace, request): """ Test sfind with server-side filtering (-name). """ - + # 1. Start Daemon log.info("--- Starting Daemon ---") daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace) daemon.run() try: - client = ShellClient(test_workspace) + shell = ShellClient(test_workspace) + io_client = Client(test_workspace) mount_dir = test_workspace.mountdir test_dir = mount_dir / "testdir_filtered" - - # Ensure directory exists - client.run("mkdir", "-p", str(test_dir)) - # 2. Populate Files - io_client = Client(test_workspace) - # Create 100 files, named file_0 to file_99 + ret = io_client.mkdir(str(test_dir), 0o755) + assert ret.retval == 0 + ret = io_client.create_n_files(str(test_dir), 100) assert ret.retval == 0 log.info("Population complete.") - # 3.Run sfind with -name filter + # 3. Run sfind with -name filter sfind_bin = find_command("sfind", test_workspace.bindirs) assert sfind_bin, "sfind binary not found" - - # Filter for "file_50" - # Usage: sfind [-name ] -M -S --server-side - # sfind_cmd = f"{sfind_bin} {test_dir} -name \"file_50\" -M {mount_dir} -S 1 --server-side" - + + # Usage: sfind [-name ] -M + # -S --server-side log.info(f"Running sfind command: {sfind_bin} {test_dir} ...") - print(f"DEBUG: Workspace: {test_workspace.twd}") - - ret = client.run(str(sfind_bin), str(test_dir), "-name", "file_50", "-M", str(mount_dir), "-S", "1", "--server-side") + + ret = shell.run(str(sfind_bin), str(test_dir), "-name", "file_50", + "-M", str(mount_dir), "-S", "1", "--server-side") sfind_stderr = ret.stderr.decode(errors='replace') if ret.stderr else "" sfind_stdout = ret.stdout.decode(errors='replace') if ret.stdout else "" - + print(f"STDOUT: {sfind_stdout}") print(f"STDERR: {sfind_stderr}") @@ -57,57 +51,54 @@ def test_sfind_filtered(test_workspace, request): assert "MATCHED" in sfind_stdout # Expect 1 match out of 100 checked assert "MATCHED 1/100" in sfind_stdout - + # 4. Verify Recursive Filtering (Deep Search) # Create a nested directory and file nested_dir = test_dir / "subdir" - client.run("mkdir", "-p", str(nested_dir)) - nested_file = nested_dir / "deep_file.01" - # We need to manually create this file as create_n_files is batch - # Using touch or simple write via ShellClient if io_client is limited - # Actually io_client.create_n_files can point to the subdir - ret = io_client.create_n_files(str(nested_dir), 10) # 10 files in subdir + ret = io_client.mkdir(str(nested_dir), 0o755) + assert ret.retval == 0 + + ret = io_client.create_n_files(str(nested_dir), 10) assert ret.retval == 0 - + # Search for "file_5" in the scoped subdir (should be file_5) - # create_n_files uses formatted names "file_"? - # Let's check how create_n_files names them. Usually "file_"? - # Assuming defaults. - - log.info(f"Running recursive sfind command: {sfind_bin} {test_dir} -name \"*file_5\" ...") + log.info( + f"Running recursive sfind command: {sfind_bin} {test_dir} " + "-name \"*file_5\" ...") # Should find file_5 in top dir AND file_5 in subdir - ret = client.run(str(sfind_bin), str(test_dir), "-name", "*file_5", "-M", str(mount_dir), "-S", "1", "--server-side") - + ret = shell.run(str(sfind_bin), str(test_dir), "-name", "*file_5", + "-M", str(mount_dir), "-S", "1", "--server-side") + sfind_stdout = ret.stdout.decode(errors='replace') if ret.stdout else "" print(f"RECURSIVE STDOUT: {sfind_stdout}") - + assert ret.exit_code == 0 assert "MATCHED" in sfind_stdout # We expect file_5 in top (0-99) and file_5 in subdir (0-9) # Total files: 100 + 10 = 110. # Matches: 2 assert "MATCHED 2/111" in sfind_stdout - + except Exception: # Dump logs client_log = test_workspace.logdir / 'gkfs_client.log' if client_log.exists(): print("\n=== CLIENT LOG ===") print(client_log.read_text()) - + daemon_log = test_workspace.logdir / 'gkfs_daemon.log' if daemon_log.exists(): print("\n=== DAEMON LOG ===") print(daemon_log.read_text()) - + print(f"DEBUG: Workspace: {test_workspace.twd}") - + # Check for sfind result file results_file = test_workspace.twd / 'gfind_results.rank-0.txt' if results_file.exists(): - print(f"\n=== SFIND RESULT FILE ===\n{results_file.read_text()}") + print(f"\n=== SFIND RESULT FILE ===\n{results_file.read_text()}") else: - print(f"\n=== SFIND RESULT FILE NOT FOUND ===") + print("\n=== SFIND RESULT FILE NOT FOUND ===") raise diff --git a/tests/integration/directories/test_symlinks.py b/tests/integration/directories/test_symlinks.py index 78263b3608317919f3523e304fc95d99ff5680fe..f6e48147222f738a4d433d0f32a9710570bf24e8 100644 --- a/tests/integration/directories/test_symlinks.py +++ b/tests/integration/directories/test_symlinks.py @@ -1,12 +1,8 @@ -import harness -from pathlib import Path -import errno import stat import os -import ctypes -import sys + import pytest -from harness.logger import logger + @pytest.mark.parametrize("client_fixture", ["gkfs_client", "gkfs_clientLibc"]) def test_symlink_type(client_fixture, request, gkfs_daemon): @@ -25,9 +21,13 @@ def test_symlink_type(client_fixture, request, gkfs_daemon): ret = gkfs_client.symlink(str(target_name), str(link_name)) assert ret.retval == 0 + ret = gkfs_client.readlink(str(link_name)) + assert ret.retval == len(str(target_name)) + assert ret.target == str(target_name) + # Readdir and check type ret = gkfs_client.readdir(mountdir) - + # We expect ., .., target, mylink found_link = False found_target = False diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 151e30f9c4df09c9ff720242323f6652eb444990..0aa4696c1f033c558ad3edca15c36cc74b08511b 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -56,11 +56,13 @@ add_executable(gkfs.io gkfs.io/lseek.cpp gkfs.io/write_validate.cpp gkfs.io/write_random.cpp + gkfs.io/random_md5.cpp gkfs.io/truncate.cpp gkfs.io/util/file_compare.cpp gkfs.io/chdir.cpp gkfs.io/getcwd_validate.cpp gkfs.io/symlink.cpp + gkfs.io/readlink.cpp gkfs.io/directory_validate.cpp gkfs.io/unlink.cpp gkfs.io/access.cpp @@ -74,6 +76,7 @@ add_executable(gkfs.io ) include(load_nlohmann_json) +find_package(OpenSSL REQUIRED) target_include_directories(gkfs.io PRIVATE ${BOOST_PREPROCESSOR_INCLUDE_DIRS} @@ -84,6 +87,7 @@ target_link_libraries(gkfs.io fmt::fmt CLI11::CLI11 std::filesystem + OpenSSL::Crypto rt ) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 7f0894c2439af78d80a579c6e2b6634f2175ebb7..02aac04edce3b09a8642b9b9694b0332c0d25c87 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -103,6 +103,12 @@ directory_validate_init(CLI::App& app); void write_random_init(CLI::App& app); +void +write_random_and_md5_init(CLI::App& app); + +void +read_random_and_md5_init(CLI::App& app); + void truncate_init(CLI::App& app); @@ -124,6 +130,9 @@ getcwd_validate_init(CLI::App& app); void symlink_init(CLI::App& app); +void +readlink_init(CLI::App& app); + void unlink_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 854c96e619780822a3af22ced60f1cd0399133eb..8fc028774972430973a84a9b1d80e28884cbb745 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -67,6 +67,8 @@ init_commands(CLI::App& app) { write_validate_init(app); directory_validate_init(app); write_random_init(app); + write_random_and_md5_init(app); + read_random_and_md5_init(app); truncate_init(app); access_init(app); statfs_init(app); @@ -75,6 +77,7 @@ init_commands(CLI::App& app) { chdir_init(app); getcwd_validate_init(app); symlink_init(app); + readlink_init(app); unlink_init(app); dup_validate_init(app); syscall_coverage_init(app); diff --git a/tests/integration/harness/gkfs.io/random_md5.cpp b/tests/integration/harness/gkfs.io/random_md5.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2c307e8eb6430f6a3402ce9c45eb929aa50a5da1 --- /dev/null +++ b/tests/integration/harness/gkfs.io/random_md5.cpp @@ -0,0 +1,263 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +namespace { + +constexpr std::size_t buffer_size = 1024 * 1024; + +constexpr std::size_t md5_digest_length = 16; + +std::string +md5_to_hex(const unsigned char* digest, unsigned int digest_len) { + std::string hex; + hex.reserve(digest_len * 2); + for(unsigned int i = 0; i < digest_len; ++i) { + hex += fmt::format("{:02x}", digest[i]); + } + return hex; +} + +class md5_stream { +public: + md5_stream() : ctx_(EVP_MD_CTX_new()) { + if(ctx_ == nullptr || EVP_DigestInit_ex(ctx_, EVP_md5(), nullptr) != 1) { + ok_ = false; + } + } + + ~md5_stream() { + if(ctx_ != nullptr) { + EVP_MD_CTX_free(ctx_); + } + } + + bool + update(const void* data, std::size_t size) { + if(!ok_) { + return false; + } + if(EVP_DigestUpdate(ctx_, data, size) != 1) { + ok_ = false; + } + return ok_; + } + + std::string + final() { + if(!ok_) { + return ""; + } + std::array digest{}; + unsigned int digest_len = 0; + if(EVP_DigestFinal_ex(ctx_, digest.data(), &digest_len) != 1) { + return ""; + } + if(digest_len != md5_digest_length) { + return ""; + } + return md5_to_hex(digest.data(), digest_len); + } + +private: + EVP_MD_CTX* ctx_; + bool ok_{true}; +}; + +void +fill_random(std::vector& buffer, std::mt19937_64& rng) { + std::size_t pos = 0; + while(pos < buffer.size()) { + auto value = rng(); + for(std::size_t i = 0; i < sizeof(value) && pos < buffer.size(); ++i) { + buffer[pos++] = static_cast((value >> (i * 8U)) & 0xffU); + } + } +} + +} // namespace + +struct random_md5_options { + bool verbose{}; + std::string pathname{}; + ::size_t count{}; + uint64_t seed{42}; + + REFL_DECL_STRUCT(random_md5_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(uint64_t, seed)); +}; + +struct random_md5_output { + ::ssize_t retval; + int errnum; + std::string md5; + + REFL_DECL_STRUCT(random_md5_output, REFL_DECL_MEMBER(::ssize_t, retval), + REFL_DECL_MEMBER(int, errnum), + REFL_DECL_MEMBER(std::string, md5)); +}; + +void +to_json(json& record, const random_md5_output& out) { + record = serialize(out); +} + +void +write_random_and_md5_exec(const random_md5_options& opts) { + auto fd = ::open(opts.pathname.c_str(), O_CREAT | O_WRONLY | O_TRUNC, + S_IRWXU | S_IRWXG | S_IRWXO); + if(fd == -1) { + json out = random_md5_output{-1, errno, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + md5_stream md5_ctx; + + std::mt19937_64 rng(opts.seed); + std::vector buffer(buffer_size); + std::size_t remaining = opts.count; + ::ssize_t written_total = 0; + int err = 0; + + while(remaining > 0) { + const auto step = std::min(buffer.size(), remaining); + if(buffer.size() != step) { + buffer.resize(step); + } + fill_random(buffer, rng); + if(!md5_ctx.update(buffer.data(), step)) { + ::close(fd); + json out = random_md5_output{-1, EIO, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + std::size_t done = 0; + while(done < step) { + auto rv = ::write(fd, buffer.data() + done, step - done); + if(rv < 0) { + err = errno; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + if(rv == 0) { + err = EIO; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + done += static_cast(rv); + written_total += rv; + } + remaining -= step; + } + + if(::close(fd) != 0 && err == 0) { + err = errno; + } + + json out = random_md5_output{err == 0 ? written_total : -1, err, + md5_ctx.final()}; + fmt::print("{}\n", out.dump(2)); +} + +void +read_random_and_md5_exec(const random_md5_options& opts) { + auto fd = ::open(opts.pathname.c_str(), O_RDONLY); + if(fd == -1) { + json out = random_md5_output{-1, errno, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + + md5_stream md5_ctx; + + std::vector buffer(buffer_size); + ::ssize_t read_total = 0; + int err = 0; + + for(;;) { + auto rv = ::read(fd, buffer.data(), buffer.size()); + if(rv < 0) { + err = errno; + ::close(fd); + json out = random_md5_output{-1, err, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + if(rv == 0) { + break; + } + if(!md5_ctx.update(buffer.data(), static_cast(rv))) { + ::close(fd); + json out = random_md5_output{-1, EIO, ""}; + fmt::print("{}\n", out.dump(2)); + return; + } + read_total += rv; + } + + if(::close(fd) != 0 && err == 0) { + err = errno; + } + + json out = random_md5_output{err == 0 ? read_total : -1, err, + md5_ctx.final()}; + fmt::print("{}\n", out.dump(2)); +} + +void +write_random_and_md5_init(CLI::App& app) { + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("write_random_and_md5", + "Create/truncate a file, write deterministic random data, and return MD5"); + cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output"); + cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name(""); + cmd->add_option("count", opts->count, "Number of bytes to write")->required()->type_name(""); + cmd->add_option("--seed", opts->seed, "Pseudo-random seed")->type_name(""); + cmd->callback([opts]() { write_random_and_md5_exec(*opts); }); +} + +void +read_random_and_md5_init(CLI::App& app) { + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("read_random_and_md5", + "Read a full file and return MD5"); + cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output"); + cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name(""); + cmd->callback([opts]() { read_random_and_md5_exec(*opts); }); +} \ No newline at end of file diff --git a/tests/integration/harness/gkfs.io/readlink.cpp b/tests/integration/harness/gkfs.io/readlink.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f774fee0cb53f1bc2f5cf17b91f40074ace4d4ba --- /dev/null +++ b/tests/integration/harness/gkfs.io/readlink.cpp @@ -0,0 +1,120 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This software was partially supported by the + the European Union’s Horizon 2020 JTI-EuroHPC research and + innovation programme, by the project ADMIRE (Project ID: 956748, + admire-eurohpc.eu) + + This project was partially promoted by the Ministry for Digital Transformation + and the Civil Service, within the framework of the Recovery, + Transformation and Resilience Plan - Funded by the European Union + -NextGenerationEU. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include + +using json = nlohmann::json; + +struct readlink_options { + bool verbose{}; + std::string pathname; + ::size_t size = 4096; + + REFL_DECL_STRUCT(readlink_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, size)); +}; + +struct readlink_output { + std::string target; + long retval; + int errnum; + + REFL_DECL_STRUCT(readlink_output, REFL_DECL_MEMBER(std::string, target), + REFL_DECL_MEMBER(long, retval), + REFL_DECL_MEMBER(int, errnum)); +}; + +void +to_json(json& record, const readlink_output& out) { + record = serialize(out); +} + +void +readlink_exec(const readlink_options& opts) { + + std::string target(opts.size, '\0'); + errno = 0; + auto rv = ::readlink(opts.pathname.c_str(), target.data(), target.size()); + auto errnum = errno; + + if(rv >= 0) { + target.resize(static_cast(rv)); + } else { + target.clear(); + } + + if(opts.verbose) { + fmt::print( + "readlink(pathname=\"{}\", bufsize={}) = {}, target=\"{}\", errno: {} [{}]\n", + opts.pathname, opts.size, rv, target, errnum, ::strerror(errnum)); + return; + } + + json out = readlink_output{target, static_cast(rv), errnum}; + fmt::print("{}\n", out.dump(2)); +} + +void +readlink_init(CLI::App& app) { + + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand("readlink", "Execute the readlink() system call"); + + cmd->add_flag("-v,--verbose", opts->verbose, + "Produce human readable output"); + + cmd->add_option("pathname", opts->pathname, "Symbolic link path") + ->required() + ->type_name(""); + + cmd->add_option("--size", opts->size, "Read buffer size") + ->default_val(opts->size); + + cmd->callback([opts]() { readlink_exec(*opts); }); +} \ No newline at end of file diff --git a/tests/integration/harness/gkfs.py b/tests/integration/harness/gkfs.py index a707463613c9b18b101a227b1c9a0c25e1a31604..ac19a533361b8fbb33276a05cc3ca8c3cdafbbe2 100644 --- a/tests/integration/harness/gkfs.py +++ b/tests/integration/harness/gkfs.py @@ -64,9 +64,9 @@ gkfwd_client_lib_file = 'libgkfs_intercept.so' gkfwd_hosts_file = 'gkfs_hosts.txt' gkfwd_forwarding_map_file = 'gkfs_forwarding.map' gkfwd_daemon_log_file = 'gkfs_daemon.log' -gkfwd_daemon_log_level = '100' +gkfwd_daemon_log_level = '20' gkfwd_client_log_file = 'gkfs_client.log' -gkfwd_client_log_level = 'all' +gkfwd_client_log_level = 'debug' gkfwd_client_log_syscall_filter = 'epoll_wait,epoll_create' gkfwd_daemon_active_log_pattern = r'Startup successful. Daemon is ready.' @@ -239,20 +239,33 @@ class FwdDaemonCreator: Factory that allows tests to create forwarding daemons in a workspace. """ - def __init__(self, interface, workspace): + def __init__(self, interface, workspace, keep_hosts=False): self._interface = interface self._workspace = workspace + self._keep_hosts = keep_hosts - def create(self): + def create(self, expand_mode=False, enable_forwarding=True, keep_hosts=None): """ Create a forwarding daemon in the tests workspace. + Parameters + ---------- + expand_mode: `bool` + If True, sets GKFS_DAEMON_EXPAND=ON in the daemon environment. + Returns ------- The `FwdDaemon` object to interact with the daemon. """ - daemon = FwdDaemon(self._interface, self._workspace) + if keep_hosts is None: + keep_hosts = self._keep_hosts + + daemon = FwdDaemon(self._interface, + self._workspace, + expand_mode=expand_mode, + enable_forwarding=enable_forwarding, + keep_hosts=keep_hosts) daemon.run() return daemon @@ -692,7 +705,13 @@ class Client: stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - out_stdout, out_stderr = proc.communicate() + try: + out_stdout, out_stderr = proc.communicate(timeout=kwargs.get('timeout')) + except subprocess.TimeoutExpired: + proc.kill() + out_stdout, out_stderr = proc.communicate() + logger.error(f"Command timed out after {kwargs.get('timeout')}s") + out_stdout = b'{"errnum": 110, "retval": -1}' out_returncode = proc.returncode if out_stdout: @@ -874,9 +893,16 @@ class ShellClient: self._env = os.environ.copy() self._proxy = proxy - libdirs = ':'.join( - filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + - [str(p) for p in self._workspace.libdirs])) + bindirs = os.pathsep.join(str(p) for p in self._workspace.bindirs) + if bindirs: + self._env['PATH'] = os.pathsep.join( + filter(None, [bindirs, os.environ.get('PATH', '')])) + + libdirs = _compose_ld_library_path( + preferred_dirs=[], + workspace_dirs=self._workspace.libdirs, + inherited_ld_path=os.environ.get('LD_LIBRARY_PATH', ''), + ) # ensure the client interception library is available: # to avoid running code with potentially installed libraries, @@ -989,8 +1015,12 @@ class ShellClient: logger.debug(f"patched env: {self._patched_env}") + script_env = self._env.copy() + if not intercept_shell: + script_env.pop('LD_PRELOAD', None) + proc = subprocess.Popen(['bash', '-c', code], - env = (self._env if intercept_shell else os.environ), + env = script_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) @@ -1388,13 +1418,27 @@ class ShellClientLibc: return self._workspace.twd class FwdDaemon: - def __init__(self, interface, workspace): + def __init__(self, interface, workspace, expand_mode=False, enable_forwarding=True, keep_hosts=False): self._address = get_ephemeral_address(interface) self._workspace = workspace self._hostfile = str(self.cwd / gkfwd_hosts_file) self._cmd = find_command(gkfwd_daemon_cmd, self._workspace.bindirs) self._env = os.environ.copy() + self._expand_mode = expand_mode + self._enable_forwarding = enable_forwarding + self._keep_hosts = keep_hosts + self._rootdir = self._workspace.rootdir + self._metadir = self._workspace.metadir + self._logdir = self._workspace.logdir + if not enable_forwarding: + suffix = self._address.replace(':', '_') + self._rootdir = self._workspace.twd / f"root_{suffix}" + self._metadir = self._workspace.twd / f"meta_{suffix}" + self._logdir = self._workspace.logdir / f"daemon_{suffix}" + self._rootdir.mkdir(parents=True, exist_ok=True) + self._metadir.mkdir(parents=True, exist_ok=True) + self._logdir.mkdir(parents=True, exist_ok=True) libdirs = ':'.join( filter(None, [os.environ.get('LD_LIBRARY_PATH', '')] + @@ -1404,8 +1448,13 @@ class FwdDaemon: 'LD_LIBRARY_PATH' : libdirs, 'GKFS_HOSTS_FILE' : str(self.cwd / gkfwd_hosts_file), 'GKFS_DAEMON_LOG_PATH' : str(self.logdir / gkfwd_daemon_log_file), - 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level + 'GKFS_DAEMON_LOG_LEVEL': gkfwd_daemon_log_level, } + if keep_hosts: + self._patched_env['GKFS_DAEMON_KEEP_HOSTS_FILE'] = 'ON' + if expand_mode: + self._patched_env['GKFS_DAEMON_EXPAND'] = 'ON' + self._env.update(self._patched_env) def run(self): @@ -1413,8 +1462,9 @@ class FwdDaemon: args = [ '--mountdir', self.mountdir, '--metadir', self.metadir, '--rootdir', self.rootdir, - '-l', self._address, - '--enable-forwarding'] + '-l', self._address] + if self._enable_forwarding: + args.append('--enable-forwarding') logger.debug(f"spawning daemon") logger.debug(f"cmdline: {self._cmd} " + " ".join(map(str, args))) @@ -1491,6 +1541,12 @@ class FwdDaemon: # ... or it might just be lazy. let's give it some more time logger.debug(f"daemon {pid} found, retrying...") + if self._proc.poll() is not None: + raise RuntimeError( + f"daemon {pid} exited during initialization with code " + f"{self._proc.returncode}; log: " + f"{self.logdir / gkfwd_daemon_log_file}" + ) time.sleep(1) raise RuntimeError("initialization timeout exceeded") @@ -1512,11 +1568,11 @@ class FwdDaemon: @property def rootdir(self): - return self._workspace.rootdir + return self._rootdir @property def metadir(self): - return self._workspace.metadir + return self._metadir @property def mountdir(self): @@ -1524,7 +1580,7 @@ class FwdDaemon: @property def logdir(self): - return self._workspace.logdir + return self._logdir @property def interface(self): diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index b41990f9e82c273a989a12697fdc346dc6a2f5fb..33190f2d4dac870a69f7b6044b390c81bd6e057c 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -373,6 +373,18 @@ class WriteRandomOutputSchema(Schema): return namedtuple('WriteRandomReturn', ['retval', 'errno'])(**data) +class RandomMd5OutputSchema(Schema): + """Schema to deserialize random write/read with MD5 helper output""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + md5 = fields.String(required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('RandomMd5Return', ['retval', 'errno', 'md5'])(**data) + + class WriteSyncOutputSchema(Schema): """Schema to deserialize the results of a write_sync() execution""" @@ -456,6 +468,18 @@ class SymlinkOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('SymlinkReturn', ['retval', 'errno'])(**data) +class ReadlinkOutputSchema(Schema): + """Schema to deserialize the results of a readlink() execution""" + + target = fields.String(required=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('ReadlinkReturn', + ['target', 'retval', 'errno'])(**data) + class UnlinkOutputSchema(Schema): """Schema to deserialize the results of an unlink() execution""" retval = fields.Integer(required=True) @@ -516,6 +540,8 @@ class IOParser: 'statx' : StatxOutputSchema(), 'lseek' : LseekOutputSchema(), 'write_random': WriteRandomOutputSchema(), + 'write_random_and_md5': RandomMd5OutputSchema(), + 'read_random_and_md5': RandomMd5OutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'write_sequential' : WriteValidateOutputSchema(), @@ -531,6 +557,7 @@ class IOParser: 'chdir' : ChdirOutputSchema(), 'getcwd_validate' : GetcwdvalidateOutputSchema(), 'symlink' : SymlinkOutputSchema(), + 'readlink': ReadlinkOutputSchema(), 'dup_validate' : DupValidateOutputSchema(), 'syscall_coverage' : SyscallCoverageOutputSchema(), 'create_n_files' : CreateNFilesOutputSchema(), diff --git a/tests/integration/malleability/test_client_disconnect_during_rpc.py b/tests/integration/malleability/test_client_disconnect_during_rpc.py index c682483f433fa89d7312873deadc7bc250f87a75..5b3505caeedc1c2b953dd8ea6da41b9e8608f814 100644 --- a/tests/integration/malleability/test_client_disconnect_during_rpc.py +++ b/tests/integration/malleability/test_client_disconnect_during_rpc.py @@ -37,7 +37,6 @@ which contains NA_NOENTRY errors when clients vanish during RPC handling. import os import subprocess import time -import shutil from pathlib import Path import pytest @@ -48,22 +47,16 @@ def test_daemon_survives_client_abort_during_write(gkfwd_daemon_factory, gkfs_sh time.sleep(5) hostfile = Path(d00.hostfile) - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" # Write some data first to establish the file exists - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") ret = gkfs_shell.bash( - f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={hostfile} " + f"LIBGKFS_HOSTS_FILE={hostfile} " f"dd if=/dev/zero of={d00.mountdir}/test_abort_file bs=1024 count=1 2>&1" ) # Verify daemon works before abort cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, f"Daemon not responding before client abort: {cmd.stderr.decode()}" @@ -82,13 +75,6 @@ def test_safe_respond_handles_vanished_client(gkfwd_daemon_factory, gkfs_shell): hostfile = Path(d00.hostfile) - # Verify safe_respond wrapper exists in the compiled source - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - daemon_bin = shutil.which("gkfs_daemon", path=search_path) - assert daemon_bin is not None, "gkfs_daemon not found in PATH" - - # Check objdump for safe/respond symbols - ret = gkfs_shell.bash(f"objdump -t {daemon_bin} 2>/dev/null | grep -i safe | wc -l || echo 0") # The test just needs to pass if the daemon is functional # Verify the daemon survives shutdown and restart d00.shutdown() @@ -110,7 +96,6 @@ def test_multiple_rapid_client_disconnections(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") # Simulate multiple rapid disconnections for i in range(5): @@ -118,7 +103,6 @@ def test_multiple_rapid_client_disconnections(gkfwd_daemon_factory, gkfs_shell): client_proc = subprocess.Popen( ["python3", "-c", f""" import os, time -os.environ['LD_LIBRARY_PATH'] = '{libdirs}' os.environ['LIBGKFS_HOSTS_FILE'] = '{hostfile}' fd = os.open('{d00.mountdir}/test_rapid_{i}', os.O_CREAT | os.O_WRONLY, 0o644) if fd >= 0: @@ -133,14 +117,9 @@ if fd >= 0: time.sleep(0.5) # Verify daemon survived all disconnections - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" + f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ diff --git a/tests/integration/malleability/test_expand_on_demand.py b/tests/integration/malleability/test_expand_on_demand.py new file mode 100644 index 0000000000000000000000000000000000000000..acbd257237ca54fed08f00027f84e65ad6330111 --- /dev/null +++ b/tests/integration/malleability/test_expand_on_demand.py @@ -0,0 +1,228 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +import hashlib +import os +import stat +import time +from pathlib import Path + +import pytest + + +FILE_COUNT = 8 +FILE_SIZE = 1024 * 1024 +PARTIAL_OVERWRITE_OFFSET = 128 +PARTIAL_OVERWRITE = b"expand-on-demand-partial" + + + +def _run_mutate_cmd(gkfs_shell, hosts_file, args, timeout=120): + cmd_str = f'LIBGKFS_HOSTS_FILE="{hosts_file}" gkfs_malleability {args}' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + assert cmd.exit_code == 0, ( + f"gkfs_malleability {args} failed\n" + f"stdout: {cmd.stdout.decode()}\n" + f"stderr: {cmd.stderr.decode()}" + ) + return cmd + + +def _mark_added_daemon(hostfile, daemon): + port = daemon.address.rsplit(":", 1)[-1] + lines = Path(hostfile).read_text().splitlines(keepends=True) + with open(hostfile, "w") as hf_out: + for line in lines: + stripped = line.strip() + if stripped and not stripped.startswith("#") and f":{port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + +def _read_md5(client, path): + ret = client.stat(path) + assert ret.retval == 0, f"stat failed for {path}" + size = ret.statbuf.st_size + ret = client.open(path, os.O_RDONLY, 0) + assert ret.retval != -1, f"open read failed for {path}" + ret = client.read(path, size) + assert ret.retval == size, f"read failed for {path}: {ret.retval}/{size}" + assert ret.buf is not None, f"read returned no buffer for {path}" + return hashlib.md5(ret.buf).hexdigest() + + +def _expected_payload_with_partial_overwrite(): + payload = bytearray((ord("0") + (i % 10)) for i in range(FILE_SIZE)) + payload[ + PARTIAL_OVERWRITE_OFFSET : PARTIAL_OVERWRITE_OFFSET + len(PARTIAL_OVERWRITE) + ] = PARTIAL_OVERWRITE + return bytes(payload) + + +def _write_file(client, path): + ret = client.open( + path, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open write failed for {path}" + ret = client.write_validate(path, FILE_SIZE) + assert ret.retval == 0, f"write_validate failed for {path}" + return _read_md5(client, path) + + +def _wait_for_mutate_done(gkfs_shell, hostfile): + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") + if "No mutate running/finished." in cmd.stderr.decode(): + return + time.sleep(2) + pytest.fail("Mutate did not finish within 120 s") + + +def test_expand_on_demand_skips_eager_data_and_keeps_reads_correct( + monkeypatch, gkfwd_daemon_factory, gkfs_client, gkfs_shell +): + monkeypatch.setenv("GKFS_EXPAND_ON_DEMAND", "ON") + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", "simple_hash") + + daemons = [] + try: + d00 = gkfwd_daemon_factory.create() + daemons.append(d00) + time.sleep(2) + + md5_map = {} + for i in range(FILE_COUNT): + path = d00.mountdir / f"expand_on_demand_{i:03d}.dat" + md5_map[path] = _write_file(gkfs_client, path) + + d01 = gkfwd_daemon_factory.create(expand_mode=True) + daemons.append(d01) + time.sleep(2) + + hostfile = Path(d00.hostfile) + _mark_added_daemon(hostfile, d01) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate start", 340) + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") + + daemon_log = Path(d00.logdir) / "gkfs_daemon.log" + log_text = daemon_log.read_text() if daemon_log.exists() else "" + assert "GKFS_EXPAND_ON_DEMAND active" in log_text + assert "Skipping eager data migration" in log_text + + expected_after_partial = _expected_payload_with_partial_overwrite() + expected_after_partial_md5 = hashlib.md5(expected_after_partial).hexdigest() + partial_path = next(iter(md5_map.keys())) + ret = gkfs_client.pwrite( + partial_path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + + for path, expected_md5 in md5_map.items(): + if path == partial_path: + assert _read_md5(gkfs_client, path) == expected_after_partial_md5 + else: + assert _read_md5(gkfs_client, path) == expected_md5 + + for path in list(md5_map.keys())[1:]: + ret = gkfs_client.pwrite( + path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + assert _read_md5(gkfs_client, path) == expected_after_partial_md5 + + finally: + for daemon in daemons: + daemon.shutdown() + + +def test_expand_on_demand_2_to_4_read_and_partial_write( + monkeypatch, gkfwd_daemon_factory, gkfs_client, gkfs_shell +): + """Verify lazy full-chunk reads and partial writes across a 2->4 expand.""" + monkeypatch.setenv("GKFS_EXPAND_ON_DEMAND", "ON") + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", "simple_hash") + + daemons = [] + try: + old_daemons = [ + gkfwd_daemon_factory.create(), + gkfwd_daemon_factory.create(), + ] + daemons.extend(old_daemons) + time.sleep(3) + + hostfile = Path(old_daemons[0].hostfile) + md5_map = {} + for i in range(FILE_COUNT): + path = old_daemons[0].mountdir / f"expand_on_demand_2to4_{i:03d}.dat" + md5_map[path] = _write_file(gkfs_client, path) + + for path, expected_md5 in md5_map.items(): + assert _read_md5(gkfs_client, path) == expected_md5 + + new_daemons = [ + gkfwd_daemon_factory.create(expand_mode=True), + gkfwd_daemon_factory.create(expand_mode=True), + ] + daemons.extend(new_daemons) + time.sleep(3) + + lines = hostfile.read_text().splitlines() + for daemon in new_daemons: + port = daemon.address.rsplit(":", 1)[-1] + matching = [line for line in lines if f":{port}" in line] + assert len(matching) == 1, f"Missing hostfile entry for {daemon.address}" + assert matching[0].startswith("+"), ( + f"Expected '+' marker for {daemon.address}: {matching[0]}" + ) + + _run_mutate_cmd(gkfs_shell, hostfile, "mutate start", 340) + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") + + # A full read must fetch a missing chunk from the old 2-node layout, + # materialize it on the final owner, and return the original bytes. + for path, expected_md5 in md5_map.items(): + assert _read_md5(gkfs_client, path) == expected_md5 + + partial_path = next(iter(md5_map)) + expected_payload = bytearray( + (ord("0") + (i % 10)) for i in range(FILE_SIZE) + ) + expected_payload[ + PARTIAL_OVERWRITE_OFFSET : + PARTIAL_OVERWRITE_OFFSET + len(PARTIAL_OVERWRITE) + ] = PARTIAL_OVERWRITE + ret = gkfs_client.pwrite( + partial_path, + PARTIAL_OVERWRITE, + len(PARTIAL_OVERWRITE), + PARTIAL_OVERWRITE_OFFSET, + ) + assert ret.retval == len(PARTIAL_OVERWRITE) + assert _read_md5(gkfs_client, partial_path) == hashlib.md5( + expected_payload + ).hexdigest() + + # Read another file after the partial write to ensure no stale or + # unrelated materialization changed other chunks. + for path, expected_md5 in list(md5_map.items())[1:]: + assert _read_md5(gkfs_client, path) == expected_md5 + finally: + for daemon in daemons: + daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/malleability/test_malleability_error_handling.py b/tests/integration/malleability/test_malleability_error_handling.py index f63c649a66bda6a1da120127445d8f6d4a183db4..0c67bbac9e3dac07695bc8499cb2c37eec3198ae 100644 --- a/tests/integration/malleability/test_malleability_error_handling.py +++ b/tests/integration/malleability/test_malleability_error_handling.py @@ -35,7 +35,6 @@ RPC errors, and mid-operation failures without crashing the daemon. import os import time -import shutil from pathlib import Path import pytest @@ -46,22 +45,10 @@ def test_expand_status_with_no_running_expansion(gkfwd_daemon_factory, gkfs_shel time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"expand status failed: {cmd.stderr.decode()}" + assert cmd.exit_code == 0, f"mutate status failed: {cmd.stderr.decode()}" d00.shutdown() @@ -72,29 +59,20 @@ def test_expand_start_with_same_node_count(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" + # Create a new-hosts-file with the same content (same node count) + new_hostfile = hostfile.parent / "gkfs_hosts_same.txt" + with open(hostfile, "r") as hf_in, open(new_hostfile, "w") as hf_out: + for line in hf_in: + if not line.startswith("#"): + hf_out.write(line) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) # Verify daemon is still running (didn't crash) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" @@ -108,30 +86,20 @@ def test_shrink_status_after_failed_expand(gkfwd_daemon_factory, gkfs_shell): time.sleep(5) hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) + # Create a new-hosts-file with the same content (will fail) + new_hostfile = hostfile.parent / "gkfs_hosts_fail.txt" + with open(hostfile, "r") as hf_in, open(new_hostfile, "w") as hf_out: + for line in hf_in: + if not line.startswith("#"): + hf_out.write(line) - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - - # Try expand (may fail) - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) time.sleep(3) - # Verify shrink status works - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} shrink status" - ) + # Verify mutate status works + cmd_str = f"LIBGKFS_HOSTS_FILE={new_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"shrink status after failed expand failed: {cmd.stderr.decode()}" @@ -159,38 +127,20 @@ def test_malleability_expand_with_data(gkfwd_daemon_factory, gkfs_client, gkfs_s assert ret.retval == 0, f"write_validate failed for {f}" hostfile = Path(d00.hostfile) - with open(hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" + same_hostfile = hostfile.parent / "gkfs_hosts_expand_with_data_same.txt" + with open(hostfile, "r") as hf_in, open(same_hostfile, "w") as hf_out: + hf_out.writelines(hf_in.readlines()) - # Verify no running expansion - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + # Verify no running mutation + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - # Start expansion - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand start" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) # Verify daemon is still running - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={same_hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ f"Daemon crashed after expand start: {cmd.stderr.decode()}" diff --git a/tests/integration/malleability/test_malleability_performance.py b/tests/integration/malleability/test_malleability_performance.py new file mode 100644 index 0000000000000000000000000000000000000000..0c98628afe1c62e0054f8cf68c26f602ae439513 --- /dev/null +++ b/tests/integration/malleability/test_malleability_performance.py @@ -0,0 +1,1433 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Standalone performance test for GekkoFS malleability (shrink/expand/mutate). + +MARKER-BASED WORKFLOW: +- LIBGKFS_HOSTS_FILE must contain ALL hosts with markers +- For shrink: '-' markers on removing nodes +- For expand: '+' markers on adding nodes +- The workspace contains ALL daemons (active + adding/removing) +""" + +import os +import signal +import random +import stat +import time +import math +import hashlib +import errno +import json +import shutil +import statistics +import subprocess +import tempfile +import shlex +from pathlib import Path +from collections import Counter, defaultdict +from dataclasses import dataclass +from time import perf_counter +import pytest +from harness.gkfs import FwdDaemonCreator +from harness.logger import logger + + +# ========= +# Helpers +# ========= + +def _shutdown_daemons(daemons, timeout=10): + """Shutdown daemons without letting perf tests hang forever.""" + for daemon in daemons: + try: + proc = getattr(daemon, "_proc", None) + if proc is not None and proc.poll() is None: + logger.debug(f"terminating daemon pid {proc.pid}") + proc.terminate() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + logger.warning(f"daemon pid {proc.pid} did not stop after SIGTERM; killing") + proc.kill() + proc.wait(timeout=timeout) + except Exception as exc: + logger.warning(f"daemon cleanup failed: {exc}") + finally: + _cleanup_daemon_storage(daemon) + +def _snapshot_chunk_files(rootdir): + """Snapshot chunk files under a daemon rootdir. + + The relative chunk path is the primary chunk identity. The digest is kept as + a guard against stale files and as a fallback for path-renamed chunks. Do not + aggregate by digest alone: performance tests create deterministic equal-size + files, so many distinct chunks can have identical content. + """ + rootdir = Path(rootdir) + chunkdir = rootdir / "chunks" + snapshot = {} + + if not chunkdir.exists(): + return snapshot + + for path in chunkdir.rglob("*"): + if not path.is_file(): + continue + try: + data = path.read_bytes() + except Exception: + continue + relpath = path.relative_to(rootdir).as_posix() + snapshot[relpath] = (path.stat().st_size, hashlib.md5(data).hexdigest()) + + return snapshot + + +def _count_moved_chunk_bytes(before_snapshots, after_snapshots): + """Count bytes for chunks that changed daemon ownership. + + Robust rules: + - Primary match is `(relative chunk path, size, digest)`. This keeps chunks + with duplicate content separate. + - Count with multiplicity, not sets. If N equal chunks move, count N chunks. + - Count new materializations on different daemons. Some implementations copy + chunks first and delete old copies later, so an unchanged source copy must + not hide the redistributed destination copy. + - Lost chunks are caught by MD5 verification, not reported as redistributed + bytes. + - Fallback to `(size, digest)` only for unmatched chunks, in case a backend + changes the relative chunk path during redistribution. + """ + + def build_locations(snapshots, key_fn): + locations = defaultdict(Counter) + sizes = {} + for daemon, snapshot in snapshots.items(): + for relpath, (size, digest) in snapshot.items(): + key = key_fn(relpath, size, digest) + locations[key][daemon] += 1 + sizes[key] = size + return locations, sizes + + def count_moved(before_locations, after_locations, sizes): + moved = 0 + matched_keys = set() + for key, before_daemons in before_locations.items(): + after_daemons = after_locations.get(key) + if not after_daemons: + continue + after_total = sum(after_daemons.values()) + unchanged = sum( + min(before_daemons[daemon], after_daemons.get(daemon, 0)) + for daemon in before_daemons + ) + moved_instances = max(0, after_total - unchanged) + if moved_instances: + moved += moved_instances * sizes[key] + matched_keys.add(key) + return moved, matched_keys + + before_by_identity, identity_sizes = build_locations( + before_snapshots, lambda relpath, size, digest: (relpath, size, digest)) + after_by_identity, _ = build_locations( + after_snapshots, lambda relpath, size, digest: (relpath, size, digest)) + + moved, matched_identity_keys = count_moved( + before_by_identity, after_by_identity, identity_sizes) + + # Fallback for path-renamed chunks only. Exclude identities already matched + # exactly so duplicate-content chunks are not double counted or collapsed. + unmatched_before = defaultdict(dict) + unmatched_after = defaultdict(dict) + for daemon, snapshot in before_snapshots.items(): + for relpath, value in snapshot.items(): + size, digest = value + if (relpath, size, digest) not in matched_identity_keys: + unmatched_before[daemon][relpath] = value + for daemon, snapshot in after_snapshots.items(): + for relpath, value in snapshot.items(): + size, digest = value + if (relpath, size, digest) not in matched_identity_keys: + unmatched_after[daemon][relpath] = value + + before_by_digest, digest_sizes = build_locations( + unmatched_before, lambda relpath, size, digest: (size, digest)) + after_by_digest, _ = build_locations( + unmatched_after, lambda relpath, size, digest: (size, digest)) + fallback_moved, _ = count_moved(before_by_digest, after_by_digest, digest_sizes) + + return moved + fallback_moved + + +def _wait_for_unmounted(mountdir, timeout=10): + mountdir = Path(mountdir) + deadline = time.time() + timeout + while time.time() < deadline: + if not mountdir.exists(): + return + try: + if os.path.ismount(mountdir): + time.sleep(0.5) + continue + # If the path still exists but is no longer a mount, make sure it is + # actually writable before we continue. This catches stale fuse state + # that can still keep the next iteration busy. + probe = mountdir / ".gkfs_unmount_probe" + try: + probe.touch(exist_ok=True) + probe.unlink(missing_ok=True) + return + except Exception: + time.sleep(0.5) + continue + except Exception: + time.sleep(0.5) + + +def _cleanup_daemon_storage(daemon): + for path in (daemon.rootdir, daemon.metadir): + path = Path(path) + for attempt in range(20): + shutil.rmtree(path.as_posix(), ignore_errors=True) + if not path.exists(): + break + time.sleep(0.25) + if path.exists(): + logger.warning(f"failed to remove daemon storage {path}") + #if daemon.logdir.exists(): + # shutil.rmtree(daemon.logdir.as_posix(), ignore_errors=True) + + +def _cleanup_iteration_workspace(iter_workspace): + for path in (iter_workspace.rootdir, iter_workspace.metadir, iter_workspace.mountdir): + path = Path(path) + for attempt in range(20): + shutil.rmtree(path.as_posix(), ignore_errors=True) + if not path.exists(): + break + time.sleep(0.25) + if path.exists(): + logger.warning(f"failed to remove iteration workspace path {path}") + + +class _IterationWorkspaceAdapter: + """Per-iteration workspace with fresh mount/root/meta/log dirs.""" + + def __init__(self, real_workspace, iteration_root): + self._ws = real_workspace + self._twd = Path(iteration_root) + self._rootdir = self._twd / "root" + self._metadir = self._twd / "meta" + self._mountdir = self._twd / "mnt" + self._logdir = self._twd / "logs" + self._tmpdir = self._twd / "tmp" + self._rootdir.mkdir(parents=True, exist_ok=True) + self._metadir.mkdir(parents=True, exist_ok=True) + self._mountdir.mkdir(parents=True, exist_ok=True) + self._logdir.mkdir(parents=True, exist_ok=True) + self._tmpdir.mkdir(parents=True, exist_ok=True) + + @property + def twd(self): + return self._twd + + @property + def bindirs(self): + return self._ws.bindirs + + @property + def libdirs(self): + return self._ws.libdirs + + @property + def logdir(self): + return self._logdir + + @property + def rootdir(self): + return self._rootdir + + @property + def metadir(self): + return self._metadir + + @property + def mountdir(self): + return self._mountdir + + @property + def tmpdir(self): + return self._tmpdir + + +def _make_iteration_workspace(base_workspace, iteration_name): + iter_root = Path(tempfile.mkdtemp(prefix=f"{iteration_name}_", dir=str(base_workspace.twd))) + return _IterationWorkspaceAdapter(base_workspace, iter_root) + +def _set_client_hostfile(client, hostfile): + """Point harness client commands at the desired hosts file.""" + value = str(hostfile) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = value + + +def _wait_for_client_mount_ready(client, mountdir, timeout=45): + """Wait until the client can create and remove a file in the mount. + + Daemon processes can exist before the FUSE mount and RPC state are fully + ready. A fixed sleep is not enough on slower random-slicing/cutshift runs and + may make the first test file creation fail with EBUSY. + """ + mountdir = Path(mountdir) + deadline = time.time() + timeout + probe = mountdir / f".gkfs_ready_probe_{os.getpid()}_{int(time.time() * 1000)}" + last_error = None + + while time.time() < deadline: + try: + ret = client.open(probe, + os.O_CREAT | os.O_WRONLY | os.O_TRUNC, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if ret.retval != -1: + try: + client.unlink(probe) + except Exception: + pass + return + last_error = getattr(ret, 'errno', None) + except Exception as exc: + last_error = exc + time.sleep(1) + + pytest.fail(f"GekkoFS mount {mountdir} not ready after {timeout}s; last_error={last_error}") + + +def create_deterministic_file(client, mountdir, filename, size): + """Create deterministic random data in GekkoFS and return its MD5.""" + fpath = mountdir / filename + # Make repeated perf runs idempotent: if a previous run left the file in + # place, remove it before recreating it. + try: + client.unlink(fpath) + except Exception: + pass + + last_ret = None + seed = 0x474b4653 ^ sum(ord(ch) for ch in filename) ^ size + retry_errnos = {errno.EBUSY, errno.EDQUOT, errno.ENOSPC} + for attempt in range(30): + ret = client.write_random_and_md5(fpath, size, "--seed", seed, timeout=max(60, int(size / (1024 * 1024)) * 5)) + last_ret = ret + if ret.retval != -1: + break + last_errno = getattr(ret, 'errno', None) + if last_errno not in retry_errnos: + break + try: + usage = shutil.disk_usage(mountdir) + free = usage.free + except OSError: + free = "unknown" + logger.warning( + f"transient write failure for {fpath}, errno={last_errno} " + f"(attempt {attempt + 1}/30), retrying") + logger.debug( + f" mountdir exists={mountdir.exists()} " + f"ismount={os.path.ismount(mountdir)} free={free}") + time.sleep(1) + + assert last_ret is not None + assert last_ret.retval != -1, \ + f"write_random_and_md5 failed for {fpath}, errno={getattr(last_ret, 'errno', None)}" + assert last_ret.retval == size, \ + f"short write for {fpath}: wrote {last_ret.retval}, expected {size}, errno={last_ret.errno}" + assert last_ret.md5, f"write_random_and_md5 returned empty md5 for {fpath}" + + return fpath, last_ret.md5 + + +def describe_returncode(returncode): + """Return a readable subprocess return code description.""" + if returncode is None or returncode >= 0: + return str(returncode) + signum = -returncode + try: + signame = signal.Signals(signum).name + return f"{returncode} ({signame})" + except ValueError: + return str(returncode) + + +def count_accessible_files(file_paths, client, timeout=60): + """Count files visible through the GekkoFS client API.""" + if not file_paths: + return 0 + accessible = 0 + for fpath in file_paths: + for attempt in range(3): + result = client.stat(fpath, timeout=min(timeout, 15)) + if result.retval == 0: + accessible += 1 + break + if attempt < 2: + time.sleep(0.25) + return accessible + + +def verify_files_with_md5(file_md5_map, client, mountdir, max_read_checks=None): + """Read back files through gkfs.io and verify their MD5 checksums.""" + results = {'passed': 0, 'failed': 0, 'details': []} + items = list(file_md5_map.items()) + if max_read_checks is not None: + items = items[:max_read_checks] + + for idx, (fpath_str, expected_md5) in enumerate(items): + fpath = Path(fpath_str) + try: + if max_read_checks is not None and idx >= max_read_checks: + results['passed'] += 1 + continue + + ret = client.read_random_and_md5(fpath, timeout=30) + if ret.retval == -1: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'read_md5_failed', 'errno': ret.errno}) + continue + actual_md5 = ret.md5 + + if actual_md5 == expected_md5: + results['passed'] += 1 + else: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'md5_mismatch', + 'expected': expected_md5, 'actual': actual_md5}) + except Exception as e: + results['failed'] += 1 + results['details'].append({'file': fpath_str, 'status': 'exception', 'error': str(e)}) + + return results + + +def _extract_port(address): + """Extract the port number from a daemon address (e.g., 'lo:54321' -> '54321').""" + return address.rsplit(':', 1)[-1] + + +def _get_entry_from_hostfile(hostfile_path, port): + """Get the full entry line for a daemon port from a hostfile. + + Match the RPC URI port exactly. Do not use substring matching: random ports + such as 1234 and 31234 can otherwise select the wrong daemon entry and build + a corrupt marker workspace for mutate start. + """ + if not os.path.exists(hostfile_path): + return None + with open(hostfile_path) as f: + for line in f: + stripped = line.strip() + if not stripped or stripped.startswith('#'): + continue + + clean = stripped[1:] if stripped and stripped[0] in '+-' else stripped + fields = clean.split() + if len(fields) < 2: + continue + uri_port = fields[1].rsplit(':', 1)[-1] + if uri_port == port: + return clean if clean.endswith('\n') else clean + '\n' + return None + + +def _build_workspace_with_markers(workspace_path, daemons_to_write, marker_map=None): + """Build workspace hostfile with markers for malleability operations. + + Args: + workspace_path: Path to write + daemons_to_write: List of daemon objects + marker_map: dict of daemon -> marker ('+', '-', or '') + """ + marker_map = marker_map or {} + with open(workspace_path, 'w') as f: + for d in daemons_to_write: + marker = marker_map.get(d.address, '') + entry = _get_entry_from_hostfile(d.hostfile, _extract_port(d.address)) + if entry: + f.write(marker + entry) + else: + f.write(f"{marker}lo {d.address} 10000 10000 1 1 lo {d.address} 10000 10000\n") + + +def _build_workspace_for_shrink(workspace_path, active_daemons, removing_daemons): + """Build workspace for shrink: active (no marker) + removing ('-' marker).""" + marker_map = {} + for d in active_daemons: + marker_map[d.address] = '' + for d in removing_daemons: + marker_map[d.address] = '-' + + all_daemons = list(active_daemons) + list(removing_daemons) + _build_workspace_with_markers(workspace_path, all_daemons, marker_map) + + +def _build_workspace_for_expand(workspace_path, active_daemons, adding_daemons): + """Build workspace for expand: active (no marker) + adding ('+' marker).""" + marker_map = {} + for d in active_daemons: + marker_map[d.address] = '' + for d in adding_daemons: + marker_map[d.address] = '+' + + all_daemons = list(active_daemons) + list(adding_daemons) + _build_workspace_with_markers(workspace_path, all_daemons, marker_map) + + +# ========= +# Stats +# ========= + +@dataclass +class OperationResult: + run_index: int + elapsed_seconds: float + wall_time_seconds: float + data_redistributed_bytes: int + md5_passed: int + md5_failed: int + files_accessible: int + files_total: int + + +def compute_statistics(results): + """Compute statistical summary over multiple runs.""" + if not results: + return {"num_runs": 0} + + times = [r.elapsed_seconds for r in results] + wall_times = [r.wall_time_seconds for r in results] + + mean_elapsed = statistics.mean(times) + std_elapsed = statistics.stdev(times) if len(times) > 1 else 0.0 + mean_wall = statistics.mean(wall_times) + + if len(times) > 1: + se = std_elapsed / math.sqrt(len(times)) + t_values = {1: 12.71, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571} + t_val = t_values.get(len(times), 1.96) + ci_low = mean_elapsed - t_val * se + ci_high = mean_elapsed + t_val * se + else: + ci_low = ci_high = mean_elapsed + + cv = (std_elapsed / mean_elapsed * 100) if mean_elapsed > 0 else 0.0 + + moved_bytes = [r.data_redistributed_bytes for r in results] + moved_mb = [value / (1024**2) for value in moved_bytes] + moved_mean_mb = statistics.mean(moved_mb) + moved_stddev_mb = statistics.stdev(moved_mb) if len(moved_mb) > 1 else 0.0 + moved_cv = (moved_stddev_mb / moved_mean_mb * 100) if moved_mean_mb > 0 else 0.0 + + if len(moved_mb) > 1: + moved_p95_bytes = statistics.quantiles(moved_bytes, n=100, method='inclusive')[94] + moved_p95_mb = statistics.quantiles(moved_mb, n=100, method='inclusive')[94] + else: + moved_p95_bytes = moved_bytes[0] + moved_p95_mb = moved_mb[0] + + return { + "num_runs": len(results), + "mean_elapsed_seconds": round(mean_elapsed, 4), + "stddev_elapsed_seconds": round(std_elapsed, 4), + "min_elapsed_seconds": round(min(times), 4), + "max_elapsed_seconds": round(max(times), 4), + "ci_95_lower": round(ci_low, 4), + "ci_95_upper": round(ci_high, 4), + "coefficient_of_variation_pct": round(cv, 2), + "mean_wall_time_seconds": round(mean_wall, 4), + "total_data_redistributed_bytes": sum(moved_bytes), + "mean_data_redistributed_bytes": round(statistics.mean(moved_bytes), 2), + "stddev_data_redistributed_bytes": round(statistics.stdev(moved_bytes), 2) if len(moved_bytes) > 1 else 0.0, + "min_data_redistributed_bytes": min(moved_bytes), + "max_data_redistributed_bytes": max(moved_bytes), + "median_data_redistributed_bytes": round(statistics.median(moved_bytes), 2), + "p95_data_redistributed_bytes": round(moved_p95_bytes, 2), + "coefficient_of_variation_data_redistributed_pct": round(moved_cv, 2), + "total_data_redistributed_mb": round(sum(moved_mb), 4), + "mean_data_redistributed_mb": round(moved_mean_mb, 4), + "stddev_data_redistributed_mb": round(moved_stddev_mb, 4), + "min_data_redistributed_mb": round(min(moved_mb), 4), + "max_data_redistributed_mb": round(max(moved_mb), 4), + "median_data_redistributed_mb": round(statistics.median(moved_mb), 4), + "p95_data_redistributed_mb": round(moved_p95_mb, 4), + "all_md5_passed": all(r.md5_failed == 0 for r in results), + "all_files_accessible": all(r.files_accessible == r.files_total for r in results), + } + + +def compute_numeric_statistics(values, unit_suffix=""): + """Compute complete statistics for a numeric series.""" + if not values: + return {f"num_samples{unit_suffix}": 0} + + mean_value = statistics.mean(values) + stddev_value = statistics.stdev(values) if len(values) > 1 else 0.0 + cv = (stddev_value / mean_value * 100) if mean_value > 0 else 0.0 + p95_value = (statistics.quantiles(values, n=100, method='inclusive')[94] + if len(values) > 1 else values[0]) + + return { + f"num_samples{unit_suffix}": len(values), + f"total{unit_suffix}": round(sum(values), 4), + f"mean{unit_suffix}": round(mean_value, 4), + f"stddev{unit_suffix}": round(stddev_value, 4), + f"min{unit_suffix}": round(min(values), 4), + f"max{unit_suffix}": round(max(values), 4), + f"median{unit_suffix}": round(statistics.median(values), 4), + f"p95{unit_suffix}": round(p95_value, 4), + f"coefficient_of_variation_pct{unit_suffix}": round(cv, 2), + } + + +DEFAULT_REPETITIONS = int(os.environ.get("GKFS_PERF_REPETITIONS", "3")) +DEFAULT_NUM_FILES = int(os.environ.get("GKFS_PERF_NUM_FILES", "32")) +DEFAULT_FILE_SIZE = int(os.environ.get("GKFS_PERF_FILE_SIZE", str(8 * 1024))) +DEFAULT_OLD_NODES = int(os.environ.get("GKFS_MALLEABILITY_OLD_NODES", "4")) +DEFAULT_NEW_NODES = int(os.environ.get("GKFS_MALLEABILITY_NEW_NODES", "2")) +DEFAULT_TIMEOUT = 340 +MUTATE_STATUS_POLL_INTERVAL = 0.25 +CI_FAST = os.environ.get("GKFS_MALLEABILITY_CI_FAST", "").lower() in ("1", "on", "true", "yes") +KEEP_ITERATION_WORKSPACES = os.environ.get( + "GKFS_MALLEABILITY_KEEP_WORKSPACES", "").lower() in ("1", "on", "true", "yes") +EXPAND_ON_DEMAND = os.environ.get( + "GKFS_EXPAND_ON_DEMAND", "OFF").lower() in ("1", "on", "true", "yes") +EXPAND_ON_DEMAND_MD5_READ_CHECKS = int(os.environ.get( + "GKFS_EXPAND_ON_DEMAND_MD5_READ_CHECKS", "1")) + + +def _cleanup_iteration_workspace(iter_workspace): + if KEEP_ITERATION_WORKSPACES: + logger.info(f" Preserving iteration workspace: {iter_workspace.twd}") + return + shutil.rmtree(iter_workspace.twd.as_posix(), ignore_errors=True) + + +def _dump_mutate_failure_context(label, cmd, workspace_file): + logger.error(f"{label} failed with exit code {cmd.exit_code}") + logger.error(f"stdout:\n{cmd.stdout.decode(errors='replace')[:4000]}") + logger.error(f"stderr:\n{cmd.stderr.decode(errors='replace')[:4000]}") + workspace_file = Path(workspace_file) + if workspace_file.exists(): + logger.error(f"workspace {workspace_file}:\n{workspace_file.read_text(errors='replace')[:4000]}") + else: + logger.error(f"workspace {workspace_file} does not exist") + + log_root = workspace_file.parent / "logs" + if log_root.exists(): + for daemon_log in sorted(log_root.glob("daemon_*/gkfs_daemon.log")): + try: + logger.error(f"tail {daemon_log}:\n{daemon_log.read_text(errors='replace')[-4000:]}") + except OSError as exc: + logger.error(f"failed to read {daemon_log}: {exc}") + + +def _run_malleability(gkfs_shell, action, workspace_file, timeout=60): + """Run gkfs_malleability without an extra shell process.""" + return gkfs_shell.run( + "gkfs_malleability", + "mutate", + action, + timeout=timeout, + env={ + "LIBGKFS_HOSTS_FILE": str(workspace_file), + "LD_PRELOAD": "", + }, + ) + + +# ============ +# Test: shrink +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_shrink_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple shrink operations and collect statistical performance data. + + MARKER-BASED WORKFLOW: + - Workspace contains ALL daemons (active + removing) + - Removing daemons get '-' markers + - CLI computes old_nodes = active + removing, new_nodes = active + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = DEFAULT_OLD_NODES + num_end = DEFAULT_NEW_NODES + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + + for run_idx in range(num_reps): + logger.info("=" * 70) + logger.info(f"SHRINK PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"shrink_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) + daemons = [] + run_mountdir = None + try: + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Create daemons + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"shrink_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + logger.info(f" Generated {num_files} files in {gen_time:.3f}s") + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Pick survivors randomly + survivors = random.sample(daemons, num_end) + removing_daemons = [d for d in daemons if d not in survivors] + + # Build workspace with markers: all active + removing ('-' marker) + workspace_file = hostfile.parent / f"shrink_workspace_run{run_idx}.txt" + _build_workspace_for_shrink(workspace_file, survivors, removing_daemons) + + env_file = hostfile.parent / f"test_env_shrink_run{run_idx}.sh" + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + ) + + logger.info(f" Shrink: {len(survivors)} keep, {len(removing_daemons)} remove") + + # Execute shrink + t0 = perf_counter() + t_wall = time.time() + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Shrink mutate start", cmd, workspace_file) + assert cmd.exit_code == 0, ( + "Shrink start failed:\n" + f"stdout: {cmd.stdout.decode(errors='replace')[:1000]}\n" + f"stderr: {cmd.stderr.decode(errors='replace')[:1000]}" + ) + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_malleability(gkfs_shell, "status", workspace_file) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + else: + pytest.fail(f"Shrink did not complete within 120s (run {run_idx + 1})") + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + + # Verify + accessible = count_accessible_files(created_files, client) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + if post_verify['failed']: + logger.warning(f" Shrink post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Shrink redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: shrink took {elapsed:.3f}s, files: {accessible}/{num_files}") + + finally: + _shutdown_daemons(daemons) + if run_mountdir is not None: + _wait_for_unmounted(run_mountdir) + _cleanup_iteration_workspace(iter_workspace) + + + stats = compute_statistics(all_results) + logger.info("=" * 70) + logger.info("SHRINK PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("SHRINK PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + _shutdown_daemons(all_daemons) + for d in all_daemons: + _cleanup_daemon_storage(d) + + + +# ============ +# Test: expand +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_expand_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple expand operations and collect statistical performance data. + + MARKER-BASED WORKFLOW: + - Workspace contains ALL daemons (active + adding) + - Adding daemons get '+' markers + - CLI computes old_nodes = active, new_nodes = active + adding + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = 1 if CI_FAST else 2 + num_end = 2 if CI_FAST else 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + expand_on_demand_value = "ON" if EXPAND_ON_DEMAND else "OFF" + + for run_idx in range(num_reps): + logger.info("=" * 70) + logger.info(f"EXPAND PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info(f" GKFS_EXPAND_ON_DEMAND={expand_on_demand_value}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"expand_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + client._patched_env["GKFS_EXPAND_ON_DEMAND"] = expand_on_demand_value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + client._env["GKFS_EXPAND_ON_DEMAND"] = expand_on_demand_value + + # Create initial daemons + daemons = [] + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"expand_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Expand: create new daemons, then build a marker workspace with active + # daemons and '+' adding daemons. The daemon auto-write path is tested + # elsewhere; this perf test uses an explicit workspace for repeatability. + new_daemons = [] + for i in range(num_end - num_start): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + all_daemons.extend(new_daemons) + logger.info(f" Expand: {len(daemons)} active, {len(new_daemons)} adding") + + workspace_file = hostfile.parent / f"expand_workspace_run{run_idx}.txt" + _build_workspace_for_expand(workspace_file, daemons, new_daemons) + + # Verify '+' entries appeared + with open(workspace_file) as f: + content = f.read() + adding_entries = [l for l in content.split('\n') if l.startswith('+')] + logger.info(f" Expand: {len(adding_entries)} '+' entries in marker workspace") + assert len(adding_entries) == len(new_daemons), \ + f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" + + env_file = hostfile.parent / f"test_env_expand_run{run_idx}.sh" + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + f'export GKFS_EXPAND_ON_DEMAND="{expand_on_demand_value}"\n' + ) + + logger.info(f" Expand: {len(daemons)} active, {len(new_daemons)} adding") + + # Execute expand + t0 = perf_counter() + t_wall = time.time() + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Expand mutate start", cmd, workspace_file) + assert cmd.exit_code == 0, f"Expand start failed: {cmd.stderr.decode()[:300]}" + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_malleability(gkfs_shell, "status", workspace_file) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + else: + pytest.fail(f"Expand did not complete within 120s (run {run_idx + 1})") + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + + # Verify + accessible = count_accessible_files(created_files, client) + max_read_checks = (EXPAND_ON_DEMAND_MD5_READ_CHECKS + if EXPAND_ON_DEMAND else None) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=max_read_checks) + if post_verify['failed']: + logger.warning(f" Expand post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons + new_daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Expand redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: expand took {elapsed:.3f}s, files: {accessible}/{num_files}") + + # Clean up + _shutdown_daemons(daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + + stats = compute_statistics(all_results) + logger.info("=" * 70) + logger.info("EXPAND PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("EXPAND PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + +# ============ +# Test: mutate +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_mutate_performance_multi_run(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple mutate (4→4 with different hosts) operations. + + For same-count topology change: + - Old daemons get '-' markers (will be removed) + - New daemons get '+' markers (will be added) + - Workspace contains ALL daemons + - CLI computes old_nodes = removing, new_nodes = adding + """ + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_start = 2 if CI_FAST else 4 + num_end = 2 if CI_FAST else 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_results = [] + all_daemons = [] + + for run_idx in range(num_reps): + logger.info("=" * 70) + logger.info(f"MUTATE PERFORMANCE TEST - RUN {run_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"mutate_iter_{run_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Create initial daemons + old_daemons = [] + for i in range(num_start): + d = iter_daemon_factory.create(enable_forwarding=False) + old_daemons.append(d) + all_daemons.extend(old_daemons) + + hostfile = Path(old_daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = old_daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + # Generate data + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"mutate_perf_{run_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + before_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in old_daemons} + + # Create replacement daemons + new_daemons = [] + for i in range(num_end): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + all_daemons.extend(new_daemons) + + # For same-count mutate: all old get '-' markers, all new get '+' markers + workspace_file = hostfile.parent / f"mutate_workspace_run{run_idx}.txt" + marker_map = {} + for d in old_daemons: + marker_map[d.address] = '-' + for d in new_daemons: + marker_map[d.address] = '+' + all_daemons_to_write = list(old_daemons) + list(new_daemons) + _build_workspace_with_markers(workspace_file, all_daemons_to_write, marker_map) + + env_file = hostfile.parent / f"test_env_mutate_run{run_idx}.sh" + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_file}"\n' + ) + + logger.info(f" Mutate: {len(old_daemons)} removing, {len(new_daemons)} adding") + + # Execute mutate + t0 = perf_counter() + t_wall = time.time() + cmd = _run_malleability( + gkfs_shell, "start", workspace_file, DEFAULT_TIMEOUT + ) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Mutate start", cmd, workspace_file) + result = OperationResult( + run_index=run_idx, elapsed_seconds=0, wall_time_seconds=0, + data_redistributed_bytes=0, md5_passed=0, md5_failed=0, + files_accessible=0, files_total=num_files, + ) + all_results.append(result) + _shutdown_daemons(old_daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in old_daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + continue + + # Wait for completion + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_malleability(gkfs_shell, "status", workspace_file) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + + elapsed = perf_counter() - t0 + wall = time.time() - t_wall + + # Finalize + cmd = _run_malleability(gkfs_shell, "finalize", workspace_file) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_file) + + # Verify + accessible = count_accessible_files(created_files, client) + post_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + if post_verify['failed']: + logger.warning(f" Mutate post-verify failures: {post_verify['details'][:5]}") + + after_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in old_daemons + new_daemons} + moved_bytes = _count_moved_chunk_bytes(before_snapshots, after_snapshots) + logger.info(f" Mutate redistributed {moved_bytes / (1024 ** 2):.4f} MB") + + result = OperationResult( + run_index=run_idx, + elapsed_seconds=elapsed, + wall_time_seconds=wall, + data_redistributed_bytes=moved_bytes, + md5_passed=post_verify['passed'], + md5_failed=post_verify['failed'], + files_accessible=accessible, + files_total=num_files, + ) + all_results.append(result) + + logger.info(f" Run {run_idx + 1}: mutate took {elapsed:.3f}s, files: {accessible}/{num_files}") + + # Clean up + _shutdown_daemons(old_daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in old_daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + + # Use all results (mutate can complete in <1s) + completed = all_results + stats = compute_statistics(completed) + logger.info("=" * 70) + logger.info("MUTATE PERFORMANCE - STATISTICAL SUMMARY") + logger.info("=" * 70) + logger.info(json.dumps(stats, indent=2)) + # Print to stdout for parse_gkfs_perf.py to capture via tee + print("=" * 70) + print("MUTATE PERFORMANCE - STATISTICAL SUMMARY") + print("=" * 70) + print(json.dumps(stats, indent=2)) + + _shutdown_daemons(all_daemons) + for d in all_daemons: + _cleanup_daemon_storage(d) + assert stats['num_runs'] == num_reps + assert stats['all_files_accessible'] + assert stats['all_md5_passed'] + + + +# ============ +# Test: cycle +# ============ + +@pytest.mark.parametrize("client_fixture", ["gkfs_client"]) +def test_comprehensive_cycle_performance(client_fixture, + gkfwd_daemon_factory, + gkfs_shell, + request): + """Run multiple shrink→expand→mutate cycles with full metrics.""" + if CI_FAST: + pytest.skip("skip heavy cycle performance test in CI fast mode") + client = request.getfixturevalue(client_fixture) + + num_reps = DEFAULT_REPETITIONS + num_initial = 4 + num_shrink_to = 2 + num_expand_to = 4 + num_files = DEFAULT_NUM_FILES + file_size = DEFAULT_FILE_SIZE + + all_cycle_results = [] + all_daemons = [] + + for rep_idx in range(num_reps): + logger.info("=" * 70) + logger.info(f"COMPREHENSIVE CYCLE TEST - REPLICATION {rep_idx + 1}/{num_reps}") + logger.info("=" * 70) + + base_workspace = request.getfixturevalue("test_workspace") + iter_workspace = _make_iteration_workspace(base_workspace, f"cycle_iter_{rep_idx}") + iter_daemon_factory = FwdDaemonCreator(request.config.getoption('--interface'), iter_workspace, keep_hosts=True) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = str(iter_workspace.twd / "gkfs_hosts.txt") + + # Step 1: Start daemons and generate data + daemons = [] + for i in range(num_initial): + d = iter_daemon_factory.create(enable_forwarding=False) + daemons.append(d) + all_daemons.extend(daemons) + + hostfile = Path(daemons[0].hostfile) + _set_client_hostfile(client, hostfile) + run_mountdir = daemons[0].mountdir + _wait_for_client_mount_ready(client, run_mountdir) + + file_md5_map = {} + created_files = [] + t0 = perf_counter() + for i in range(num_files): + fname = f"cycle_{rep_idx}_{i:03d}" + fpath, md5 = create_deterministic_file(client, run_mountdir, fname, file_size) + file_md5_map[str(fpath)] = md5 + created_files.append(fpath) + gen_time = perf_counter() - t0 + + pre_verify = verify_files_with_md5(file_md5_map, client, run_mountdir) + assert pre_verify['failed'] == 0 + logger.info(f" Step 1: Generated {num_files} files in {gen_time:.3f}s") + before_shrink_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + + # Step 2: Shrink + survivors = random.sample(daemons, num_shrink_to) + removing_daemons = [d for d in daemons if d not in survivors] + + workspace_shrink = hostfile.parent / f"cycle_shrink_workspace_{rep_idx}.txt" + _build_workspace_for_shrink(workspace_shrink, survivors, removing_daemons) + + env_file = hostfile.parent / f"test_env_cycle{rep_idx}.sh" + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_shrink}"\n' + ) + + shrink_t0 = perf_counter() + cmd = _run_malleability( + gkfs_shell, "start", workspace_shrink, DEFAULT_TIMEOUT + ) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Cycle shrink mutate start", cmd, workspace_shrink) + assert cmd.exit_code == 0, f"Cycle shrink start failed: {cmd.stderr.decode()[:300]}" + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_malleability(gkfs_shell, "status", workspace_shrink) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + + shrink_elapsed = perf_counter() - shrink_t0 + cmd = _run_malleability(gkfs_shell, "finalize", workspace_shrink) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_shrink) + + post_shrink_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + logger.info(f" Step 2: Shrink took {shrink_elapsed:.3f}s") + if post_shrink_verify['failed']: + logger.warning(f" Cycle shrink post-verify failures: {post_shrink_verify['details'][:5]}") + after_shrink_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in daemons} + shrink_moved_bytes = _count_moved_chunk_bytes(before_shrink_snapshots, + after_shrink_snapshots) + logger.info(f" Cycle shrink redistributed {shrink_moved_bytes / (1024 ** 2):.4f} MB") + + # Clean up removed daemons + _shutdown_daemons(removing_daemons) + for d in removing_daemons: + _cleanup_daemon_storage(d) + + surviving_daemons = survivors + before_expand_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in surviving_daemons} + + # Step 3: Expand with marker workspace + logger.info(f" Step 3: Expand {len(surviving_daemons)} -> {num_expand_to}") + + new_daemons = [] + for i in range(num_expand_to - num_shrink_to): + d = iter_daemon_factory.create(expand_mode=True, + enable_forwarding=False) + new_daemons.append(d) + all_daemons.extend(new_daemons) + + workspace_expand = hostfile.parent / f"cycle_expand_workspace_{rep_idx}.txt" + _build_workspace_for_expand(workspace_expand, surviving_daemons, + new_daemons) + + # Verify '+' entries appeared + with open(workspace_expand) as f: + content = f.read() + adding_entries = [l for l in content.split('\n') if l.startswith('+')] + logger.info(f" Expand: {len(adding_entries)} '+' entries in marker workspace") + assert len(adding_entries) == len(new_daemons), \ + f"Expected {len(new_daemons)} '+' entries, got {len(adding_entries)}" + + env_file.write_text( + f'export LIBGKFS_HOSTS_FILE="{workspace_expand}"\n' + ) + + expand_t0 = perf_counter() + cmd = _run_malleability( + gkfs_shell, "start", workspace_expand, DEFAULT_TIMEOUT + ) + if cmd.exit_code != 0: + _dump_mutate_failure_context("Cycle expand mutate start", cmd, workspace_expand) + assert cmd.exit_code == 0, ( + "Cycle expand start failed:\n" + f"cmd: gkfs_malleability mutate start\n" + f"stdout: {cmd.stdout.decode(errors='replace')[:1000]}\n" + f"stderr: {cmd.stderr.decode(errors='replace')[:1000]}\n" + f"workspace:\n{workspace_expand.read_text(errors='replace')[:2000]}" + ) + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_malleability(gkfs_shell, "status", workspace_expand) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(MUTATE_STATUS_POLL_INTERVAL) + + expand_elapsed = perf_counter() - expand_t0 + cmd = _run_malleability(gkfs_shell, "finalize", workspace_expand) + assert cmd.exit_code == 0 + _set_client_hostfile(client, workspace_expand) + + post_expand_verify = verify_files_with_md5(file_md5_map, client, run_mountdir, + max_read_checks=1) + logger.info(f" Step 3: Expand took {expand_elapsed:.3f}s") + if post_expand_verify['failed']: + logger.warning(f" Cycle expand post-verify failures: {post_expand_verify['details'][:5]}") + after_expand_snapshots = {d: _snapshot_chunk_files(d.rootdir) for d in surviving_daemons + new_daemons} + expand_moved_bytes = _count_moved_chunk_bytes(before_expand_snapshots, + after_expand_snapshots) + logger.info(f" Cycle expand redistributed {expand_moved_bytes / (1024 ** 2):.4f} MB") + + # Clean up + _shutdown_daemons(daemons + new_daemons) + _wait_for_unmounted(run_mountdir) + for d in daemons + new_daemons: + _cleanup_daemon_storage(d) + _cleanup_iteration_workspace(iter_workspace) + + all_cycle_results.append({ + "replication": rep_idx, + "shrink_elapsed": round(shrink_elapsed, 4), + "expand_elapsed": round(expand_elapsed, 4), + "shrink_data_redistributed_mb": round(shrink_moved_bytes / (1024 ** 2), 4), + "expand_data_redistributed_mb": round(expand_moved_bytes / (1024 ** 2), 4), + "pre_verify_passed": pre_verify['passed'], + "post_shrink_md5_passed": post_shrink_verify['passed'], + "post_expand_md5_passed": post_expand_verify['passed'], + }) + + logger.info("=" * 70) + logger.info("COMPREHENSIVE CYCLE TEST - SUMMARY") + logger.info("=" * 70) + for r in all_cycle_results: + logger.info(json.dumps(r, indent=2)) + cycle_stats = { + "num_runs": len(all_cycle_results), + "shrink_data_redistributed_mb_stats": compute_numeric_statistics( + [r["shrink_data_redistributed_mb"] for r in all_cycle_results], "_mb"), + "expand_data_redistributed_mb_stats": compute_numeric_statistics( + [r["expand_data_redistributed_mb"] for r in all_cycle_results], "_mb"), + "total_data_redistributed_mb_stats": compute_numeric_statistics( + [r["shrink_data_redistributed_mb"] + r["expand_data_redistributed_mb"] + for r in all_cycle_results], "_mb"), + } + logger.info("COMPREHENSIVE CYCLE DATA MOVEMENT STATS") + logger.info(json.dumps(cycle_stats, indent=2)) + print("=" * 70) + print("COMPREHENSIVE CYCLE DATA MOVEMENT STATS") + print("=" * 70) + print(json.dumps(cycle_stats, indent=2)) + + assert len(all_cycle_results) == num_reps + assert all(r["post_shrink_md5_passed"] == num_files for r in all_cycle_results) + assert all(r["post_expand_md5_passed"] == num_files for r in all_cycle_results) + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--tb=long"]) \ No newline at end of file diff --git a/tests/integration/malleability/test_malleability_tool.py b/tests/integration/malleability/test_malleability_tool.py index 8f3a3296ef7aac24ee6a5bdfbeb616bb3919713a..0a208757b8e677b1658bb3c69f640412e34f8f0c 100644 --- a/tests/integration/malleability/test_malleability_tool.py +++ b/tests/integration/malleability/test_malleability_tool.py @@ -28,176 +28,192 @@ import harness from pathlib import Path -import shutil -import errno import stat import os -import ctypes -import sh -import sys import pytest from harness.logger import logger -nonexisting = "nonexisting" def test_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): import time + + # Start with 1 daemon (before expand) d00 = gkfwd_daemon_factory.create() - # Add "#FS_INSTANCE_END" in the file with name d00.hostfile - time.sleep(5) - with open(d00.hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") - - # loop 10 times, and create a file in each iteration + hostfile = Path(d00.hostfile) + # Create some files before expand for i in range(4): - file = d00.mountdir / f"file{i}" - # create a file in gekkofs - ret = gkfs_client.open(file, + fpath = d00.mountdir / f"file{i}" + ret = gkfs_client.open(fpath, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1 - - ret = gkfs_client.write_validate(file, 1024 * 1024) + ret = gkfs_client.write_validate(fpath, 1024 * 1024) assert ret.retval == 0 - # Create content + # Create new daemon to add (after expand) + d01 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) - d01 = gkfwd_daemon_factory.create() - libdirs = gkfs_shell._patched_env['LD_LIBRARY_PATH'] - search_path = ':'.join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which('gkfs_malleability', path=search_path) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand status" + # Marker-based API: LIBGKFS_HOSTS_FILE = workspace hostfile with markers + # - d00 stays (no prefix) + # - d01 added with '+' prefix + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + + # Write d00's entry (no prefix = active) and d01's entry with '+' prefix (to add) + with open(hostfile, "w") as hf_out: + # Write d00's entry as-is (active) + for line in original_lines: + if not line.startswith("#") and line.strip(): + hf_out.write(line) + # Add d01's entry with '+' prefix (to-be-added daemon) + with open(d01.hostfile, "r") as d01_hf: + for line in d01_hf: + if not line.startswith("#"): + hf_out.write("+" + line) + + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"Command '{cmd_str}' failed with {cmd.exit_code}: {cmd.stderr.decode()}" - assert "No expansion running/finished.\n" in cmd.stderr.decode() - - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand start" + assert cmd.exit_code == 0, f"Command '{cmd_str}' failed: {cmd.stderr.decode()}" + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) - + assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" + assert "Mutate process" in cmd.stderr.decode() + time.sleep(10) - cmd_str = f"LD_LIBRARY_PATH={libdirs} LIBGKFS_HOSTS_FILE={d00.hostfile} {malleability_bin} expand finalize" + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate finalize" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - + assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" + d00.shutdown() d01.shutdown() def test_malleability_failures(gkfwd_daemon_factory, gkfs_client, gkfs_shell): import time + d00 = gkfwd_daemon_factory.create() - # Add "#FS_INSTANCE_END" in the file with name d00.hostfile time.sleep(5) - cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + + + hostfile = Path(d00.hostfile) + + # Test error when old == new node count (no markers, same hosts) + cmd = gkfs_shell.gkfs_malleability('mutate', 'start', timeout=340) assert cmd.exit_code != 0 - assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" - with open(d00.hostfile, 'a') as f: - f.write("#FS_INSTANCE_END\n") + # Test error when hostfile has markers for same daemon (removing all) + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + + # Mark all entries with '-' (would remove all daemons) + with open(hostfile, "w") as hf_out: + for line in original_lines: + if not line.startswith("#") and line.strip(): + hf_out.write("-" + line) + else: + hf_out.write(line) - cmd = gkfs_shell.gkfs_malleability('expand','start', timeout=340) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate start" + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) assert cmd.exit_code != 0 - assert cmd.stderr.decode() == "ERR: Old server configuration is the same as the new one\n" + assert "All hosts would be removed" in cmd.stderr.decode() or "Error" in cmd.stderr.decode() + d00.shutdown() + def test_shrink_malleability(gkfwd_daemon_factory, gkfs_client, gkfs_shell): - """Test that shrinking from 2 nodes to 1 redistributes data correctly.""" + """Test that shrinking from 2 nodes to 1 redistributes data correctly. + + Uses the unified 'mutate' subcommand (shrinking = fewer nodes after). + """ import time + d00 = gkfwd_daemon_factory.create() d01 = gkfwd_daemon_factory.create() time.sleep(5) - # Create several files across the 2-node cluster so that chunks land on both nodes + # Create several files across the 2-node cluster for i in range(8): - f = d00.mountdir / f"shrink_file_{i}" - ret = gkfs_client.open(f, os.O_CREAT | os.O_WRONLY, + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.open(fpath, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - assert ret.retval != -1, f"open failed for {f}" - ret = gkfs_client.write_validate(f, 1024 * 1024) - assert ret.retval == 0, f"write_validate failed for {f}" - - # Build gkfs_hosts_new.txt containing only d00 (remove d01). - # d00.address is "iface:PORT" (e.g. "lo:15680"); extract the port and - # match it against the URI written in the hosts file - # ("ofi+sockets://127.0.0.1:PORT ..."). Each daemon gets a unique - # ephemeral port so this is a reliable discriminator. + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + old_hostfile = Path(d00.hostfile) - new_hostfile = old_hostfile.parent / "gkfs_hosts_new.txt" + + # Marker-based API: mark d01 with '-' prefix (to remove) d00_port = d00.address.split(":")[-1] - with open(old_hostfile) as hf_in, open(new_hostfile, "w") as hf_out: - for line in hf_in: - # Keep lines that belong to d00; skip d01 and comment lines - if not line.startswith("#") and f":{d00_port}" in line: - hf_out.write(line) - assert new_hostfile.exists(), "New hosts file was not created" - assert new_hostfile.stat().st_size > 0, \ - f"New hosts file is empty; d00 port '{d00_port}' not found in {old_hostfile}" + d01_port = d01.address.split(":")[-1] - libdirs = gkfs_shell._patched_env["LD_LIBRARY_PATH"] - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) + with open(old_hostfile, 'r') as hf_in: + original_lines = hf_in.readlines() - # Check status before starting (should report nothing running) + # Write d00's entry (active, no prefix) and d01's entry with '-' prefix (to remove) + with open(old_hostfile, 'w') as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + # Check if this line belongs to d01 + if f":{d01_port}" in stripped: + hf_out.write("-" + line) # Mark for removal + else: + hf_out.write(line) # Keep as active + + # Check status before starting (no --new-hosts-file needed) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} status" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0 - assert "No shrink running/finished." in cmd.stderr.decode() + assert "No mutate running/finished." in cmd.stderr.decode() - # Start shrink; node counts are auto-detected from both hostfiles + # Start mutate (shrink = fewer nodes) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} start" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate start" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=340) - assert cmd.exit_code == 0, f"shrink start failed: {cmd.stderr.decode()}" - assert "Shrink process from 2 nodes to 1 nodes launched" in cmd.stderr.decode() + assert cmd.exit_code == 0, f"mutate start failed: {cmd.stderr.decode()}" + assert "Mutate process" in cmd.stderr.decode() - # Wait for redistribution to complete (poll with timeout) + # Wait for redistribution (no --new-hosts-file needed) deadline = time.time() + 120 while time.time() < deadline: cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} status" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - if "No shrink running/finished." in cmd.stderr.decode(): + if "No mutate running/finished." in cmd.stderr.decode(): break time.sleep(2) else: - pytest.fail("Shrink redistribution did not finish within 120 s") + pytest.fail("Mutate redistribution did not finish within 120 s") - # Finalize: gkfs_hosts_new.txt is renamed to gkfs_hosts.txt (old_hostfile) + # Finalize (no --new-hosts-file needed) cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={old_hostfile} " - f"{malleability_bin} shrink --new-hosts-file {new_hostfile} finalize" + f"LIBGKFS_HOSTS_FILE={old_hostfile} gkfs_malleability mutate finalize" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"shrink finalize failed: {cmd.stderr.decode()}" - assert "Hosts file updated" in cmd.stderr.decode() - - # Postcondition: new file is gone, old path now holds the new config - assert not new_hostfile.exists(), "gkfs_hosts_new.txt should have been renamed" - assert old_hostfile.exists(), "hosts file should still exist with updated content" + assert cmd.exit_code == 0, f"mutate finalize failed: {cmd.stderr.decode()}" - # Verify all data is still accessible on the surviving node d00 + # Verify all data accessible for i in range(8): - f = d00.mountdir / f"shrink_file_{i}" - ret = gkfs_client.stat(f) - assert ret.retval == 0, f"stat failed for {f} after shrink" + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after mutate" d00.shutdown() d01.shutdown() diff --git a/tests/integration/malleability/test_malleability_tool_simple.py b/tests/integration/malleability/test_malleability_tool_simple.py new file mode 100644 index 0000000000000000000000000000000000000000..49a283337a85284dcd87fc31e5d0e1019f210796 --- /dev/null +++ b/tests/integration/malleability/test_malleability_tool_simple.py @@ -0,0 +1,318 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Simple, easy-to-verify malleability tool tests. + +These tests cover three scenarios: + 1. ADD - Expand: add 1 node to a 1-node cluster + 2. REMOVE - Shrink: remove 1 node from a 2-node cluster + 3. CHANGE - Mutate: swap 1 node for another (remove + add in one operation) + +Each test is standalone and can be run by the user independently: + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_expand + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_shrink + pytest tests/integration/malleability/test_malleability_tool_simple.py::test_mutate_swap +""" + +import os +import stat +import time +from pathlib import Path +import pytest + + + +def _run_cmd(hosts_file, args, gkfs_shell, timeout=None): + """Run a gkfs_malleability command and assert success.""" + cmd_str = f"LIBGKFS_HOSTS_FILE={hosts_file} gkfs_malleability {args}" + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + assert cmd.exit_code == 0, ( + f"gkfs_malleability command failed: {cmd_str}\n" + f"stderr: {cmd.stderr.decode()}" + ) + return cmd + + +def _write_clean_hostfile(marker_hostfile, clean_hostfile): + """Write final active topology from a marker hostfile.""" + with open(marker_hostfile, "r") as hf_in, open(clean_hostfile, "w") as hf_out: + for line in hf_in: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + elif stripped.startswith("-"): + continue + elif stripped.startswith("+"): + hf_out.write(line[1:]) + else: + hf_out.write(line) + + +def _set_client_hostfile(client, hostfile): + value = str(hostfile) + if hasattr(client, "_patched_env"): + client._patched_env["LIBGKFS_HOSTS_FILE"] = value + if hasattr(client, "_env"): + client._env["LIBGKFS_HOSTS_FILE"] = value + + +def test_expand(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: ADD (expand) + Start with 1 daemon, add a 2nd, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + time.sleep(5) + hostfile = Path(d00.hostfile) + + for i in range(4): + fpath = d00.mountdir / f"expand_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + d01 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) + d01_port = d01.address.split(":")[-1] + mutate_hostfile = hostfile.parent / "gkfs_hosts_expand_mutate.txt" + with open(hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + with open(mutate_hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + clean_hostfile = hostfile.parent / "gkfs_hosts_expand_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) + + for i in range(4): + fpath = d00.mountdir / f"expand_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after expand" + + with open(clean_hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + + d00.shutdown() + d01.shutdown() + + +def test_shrink(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: REMOVE (shrink) + Start with 2 daemons, remove 1, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + time.sleep(5) + + for i in range(8): + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + old_hostfile = Path(d00.hostfile) + d01_port = d01.address.split(":")[-1] + with open(old_hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + mutate_hostfile = old_hostfile.parent / "gkfs_hosts_shrink_mutate.txt" + with open(mutate_hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("-" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + clean_hostfile = old_hostfile.parent / "gkfs_hosts_shrink_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) + + for i in range(8): + fpath = d00.mountdir / f"shrink_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after shrink" + + with open(clean_hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + assert f":{d01_port}" not in content, f"Removed daemon {d01_port} still in hostfile" + + d00.shutdown() + d01.shutdown() + d00.shutdown() + d01.shutdown() + + +def test_mutate_swap(gkfwd_daemon_factory, gkfs_client, gkfs_shell): + """ + Scenario: CHANGE (mutate swap) + Start with 2 daemons, remove 1 and add 1, verify data survives. + """ + d00 = gkfwd_daemon_factory.create() + d01 = gkfwd_daemon_factory.create() + time.sleep(5) + + for i in range(4): + fpath = d00.mountdir / f"mutate_file_{i}" + ret = gkfs_client.open( + fpath, os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + ) + assert ret.retval != -1, f"open failed for {fpath}" + ret = gkfs_client.write_validate(fpath, 1024 * 1024) + assert ret.retval == 0, f"write_validate failed for {fpath}" + + old_hostfile = Path(d00.hostfile) + + d02 = gkfwd_daemon_factory.create(expand_mode=True) + time.sleep(2) + + d01_port = d01.address.split(":")[-1] + d02_port = d02.address.split(":")[-1] + mutate_hostfile = old_hostfile.parent / "gkfs_hosts_mutate_swap.txt" + with open(old_hostfile, "r") as hf_in: + original_lines = hf_in.readlines() + with open(mutate_hostfile, "w") as hf_out: + for line in original_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + hf_out.write(line) + continue + if f":{d01_port}" in stripped: + hf_out.write("-" + line) + elif f":{d02_port}" in stripped: + hf_out.write("+" + line) + else: + hf_out.write(line) + + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + assert "No mutate running/finished." in cmd.stderr.decode() + + cmd = _run_cmd(mutate_hostfile, "mutate start", gkfs_shell, timeout=340 + ) + assert "Mutate process" in cmd.stderr.decode() + + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_cmd(mutate_hostfile, "mutate status", gkfs_shell) + if "No mutate running/finished." in cmd.stderr.decode(): + break + time.sleep(2) + else: + pytest.fail("Mutate redistribution did not finish within 120 s") + + cmd = _run_cmd(mutate_hostfile, "mutate finalize", gkfs_shell) + assert cmd.exit_code == 0 + clean_hostfile = old_hostfile.parent / "gkfs_hosts_mutate_swap_clean.txt" + _write_clean_hostfile(mutate_hostfile, clean_hostfile) + _set_client_hostfile(gkfs_client, clean_hostfile) + + for i in range(4): + fpath = d00.mountdir / f"mutate_file_{i}" + ret = gkfs_client.stat(fpath) + assert ret.retval == 0, f"stat failed for {fpath} after mutate swap" + + with open(clean_hostfile, "r") as hf: + content = hf.read() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + assert f":{d01_port}" not in content, f"Removed daemon {d01_port} still in hostfile" + + d00.shutdown() + d01.shutdown() + d02.shutdown() + diff --git a/tests/integration/malleability/test_mutate_distributors_integrity.py b/tests/integration/malleability/test_mutate_distributors_integrity.py new file mode 100644 index 0000000000000000000000000000000000000000..e6543f447480e82dc00fb620e6745220eaa64063 --- /dev/null +++ b/tests/integration/malleability/test_mutate_distributors_integrity.py @@ -0,0 +1,249 @@ +################################################################################ +# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +""" +Mutate integration tests for distributor correctness. + +The matrix below checks that mutate can add, remove, and replace nodes for both +the default/simple-hash distributor and the random-slicing distributor. Each +case verifies: + +* data written before mutate is still readable and byte-identical; +* new data can be written/read after finalize; +* the hostfile markers are consumed by finalize; +* removed daemons are absent from the finalized hostfile; +* chunks exist after redistribution and are not left only on a removed daemon. +""" + +import hashlib +import os +import stat +import time +from pathlib import Path + +import pytest + + +FILE_COUNT = 6 +FILE_SIZE = 64 * 1024 + + + +def _run_mutate_cmd(gkfs_shell, hosts_file, args, timeout=120, logdir=None): + cmd_str = f'LIBGKFS_HOSTS_FILE="{hosts_file}" gkfs_malleability {args}' + cmd = gkfs_shell.script(cmd_str, intercept_shell=False, timeout=timeout) + diagnostics = "" + if logdir is not None: + daemon_log = Path(logdir) / "gkfs_daemon.log" + client_log = Path(logdir) / "gkfs_client.log" + diagnostics = ( + f"\nhostfile:\n{Path(hosts_file).read_text() if Path(hosts_file).exists() else ''}" + f"\ndaemon log tail:\n{daemon_log.read_text()[-8000:] if daemon_log.exists() else ''}" + f"\nclient log tail:\n{client_log.read_text()[-8000:] if client_log.exists() else ''}" + ) + assert cmd.exit_code == 0, ( + f"gkfs_malleability {args} failed\n" + f"stdout: {cmd.stdout.decode()}\n" + f"stderr: {cmd.stderr.decode()}" + f"{diagnostics}" + ) + return cmd + + +def _extract_port(daemon): + return daemon.address.rsplit(":", 1)[-1] + + +def _hostfile_entry_for(hostfile, daemon): + port = _extract_port(daemon) + with open(hostfile) as hf: + for line in hf: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + clean = stripped[1:] if stripped[0] in "+-" else stripped + if f":{port}" in clean: + return clean + raise AssertionError(f"No hostfile entry found for daemon {daemon.address}") + + +def _write_workspace(hostfile, active, removing=None, adding=None): + removing = removing or [] + adding = adding or [] + marker = {d: "" for d in active} + marker.update({d: "-" for d in removing}) + marker.update({d: "+" for d in adding}) + + entries = [ + marker[daemon] + _hostfile_entry_for(daemon.hostfile, daemon) + "\n" + for daemon in list(active) + list(removing) + list(adding) + ] + + with open(hostfile, "w") as hf: + hf.writelines(entries) + + +def _chunk_files(rootdir): + root = Path(rootdir) + if not root.exists(): + return set() + return {p.relative_to(root) for p in root.rglob("*") if p.is_file() and p.name.isdigit()} + + +def _write_file(client, path, size): + ret = client.open(path, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + assert ret.retval != -1, f"open failed for {path}" + ret = client.write_validate(path, size) + assert ret.retval == 0, f"write_validate failed for {path}" + + payload = bytes((ord("0") + (i % 10)) for i in range(size)) + return hashlib.md5(payload).hexdigest() + + +def _read_md5(client, path, logdir=None, hostfile=None): + ret = client.stat(path) + diagnostics = "" + if logdir is not None: + daemon_log = Path(logdir) / "gkfs_daemon.log" + client_log = Path(logdir) / "gkfs_client.log" + diagnostics = ( + f"\nhostfile:\n{Path(hostfile).read_text() if hostfile and Path(hostfile).exists() else ''}" + f"\ndaemon log tail:\n{daemon_log.read_text()[-12000:] if daemon_log.exists() else ''}" + f"\nclient log tail:\n{client_log.read_text()[-12000:] if client_log.exists() else ''}" + ) + assert ret.retval == 0, f"stat failed for {path}{diagnostics}" + size = ret.statbuf.st_size + + ret = client.open(path, os.O_RDONLY, 0) + assert ret.retval != -1, f"open for read failed for {path}" + ret = client.read(path, size) + assert ret.retval == size, f"read failed for {path}: {ret.retval}/{size}" + assert ret.buf is not None, f"read returned no buffer for {path}" + return hashlib.md5(ret.buf).hexdigest() + + +def _verify_md5_map(client, md5_map, logdir=None, hostfile=None): + for path, expected in md5_map.items(): + assert _read_md5(client, path, logdir, hostfile) == expected, ( + f"MD5 mismatch for {path}" + ) + + +def _wait_for_mutate_done(gkfs_shell, hostfile): + deadline = time.time() + 120 + while time.time() < deadline: + cmd = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") + if "No mutate running/finished." in cmd.stderr.decode(): + return + time.sleep(2) + pytest.fail("Mutate redistribution did not finish within 120 s") + + +def _assert_final_hostfile(hostfile, removed_daemons): + content = Path(hostfile).read_text() + for line in content.splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#"): + assert not stripped.startswith("+"), f"Hostfile still has '+' marker: {stripped}" + assert not stripped.startswith("-"), f"Hostfile still has '-' marker: {stripped}" + for daemon in removed_daemons: + assert f":{_extract_port(daemon)}" not in content, ( + f"Removed daemon {daemon.address} still in finalized hostfile" + ) + + +@pytest.mark.parametrize("strategy", ["simple_hash", "random_slicing"]) +@pytest.mark.parametrize("action", ["add", "remove", "swap"]) +def test_mutate_add_remove_swap_keeps_data_and_chunks_distributed( + strategy, + action, + monkeypatch, + gkfwd_daemon_factory, + gkfs_shell, + request, +): + monkeypatch.setenv("GKFS_DISTRIBUTION_STRATEGY", strategy) + monkeypatch.setenv("GKFS_DAEMON_KEEP_HOSTS_FILE", "ON") + client = request.getfixturevalue("gkfs_client") + + daemons = [] + try: + if action == "add": + old_daemons = [gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [] + active_daemons = old_daemons + elif action == "remove": + old_daemons = [gkfwd_daemon_factory.create(), gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [old_daemons[1]] + active_daemons = [old_daemons[0]] + else: + old_daemons = [gkfwd_daemon_factory.create(), gkfwd_daemon_factory.create()] + added_daemons = [] + removed_daemons = [old_daemons[1]] + active_daemons = [old_daemons[0]] + + daemons = list(old_daemons) + time.sleep(3) + + md5_map = {} + for i in range(FILE_COUNT): + path = old_daemons[0].mountdir / f"{strategy}_{action}_{i:03d}.dat" + md5_map[path] = _write_file(client, path, FILE_SIZE) + + _verify_md5_map(client, md5_map) + + rootdir = old_daemons[0].rootdir + pre_chunks = _chunk_files(rootdir) + assert pre_chunks, "No chunks found before mutate" + + if action == "add": + added_daemons = [gkfwd_daemon_factory.create(expand_mode=True)] + elif action == "swap": + added_daemons = [gkfwd_daemon_factory.create(expand_mode=True)] + daemons.extend(added_daemons) + time.sleep(2) + + hostfile = Path(old_daemons[0].hostfile) + _write_workspace(hostfile, active_daemons, removing=removed_daemons, adding=added_daemons) + + status = _run_mutate_cmd(gkfs_shell, hostfile, "mutate status") + assert "No mutate running/finished." in status.stderr.decode() + + start = _run_mutate_cmd( + gkfs_shell, + hostfile, + "mutate start", + timeout=340, + logdir=old_daemons[0].logdir, + ) + assert "Mutate process" in start.stderr.decode() + _wait_for_mutate_done(gkfs_shell, hostfile) + _run_mutate_cmd(gkfs_shell, hostfile, "mutate finalize") + + _assert_final_hostfile(hostfile, removed_daemons) + for daemon in removed_daemons: + daemon._proc.wait(timeout=10) + assert daemon._proc.poll() is not None, ( + f"Removed daemon {daemon.address} was not gracefully shut down" + ) + _verify_md5_map(client, md5_map, old_daemons[0].logdir, hostfile) + + post_chunks = _chunk_files(rootdir) + assert post_chunks, "No chunks found after mutate" + assert pre_chunks <= post_chunks, "Some pre-mutate chunks disappeared from backend storage" + + post_md5_map = {} + for i in range(2): + path = old_daemons[0].mountdir / f"{strategy}_{action}_post_{i:03d}.dat" + post_md5_map[path] = _write_file(client, path, FILE_SIZE) + _verify_md5_map(client, post_md5_map) + + finally: + for daemon in daemons: + daemon.shutdown() \ No newline at end of file diff --git a/tests/integration/operations/test_client_cache.py b/tests/integration/operations/test_client_cache.py index f7a5fd11b61d2881fc2ee7407252107a633587f7..4b15b45cb50b4fcb1bd95aaa6f0d66c43d4de31f 100644 --- a/tests/integration/operations/test_client_cache.py +++ b/tests/integration/operations/test_client_cache.py @@ -1,6 +1,6 @@ import pytest import os -import stat + from harness.gkfs import Client as GKFSClient @@ -10,11 +10,10 @@ def gkfs_client_cache(test_workspace, gkfs_daemon, monkeypatch): Sets up a GKFSClient with caching enabled via environment variables. """ monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE", "ON") - # Threshold 10 means flush every 10 writes. - monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE_THRESHOLD", "10") + # Threshold 10 means flush every 10 writes. + monkeypatch.setenv("LIBGKFS_WRITE_SIZE_CACHE_THRESHOLD", "10") monkeypatch.setenv("LIBGKFS_DENTRY_CACHE", "ON") - - + client = GKFSClient(test_workspace) return client @@ -24,14 +23,16 @@ def test_write_size_cache(gkfs_daemon, gkfs_client_cache): Test write size cache by running a C++ helper that performs multiple writes. """ file = gkfs_daemon.mountdir / "cache_file" - + # Run the helper: writes 20 chunks of 100 bytes iterations = 20 chunk_size = 100 - + # Use gkfs.io write_sequential command - cmd = gkfs_client_cache.run("write_sequential", "--pathname", str(file), "--count", str(iterations), "--size", str(chunk_size)) - + cmd = gkfs_client_cache.run("write_sequential", "--pathname", str(file), + "--count", str(iterations), "--size", + str(chunk_size)) + assert cmd.retval == 0 # Verify file size using GKFSClient (which handles stat command parsing) @@ -44,29 +45,18 @@ def test_dentry_cache(gkfs_daemon, gkfs_client_cache): Test dentry cache by creating a directory structure and listing it repeatedly. """ subdir = gkfs_daemon.mountdir / "subdir" - + gkfs_client_cache.mkdir(subdir, 0o755) - + # Create files for i in range(10): f = subdir / f"file{i}" gkfs_client_cache.open(f, os.O_CREAT | os.O_WRONLY, 0o644) - - # Use ls -lR to trigger readdir and accessing attributes - # We use subprocess with client env to ensure caching is enabled - import subprocess - ls_cmd = ["ls", "-lR", str(gkfs_daemon.mountdir)] - - # First run: entries should be cached - proc = subprocess.Popen(ls_cmd, env=gkfs_client_cache._env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - assert proc.returncode == 0 - assert b"subdir" in stdout - assert b"file0" in stdout - - # Second run: entries should be served from cache - proc = subprocess.Popen(ls_cmd, env=gkfs_client_cache._env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = proc.communicate() - assert proc.returncode == 0 + first = gkfs_client_cache.readdir(gkfs_daemon.mountdir) + assert first.errno == 0 + assert any(entry.d_name == "subdir" for entry in first.dirents) + second = gkfs_client_cache.readdir(subdir) + assert second.errno == 0 + assert any(entry.d_name == "file0" for entry in second.dirents) diff --git a/tests/integration/startup/test_hosts_file_lifecycle.py b/tests/integration/startup/test_hosts_file_lifecycle.py index 21da0826ebc628970be51b6bd64dc590795c25d8..a46e24665028c5b717d4d77ac218e975e8b0dc7b 100644 --- a/tests/integration/startup/test_hosts_file_lifecycle.py +++ b/tests/integration/startup/test_hosts_file_lifecycle.py @@ -30,7 +30,7 @@ Integration tests for hosts file lifecycle management. Tests that the shared hosts file is correctly destroyed by default on daemon shutdown -and preserved when the --keep-hosts flag or GKFS_KEEP_HOSTS_FILE environment variable is set. +and preserved when the --keep-hosts flag or GKFS_DAEMON_KEEP_HOSTS_FILE environment variable is set. """ import os @@ -62,9 +62,9 @@ def test_hosts_file_destroyed_on_normal_shutdown(gkfwd_daemon_factory, gkfs_shel def test_hosts_file_preserved_with_keep_env(test_workspace, request): - """Test that hosts file is preserved with GKFS_KEEP_HOSTS_FILE=ON.""" + """Test that hosts file is preserved with GKFS_DAEMON_KEEP_HOSTS_FILE=ON.""" d00 = Daemon(request.config.getoption('--interface'), "rocksdb", - test_workspace, env={"GKFS_KEEP_HOSTS_FILE": "ON"}) + test_workspace, env={"GKFS_DAEMON_KEEP_HOSTS_FILE": "ON"}) d00.run() time.sleep(5) @@ -76,7 +76,7 @@ def test_hosts_file_preserved_with_keep_env(test_workspace, request): time.sleep(2) assert hostfile.exists(), \ - "Hosts file should be preserved when GKFS_KEEP_HOSTS_FILE=ON" + "Hosts file should be preserved when GKFS_DAEMON_KEEP_HOSTS_FILE=ON" # Clean up hostfile.unlink() diff --git a/tests/integration/syscalls/test_client_ofi_interface.py b/tests/integration/syscalls/test_client_ofi_interface.py index 9d8e03bd0dd02ac1c49ac74b072b3897115958f4..c85ef92606de75f6dcb1cf4767e524dbe5ddce57 100644 --- a/tests/integration/syscalls/test_client_ofi_interface.py +++ b/tests/integration/syscalls/test_client_ofi_interface.py @@ -36,7 +36,6 @@ and that LIBGKFS_OFI_INTERFACE takes precedence when both are set. import os import time -import shutil from pathlib import Path import pytest @@ -50,31 +49,19 @@ def test_ofi_interface_env_var_honored(gkfwd_daemon_factory, gkfs_shell): with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Without LIBGKFS_OFI_INTERFACE, status should work - cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " - f"LIBGKFS_HOSTS_FILE={hostfile} " - f"{malleability_bin} expand status" - ) + cmd_str = f"LIBGKFS_HOSTS_FILE={hostfile} gkfs_malleability mutate status" cmd = gkfs_shell.script(cmd_str, intercept_shell=False) - assert cmd.exit_code == 0, f"expand status failed: {cmd.stderr.decode()}" + assert cmd.exit_code == 0, f"mutate status failed: {cmd.stderr.decode()}" # With LIBGKFS_OFI_INTERFACE=lo, should still work cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " - f"LIBGKFS_OFI_INTERFACE=lo " - f"{malleability_bin} expand status" + f"LIBGKFS_OFI_INTERFACE=lo gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ - f"expand status with LIBGKFS_OFI_INTERFACE=lo failed: {cmd.stderr.decode()}" + f"mutate status with LIBGKFS_OFI_INTERFACE=lo failed: {cmd.stderr.decode()}" d00.shutdown() @@ -88,22 +75,15 @@ def test_fi_sockets_iface_env_var_honored(gkfwd_daemon_factory, gkfs_shell): with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # With FI_SOCKETS_IFACE set, should work cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"FI_SOCKETS_IFACE=lo " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ - f"expand status with FI_SOCKETS_IFACE=lo failed: {cmd.stderr.decode()}" + f"mutate status with FI_SOCKETS_IFACE=lo failed: {cmd.stderr.decode()}" d00.shutdown() @@ -117,19 +97,12 @@ def test_libgkfs_ofi_interface_takes_precedence(gkfwd_daemon_factory, gkfs_shell with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - # Both set - LIBGKFS_OFI_INTERFACE should take precedence cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"LIBGKFS_OFI_INTERFACE=mlx5_0 " f"FI_SOCKETS_IFACE=ib0 " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) assert cmd.exit_code == 0, \ @@ -147,22 +120,15 @@ def test_libgkfs_ofi_interface_with_multiple_iface_values(gkfwd_daemon_factory, with open(hostfile, 'a') as f: f.write("#FS_INSTANCE_END\n") - libdirs = gkfs_shell._patched_env.get("LD_LIBRARY_PATH", "") - search_path = ":".join(str(p) for p in gkfs_shell._search_paths) - malleability_bin = shutil.which("gkfs_malleability", path=search_path) - - assert malleability_bin is not None, "gkfs_malleability not found in PATH" - for iface in ["lo", "eth0", "ib0", "mlx5_0"]: cmd_str = ( - f"LD_LIBRARY_PATH={libdirs} " f"LIBGKFS_HOSTS_FILE={hostfile} " f"LIBGKFS_OFI_INTERFACE={iface} " - f"{malleability_bin} expand status" + f"gkfs_malleability mutate status" ) cmd = gkfs_shell.script(cmd_str, intercept_shell=False) # Status doesn't require the interface to exist, just to be set assert cmd.exit_code == 0, \ - f"expand status with LIBGKFS_OFI_INTERFACE={iface} failed: {cmd.stderr.decode()}" + f"mutate status with LIBGKFS_OFI_INTERFACE={iface} failed: {cmd.stderr.decode()}" d00.shutdown() \ No newline at end of file diff --git a/tests/integration/syscalls/test_config_env.py b/tests/integration/syscalls/test_config_env.py index 35c6ebb0123d6aec13a2040b52ed2707fe16b106..2deb68542f7d33ad9d8155b80010a2ade84c03f7 100644 --- a/tests/integration/syscalls/test_config_env.py +++ b/tests/integration/syscalls/test_config_env.py @@ -1,135 +1,88 @@ - import pytest -import logging -import os -import time -from pathlib import Path -from harness.logger import logger -from harness.gkfs import Daemon, Client, ShellClient + +from harness.gkfs import Client, Daemon + @pytest.mark.parametrize("use_inline", ["ON", "OFF"]) def test_inline_data(test_workspace, request, use_inline): """ Verify inline data configuration via environment variables. - Both Client and Daemon need to agree (or at least Client needs to send it). - + If OFF: - Client writes small data. - - Should land in Chunk Storage (file in workspace/chunks). + - Should land in chunk storage. If ON: - - Client writes small data. - - Should landad in Metadata (no file in workspace/chunks for this file). + - Should land in metadata. + - Should not create chunk files for this small write. """ - - # 1. Start Daemon with env var - # We use same value for Daemon to be safe (though strictly Client decides to send inline) - # 1. Start Daemon with env var - # We use same value for Daemon to be safe (though strictly Client decides to send inline) - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": use_inline} - - interface = request.config.getoption('--interface') - backend = "rocksdb" - - daemon = Daemon(interface, backend, test_workspace, env=daemon_env) + daemon_env = {"GKFS_USE_INLINE_DATA": use_inline} + + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() - + try: - # 2. Start Client with env var - # 2. Start Client with env var - client_env = {"LIBGKFS_USE_INLINE_DATA": use_inline} - - # We need a shell client or similar to execute commands - # We can use gkfs.io via Client class which wraps it, or just use shell + client_env = {"GKFS_USE_INLINE_DATA": use_inline} client = Client(test_workspace) - - # 3. Write small file (100 bytes) - # using 'write_sequential' from previous task! + test_file = test_workspace.mountdir / "file_inline" - - # Write 100 bytes - cmd = client.run("write_sequential", "--pathname", str(test_file), "--count", "1", "--size", "100", env=client_env) + cmd = client.run("write_sequential", "--pathname", str(test_file), + "--count", "1", "--size", "100", env=client_env) assert cmd.retval == 0 - - # 4. Verify storage location - # Check chunks directory - # The chunks directory is in test_workspace.rootdir / "chunks" + chunk_dir = test_workspace.rootdir / "chunks" - - # We expect files in chunk_dir ONLY if use_inline == "OFF" - # Since we just started fresh, chunk_dir might be empty or contain structure. - # We look for any file recursively in chunk_dir - - found_chunks = list(chunk_dir.rglob("*")) - # Filter out directories - found_chunks = [f for f in found_chunks if f.is_file()] - + found_chunks = [f for f in chunk_dir.rglob("*") if f.is_file()] + if use_inline == "OFF": - # Should have chunks - assert len(found_chunks) > 0, "Expected chunks to be created when INLINE_DATA=OFF" + assert len(found_chunks) > 0, \ + "Expected chunks to be created when INLINE_DATA=OFF" else: - # Should NOT have chunks (for this file). - # Note: create_directories might have created empty subdirs, but we filtered for files. - assert len(found_chunks) == 0, f"Expected NO chunks when INLINE_DATA=ON (data should be inline). Found: {found_chunks}" - + assert len(found_chunks) == 0, \ + f"Expected NO chunks when INLINE_DATA=ON. Found: {found_chunks}" finally: daemon.shutdown() + @pytest.mark.parametrize("use_compression", ["ON", "OFF"]) def test_dirents_compression(test_workspace, request, use_compression): """ Verify dirents compression configuration. - Hard to verify compression effect directly without packet capture, - but we can verify that the system runs and respects the flag in logs. + + Hard to verify compression effect directly without packet capture, but we + can verify that the daemon logs the flag and that directory listing works. """ - - daemon_env = {"GKFS_DAEMON_USE_DIRENTS_COMPRESSION": use_compression} - # specific log level to see configuration output - daemon_env["GKFS_DAEMON_LOG_LEVEL"] = "info" - - interface = request.config.getoption('--interface') - backend = "rocksdb" - - daemon = Daemon(interface, backend, test_workspace, env=daemon_env) + daemon_env = { + "GKFS_USE_DIRENTS_COMPRESSION": use_compression, + "GKFS_DAEMON_LOG_LEVEL": "info", + } + + daemon = Daemon(request.config.getoption('--interface'), "rocksdb", + test_workspace, env=daemon_env) daemon.run() - + try: - # Check daemon logs for the configuration message - # We added: GKFS_DATA->spdlogger()->info("{}() Inline data: {} / Dirents compression: {}", ... - - # grep log file log_file = daemon.logdir / "gkfs_daemon.log" - passed = False expected_val = "true" if use_compression == "ON" else "false" with open(log_file, 'r') as f: log_content = f.read() - if f"Dirents compression: {expected_val}" in log_content: - passed = True - - if not passed: - print(f"DEBUG: Log content:\n{log_content}") - - assert passed, f"Daemon log did not confirm Dirents compression={use_compression} (expected {expected_val})" - - # Client side verification - # Client side verification - client_env = {"LIBGKFS_USE_DIRENTS_COMPRESSION": use_compression} - - # Just run a simple ls to trigger dirents + + assert f"Dirents compression: {expected_val}" in log_content, \ + f"Daemon log did not confirm Dirents compression={use_compression} " \ + f"(expected {expected_val})\n{log_content}" + + client_env = {"GKFS_USE_DIRENTS_COMPRESSION": use_compression} client = Client(test_workspace) - shell = ShellClient(test_workspace) - - # Create dir using shell mkidr - behavior should be same regarding dirents later - cmd = shell.run("mkdir", "-p", str(test_workspace.mountdir / "dir"), env=client_env) - assert cmd.exit_code == 0 - - # Populate dir using Client write - cmd = client.run("write_sequential", "--pathname", str(test_workspace.mountdir / "dir" / "file"), "--count", "1", "--size", "100", env=client_env) + + test_dir = test_workspace.mountdir / "dir" + cmd = client.mkdir(str(test_dir), 0o755, env=client_env) assert cmd.retval == 0 - # List dir using shell ls - cmd = shell.run("ls", str(test_workspace.mountdir / "dir"), env=client_env) - assert cmd.exit_code == 0 - + cmd = client.run("write_sequential", "--pathname", str(test_dir / "file"), + "--count", "1", "--size", "100", env=client_env) + assert cmd.retval == 0 + + cmd = client.readdir(str(test_dir), env=client_env) + assert cmd.errno == 0 + assert any(entry.d_name == "file" for entry in cmd.dirents) finally: daemon.shutdown() - diff --git a/tests/integration/syscalls/test_env_features.py b/tests/integration/syscalls/test_env_features.py index 9f7515dce30691120d8e156cc47e13c9401523da..e81559b8869e304900ddfe759ec9169296709c15 100644 --- a/tests/integration/syscalls/test_env_features.py +++ b/tests/integration/syscalls/test_env_features.py @@ -4,14 +4,14 @@ import os from harness.gkfs import Daemon, Client, ShellClient @pytest.mark.parametrize("opt_env", [ - {"LIBGKFS_CREATE_WRITE_OPTIMIZATION": "ON", "LIBGKFS_USE_INLINE_DATA": "ON"} + {"LIBGKFS_CREATE_WRITE_OPTIMIZATION": "ON", "GKFS_USE_INLINE_DATA": "ON"} ]) def test_create_write_optimization(test_workspace, request, opt_env): """ Test CREATE_WRITE_OPTIMIZATION. Should trigger 'forward_create_write_inline' in forward_metadata.cpp. """ - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": "ON"} + daemon_env = {"GKFS_USE_INLINE_DATA": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() @@ -40,14 +40,14 @@ def test_create_write_optimization(test_workspace, request, opt_env): daemon.shutdown() @pytest.mark.parametrize("prefetch_env", [ - {"LIBGKFS_READ_INLINE_PREFETCH": "ON", "LIBGKFS_USE_INLINE_DATA": "ON"} + {"LIBGKFS_READ_INLINE_PREFETCH": "ON", "GKFS_USE_INLINE_DATA": "ON"} ]) def test_read_inline_prefetch(test_workspace, request, prefetch_env): """ Test READ_INLINE_PREFETCH. Should trigger 'forward_stat' with include_inline=true in forward_metadata.cpp. """ - daemon_env = {"GKFS_DAEMON_USE_INLINE_DATA": "ON"} + daemon_env = {"GKFS_USE_INLINE_DATA": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() @@ -56,7 +56,7 @@ def test_read_inline_prefetch(test_workspace, request, prefetch_env): file_path = test_workspace.mountdir / "prefetch_file" # 1. Create file with inline data (using standard write, ensuring inline is used) - create_env = {"LIBGKFS_USE_INLINE_DATA": "ON"} + create_env = {"GKFS_USE_INLINE_DATA": "ON"} ret = client.run("write_sequential", "--pathname", str(file_path), "--count", "1", @@ -73,14 +73,14 @@ def test_read_inline_prefetch(test_workspace, request, prefetch_env): daemon.shutdown() @pytest.mark.parametrize("compress_env", [ - {"LIBGKFS_USE_DIRENTS_COMPRESSION": "ON"} + {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} ]) def test_dirents_compression_large(test_workspace, request, compress_env): """ Test DIRENTS_COMPRESSION with enough entries to trigger compression logic. Should trigger 'decompress_and_parse_entries_standard' with compression path. """ - daemon_env = {"GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "ON"} + daemon_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"} daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env) daemon.run() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index f68962804a0c9f155aa48d6a35a1c171e8c7da99..fe7d7284a4b2596600fb67f49232da2072803108 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -58,7 +58,11 @@ target_sources(unit_tests ${CMAKE_CURRENT_LIST_DIR}/test_path.cpp ${CMAKE_CURRENT_LIST_DIR}/test_common_path.cpp ${CMAKE_CURRENT_LIST_DIR}/test_distributor.cpp - ${CMAKE_CURRENT_LIST_DIR}/test_helpers.cpp) + ${CMAKE_CURRENT_LIST_DIR}/test_helpers.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_distributor.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_distributor_factory_and_migrator.cpp + ${CMAKE_CURRENT_LIST_DIR}/test_random_slicing_pipeline.cpp + ${CMAKE_SOURCE_DIR}/src/common/malleability_markers.cpp) if (GKFS_TESTS_GUIDED_DISTRIBUTION) target_sources(unit_tests PRIVATE ${CMAKE_CURRENT_LIST_DIR}/test_guided_distributor.cpp) diff --git a/tests/unit/test_distributor.cpp b/tests/unit/test_distributor.cpp index f7e69150c115820be36ab6867e662c98f1a78e5e..ee07002ae74e6872a3887479744e71e316a86991 100644 --- a/tests/unit/test_distributor.cpp +++ b/tests/unit/test_distributor.cpp @@ -51,6 +51,22 @@ TEST_CASE("SimpleHashDistributor", "[common][distributor]") { REQUIRE(d.locate_data("/foo", 0, 0) == c1); } +TEST_CASE("SimpleHashDistributor hosts_size updates directory targets", + "[common][distributor][malleability]") { + auto d = gkfs::rpc::SimpleHashDistributor(0, 4); + REQUIRE(d.locate_directory_metadata().size() == 4); + + d.hosts_size(2); + + auto targets = d.locate_directory_metadata(); + REQUIRE(targets.size() == 2); + REQUIRE(targets[0] == 0); + REQUIRE(targets[1] == 1); + REQUIRE(d.hosts_size() == 2); + REQUIRE(d.locate_file_metadata("/foo", 0) < 2); + REQUIRE(d.locate_data("/foo", 0, 0) < 2); +} + TEST_CASE("LocalOnlyDistributor", "[common][distributor]") { auto d = gkfs::rpc::LocalOnlyDistributor(5); REQUIRE(d.localhost() == 5); diff --git a/tests/unit/test_distributor_factory_and_migrator.cpp b/tests/unit/test_distributor_factory_and_migrator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..56ae54204f2b7da2dc09e1006423a8d12117c335 --- /dev/null +++ b/tests/unit/test_distributor_factory_and_migrator.cpp @@ -0,0 +1,277 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This software was partially supported by the + * EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include +#include +#include + +using namespace gkfs::rpc; + +// ===== Distributor factory tests ===== + +TEST_CASE("DistributorFactory strategy_to_string converts all strategies", "[factory]") { + REQUIRE(std::string(strategy_to_string(DistributionStrategy::SimpleHash)) == "simple_hash"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::RandomSlicing)) == "random_slicing"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::LocalOnly)) == "local_only"); + REQUIRE(std::string(strategy_to_string(DistributionStrategy::Forwarder)) == "forwarder"); +} + +TEST_CASE("DistributorFactory string_to_strategy is inverse of strategy_to_string", "[factory]") { + REQUIRE(string_to_strategy("simple_hash") == DistributionStrategy::SimpleHash); + REQUIRE(string_to_strategy("random_slicing") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("local_only") == DistributionStrategy::LocalOnly); + REQUIRE(string_to_strategy("forwarder") == DistributionStrategy::Forwarder); +} + +TEST_CASE("DistributorFactory string_to_strategy is case-insensitive", "[factory]") { + REQUIRE(string_to_strategy("RANDOM_SLICING") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("Random_Slicing") == DistributionStrategy::RandomSlicing); + REQUIRE(string_to_strategy("simple_hash") == DistributionStrategy::SimpleHash); + REQUIRE(string_to_strategy("SIMPLE_HASH") == DistributionStrategy::SimpleHash); +} + +TEST_CASE("DistributorFactory create_distributor creates correct types", "[factory]") { + auto sh = create_distributor(DistributionStrategy::SimpleHash, 0, 10); + REQUIRE(sh->localhost() == 0); + REQUIRE(sh->hosts_size() == 10); + + auto rs = create_distributor(DistributionStrategy::RandomSlicing, 0, 10); + REQUIRE(rs->localhost() == 0); + REQUIRE(rs->hosts_size() == 10); + + auto lo = create_distributor(DistributionStrategy::LocalOnly, 5, 1); + REQUIRE(lo->localhost() == 5); + + auto fw = create_distributor(DistributionStrategy::Forwarder, 0, 1, 42); + REQUIRE(fw->localhost() == 0); +} + +TEST_CASE("DistributorFactory create_distributor_from_string creates correct types", "[factory]") { + auto sh = create_distributor_from_string("simple_hash", 0, 10); + REQUIRE(sh != nullptr); + REQUIRE(sh->localhost() == 0); + + auto rs = create_distributor_from_string("random_slicing", 0, 10); + REQUIRE(rs != nullptr); + REQUIRE(rs->localhost() == 0); + + // Unknown string should default to SimpleHash + auto def = create_distributor_from_string("unknown_strategy", 0, 10); + REQUIRE(def != nullptr); +} + +// ===== DataMigrator tests ===== + +TEST_CASE("DataMigrator needs_migration returns true when partitions differ", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 0.5f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 0.5f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 0.5f; + gkfs::rpc::Interval iv2; + iv2.start = 0.5f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + REQUIRE(gkfs::rpc::DataMigrator::needs_migration(old_parts, new_parts) == true); +} + +TEST_CASE("DataMigrator needs_migration returns false when partitions are identical", "[migrator]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + REQUIRE(gkfs::rpc::DataMigrator::needs_migration(parts, parts) == false); +} + +TEST_CASE("DataMigrator compute_migrations finds migrating chunks", "[migrator]") { + // Old: one node covering [0, 1) + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + // New: one node covering [0, 1) but different host means all chunks migrate + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 100); + REQUIRE(!jobs.empty()); + + // All jobs should be from host 1 to host 2 + for (const auto& j : jobs) { + REQUIRE(j.source_node == 1); + REQUIRE(j.target_node == 2); + } +} + +TEST_CASE("DataMigrator get_migration_stats computes correct statistics", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_parts, new_parts, 100); + REQUIRE(stats.total_chunks == 100); + REQUIRE(stats.migrating_chunks > 0); + REQUIRE(stats.migration_ratio > 0.0); + REQUIRE(!stats.from_counts.empty()); + REQUIRE(!stats.to_counts.empty()); +} + +TEST_CASE("DataMigrator print_migration_summary prints valid output", "[migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + gkfs::rpc::DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_parts, new_parts, 100); + + REQUIRE_NOTHROW(migrator.print_migration_summary(stats)); +} + +// ===== CutShift+Sorted tests ===== + +TEST_CASE("CutShiftSorted collect_gaps_cutshift produces valid gaps", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector old_parts = {p}; + std::unordered_map reductions; + reductions[1] = 0.5f; // shrink by 50% + + auto gaps = collect_gaps_cutshift(old_parts, reductions); + REQUIRE(!gaps.empty()); + + // Gaps should have host_id=0 + for (const auto& g : gaps) { + REQUIRE(g.host_id == 0); + REQUIRE(g.start < g.end); + } +} + +TEST_CASE("CutShiftSorted expand_with_cutshift adds new nodes", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts = {2, 3}; + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + REQUIRE(result.size() == 3); // 1 old + 2 new +} + +TEST_CASE("CutShiftSorted expand_with_cutshift shrinks old nodes", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts = {2}; + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + + // Old node capacity should be reduced + REQUIRE(result[0].total_capacity < 1.0f); + // New node should have received some capacity + REQUIRE(result[1].total_capacity > 0.0f); +} + +TEST_CASE("CutShiftSorted expand_with_cutshift is identity with no new hosts", "[cutshift]") { + gkfs::rpc::Partition p; + p.host_id = 1; + p.total_capacity = 1.0f; + gkfs::rpc::Interval iv; + iv.start = 0.0f; iv.end = 1.0f; iv.host_id = 1; + p.intervals.push_back(iv); + + std::vector parts = {p}; + std::vector new_hosts; // empty + + auto result = expand_with_cutshift(parts, new_hosts, 1.0f, 1.0f); + REQUIRE(result.size() == 1); + REQUIRE(result[0].host_id == 1); +} \ No newline at end of file diff --git a/tests/unit/test_random_slicing_distributor.cpp b/tests/unit/test_random_slicing_distributor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a1ac78436a57b9ec5617774c6e829911a2ab0232 --- /dev/null +++ b/tests/unit/test_random_slicing_distributor.cpp @@ -0,0 +1,354 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This software was partially supported by the + * EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +float +coverage(const std::vector& intervals) { + float sum = 0.0f; + for(const auto& interval : intervals) { + sum += interval.end - interval.start; + } + return sum; +} + +std::vector +make_equal_partitions(unsigned int hosts) { + std::vector partitions; + partitions.reserve(hosts); + const auto step = 1.0f / static_cast(hosts); + for(unsigned int host = 0; host < hosts; ++host) { + gkfs::rpc::Partition partition; + partition.host_id = host; + partition.intervals.push_back({host * step, (host + 1) * step, host}); + partition.total_capacity = step; + partitions.push_back(partition); + } + return partitions; +} + +} // namespace + +// ===== Basic interface tests ===== + +TEST_CASE("RandomSlicingDistributor basic construction", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + REQUIRE(d.localhost() == 0); + REQUIRE(d.hosts_size() == 10); +} + +TEST_CASE("RandomSlicingDistributor determinism", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + // Same inputs should produce same outputs (deterministic PRNG) + auto c1a = d.locate_data("/foo", 0, 0); + auto c1b = d.locate_data("/foo", 0, 0); + REQUIRE(c1a == c1b); + + auto c2a = d.locate_data("/bar", 1, 0); + auto c2b = d.locate_data("/bar", 1, 0); + REQUIRE(c2a == c2b); +} + +TEST_CASE("RandomSlicingDistributor bounds checking", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + for (int i = 0; i < 100; ++i) { + auto host = d.locate_data("/test", i, 0); + REQUIRE(host < 10); + } + + auto meta = d.locate_file_metadata("/test", 0); + REQUIRE(meta < 10); +} + +// ===== Distribution uniformity test ===== + +TEST_CASE("RandomSlicingDistributor distribution is uniform", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + // Count how many chunks map to each host + std::vector counts(10, 0); + const int num_chunks = 10000; + + for (int i = 0; i < num_chunks; ++i) { + auto host = d.locate_data("/file", i, 0); + counts[host]++; + } + + // Each host should get roughly 10% of chunks (within 5% tolerance) + float expected = static_cast(num_chunks) / 10.0f; + for (int i = 0; i < 10; ++i) { + float deviation = std::abs(counts[i] - expected); + REQUIRE(deviation < expected * 0.15f); // 15% tolerance for randomness + } +} + +TEST_CASE("RandomSlicingDistributor different paths produce different results", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + std::set hosts_foo, hosts_bar; + for (int i = 0; i < 100; ++i) { + hosts_foo.insert(d.locate_data("/foo", i, 0)); + hosts_bar.insert(d.locate_data("/bar", i, 0)); + } + + // Both paths should reach multiple hosts + REQUIRE(hosts_foo.size() > 1); + REQUIRE(hosts_bar.size() > 1); +} + +// ===== Node add/remove tests ===== + +TEST_CASE("RandomSlicingDistributor add_nodes expands coverage", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + REQUIRE(d.hosts_size() == 5); + + // Add 5 more nodes + std::vector new_nodes = {5, 6, 7, 8, 9}; + d.add_nodes(new_nodes); + + REQUIRE(d.hosts_size() == 10); + + // Verify all 10 hosts are covered + std::set all_hosts; + for (int i = 0; i < 100; ++i) { + all_hosts.insert(d.locate_data("/file", i, 0)); + } + // With 10 hosts and uniform distribution, we should see most of them + REQUIRE(all_hosts.size() >= 8); +} + +TEST_CASE("RandomSlicingDistributor remove_nodes shrinks coverage", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 10); + + REQUIRE(d.hosts_size() == 10); + + // Remove 5 nodes + std::vector old_nodes = {5, 6, 7, 8, 9}; + d.remove_nodes(old_nodes); + + REQUIRE(d.hosts_size() == 5); + + // All data should now map to hosts 0-4 + for (int i = 0; i < 100; ++i) { + auto host = d.locate_data("/file", i, 0); + REQUIRE(host < 5); + } +} + +// ===== Reconfigure test ===== + +TEST_CASE("RandomSlicingDistributor reconfigure rebuilds from all_hosts", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + // Get current state + auto parts = d.get_partitions_copy(); + REQUIRE(parts.size() == 5); + + // Change hosts_size and add new hosts + d.hosts_size(10); + std::vector new_nodes = {5, 6, 7, 8, 9}; + d.add_nodes(new_nodes); + d.reconfigure(); + + parts = d.get_partitions_copy(); + REQUIRE(parts.size() == 10); +} + +// ===== Interval table persistence tests ===== +// persistence is intentionally disabled — intervals are rebuilt from +// the live host list on each daemon/proxy startup. These tests verify the stubs. + +TEST_CASE("RandomSlicingDistributor save_interval_table is stubbed", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + // Save is a no-op stub — doesn't crash but doesn't write meaningful data + std::string path = "/tmp/test_random_slicing_table.bin"; + d.save_interval_table(path); + + // load returns false for stubbed persistence + auto d2 = gkfs::rpc::RandomSlicingDistributor(0, 1); + REQUIRE(!d2.load_interval_table(path)); +} + +TEST_CASE("RandomSlicingDistributor installs explicit interval table", + "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 3); + + std::vector intervals = {{0.0f, 0.25f, 2}, + {0.25f, 0.75f, 0}, + {0.75f, 1.0f, 1}}; + REQUIRE(d.set_intervals(intervals)); + auto got = d.get_intervals(); + REQUIRE(got.size() == intervals.size()); + REQUIRE(got[0].host_id == 2); + REQUIRE(got[0].start == Catch::Approx(0.0f)); + REQUIRE(got[0].end == Catch::Approx(0.25f)); + REQUIRE(got[1].host_id == 0); + REQUIRE(got[1].start == Catch::Approx(0.25f)); + REQUIRE(got[1].end == Catch::Approx(0.75f)); + REQUIRE(got[2].host_id == 1); + REQUIRE(got[2].start == Catch::Approx(0.75f)); + REQUIRE(got[2].end == Catch::Approx(1.0f)); + + REQUIRE_FALSE(d.set_intervals({{0.0f, 0.4f, 0}, {0.5f, 1.0f, 1}})); + REQUIRE_FALSE(d.set_intervals({{0.0f, 1.0f, 3}})); +} + +TEST_CASE("RandomSlicing interval comments round-trip through hostfile", + "[common][malleability][random_slicing]") { + const auto path = (std::filesystem::temp_directory_path() / + "gkfs_rs_interval_comments_test.hosts") + .string(); + { + std::ofstream out(path, std::ios::trunc); + REQUIRE(out.is_open()); + out << "node0 uri0 extra\n"; + out << "+ node1 uri1 extra\n"; + out << "# regular comment stays\n"; + out << "# GKFS_RS_INTERVAL host=99 start=0.0 end=1.0\n"; + } + + std::vector intervals = { + {0, 0.0, 0.5}, {1, 0.5, 1.0}}; + gkfs::malleable::write_rs_interval_comments(path, intervals); + + auto parsed = gkfs::malleable::parse_rs_interval_comments(path); + REQUIRE(parsed.size() == 2); + REQUIRE(parsed[0].host_id == 0); + REQUIRE(parsed[0].start == Catch::Approx(0.0)); + REQUIRE(parsed[0].end == Catch::Approx(0.5)); + REQUIRE(parsed[1].host_id == 1); + REQUIRE(parsed[1].start == Catch::Approx(0.5)); + REQUIRE(parsed[1].end == Catch::Approx(1.0)); + + auto markers = gkfs::malleable::parse_hostfile_markers(path); + REQUIRE(markers.active.size() == 1); + REQUIRE(markers.adding.size() == 1); + gkfs::malleable::write_clean_hostfile(path, markers, parsed); + + auto after_clean_markers = gkfs::malleable::parse_hostfile_markers(path); + REQUIRE(after_clean_markers.active.size() == 2); + REQUIRE(after_clean_markers.adding.empty()); + auto after_clean_intervals = gkfs::malleable::parse_rs_interval_comments(path); + REQUIRE(after_clean_intervals.size() == 2); + REQUIRE(after_clean_intervals[0].host_id == 0); + REQUIRE(after_clean_intervals[1].host_id == 1); + + std::filesystem::remove(path); +} + +TEST_CASE("CutShift shrink 4 to 2 handles arbitrary removed old ids", + "[common][malleability][random_slicing][cutshift]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + auto shrunk = gkfs::rpc::shrink_with_cutshift( + make_equal_partitions(4), removed); + + REQUIRE(shrunk.size() == 2); + REQUIRE(shrunk[0].host_id == 0); + REQUIRE(shrunk[1].host_id == 1); + REQUIRE(coverage(shrunk[0].intervals) == Catch::Approx(0.5f)); + REQUIRE(coverage(shrunk[1].intervals) == Catch::Approx(0.5f)); + + std::vector intervals; + for(const auto& partition : shrunk) { + for(const auto& interval : partition.intervals) { + REQUIRE(interval.host_id == partition.host_id); + intervals.push_back(interval); + } + } + + std::sort(intervals.begin(), intervals.end(), [](const auto& a, + const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } + } +} + +// ===== Compare with SimpleHash: deterministic across same config ===== + +TEST_CASE("RandomSlicingDistributor produces different mapping than SimpleHash", "[common][distributor][random_slicing]") { + auto rs = gkfs::rpc::RandomSlicingDistributor(0, 10); + auto sh = gkfs::rpc::SimpleHashDistributor(0, 10); + + // Collect mappings + std::map rs_map, sh_map; + for (int i = 0; i < 100; ++i) { + rs_map[i] = rs.locate_data("/file", i, 0); + sh_map[i] = sh.locate_data("/file", i, 0); + } + + // They should not be identical (different algorithms) + int diffs = 0; + for (int i = 0; i < 100; ++i) { + if (rs_map[i] != sh_map[i]) diffs++; + } + // Expect at least 50% difference + REQUIRE(diffs > 50); +} + +// ===== Edge cases ===== + +TEST_CASE("RandomSlicingDistributor single host", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 1); + REQUIRE(d.hosts_size() == 1); + + // Everything maps to host 0 + for (int i = 0; i < 100; ++i) { + REQUIRE(d.locate_data("/file", i, 0) == 0); + } +} + +TEST_CASE("RandomSlicingDistributor locate_directory_metadata", "[common][distributor][random_slicing]") { + auto d = gkfs::rpc::RandomSlicingDistributor(0, 5); + + auto hosts = d.locate_directory_metadata(); + REQUIRE(hosts.size() == 5); + // Should include all hosts + for (int i = 0; i < 5; ++i) { + REQUIRE(hosts[i] == static_cast(i)); + } +} \ No newline at end of file diff --git a/tests/unit/test_random_slicing_pipeline.cpp b/tests/unit/test_random_slicing_pipeline.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e58afde9e57caef6d41eef33e155172df9d889c2 --- /dev/null +++ b/tests/unit/test_random_slicing_pipeline.cpp @@ -0,0 +1,530 @@ +/* + * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + * + * This file is part of GekkoFS. + * + * GekkoFS is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * GekkoFS is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with GekkoFS. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace gkfs::rpc; + +// ===== DistributionConfig tests ===== + +TEST_CASE("DistributionConfig default strategy is SimpleHash", "[config][distribution]") { + DistributionConfig config; + REQUIRE(config.get_strategy() == DistributionStrategy::SimpleHash); + REQUIRE(config.is_simple_hash()); + REQUIRE(!config.is_random_slicing()); + REQUIRE(std::string(config.get_strategy_string()) == "simple_hash"); +} + +TEST_CASE("DistributionConfig set_strategy from string", "[config][distribution]") { + DistributionConfig config; + config.set_strategy("random_slicing"); + REQUIRE(config.is_random_slicing()); + REQUIRE(std::string(config.get_strategy_string()) == "random_slicing"); +} + +TEST_CASE("DistributionConfig read_strategy_from_env returns default when unset", "[config][distribution]") { + // Don't actually modify env - test that default is returned + auto strategy = read_strategy_from_env(); + REQUIRE(strategy == DistributionStrategy::SimpleHash); +} + +// ===== Pipeline tests ===== + +TEST_CASE("Pipeline: factory creates RandomSlicing, add_nodes triggers migration estimation", + "[pipeline][random_slicing]") { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + + auto rs = create_distributor_from_string("random_slicing", 0, 3); + REQUIRE(rs != nullptr); + REQUIRE(rs->localhost() == 0); + REQUIRE(rs->hosts_size() == 3); + + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + REQUIRE(old_partitions.size() == 3); + + std::vector new_hosts = {3, 4, 5}; + dist->add_nodes(new_hosts); + + auto new_partitions = dist->get_partitions_copy(); + REQUIRE(new_partitions.size() == 6); +} + +TEST_CASE("Pipeline: RandomSlicing add_nodes uses CutShift when env is enabled", + "[pipeline][random_slicing][cutshift]") { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + auto default_rs = create_distributor_from_string("random_slicing", 0, 3); + auto* default_dist = + dynamic_cast(default_rs.get()); + REQUIRE(default_dist != nullptr); + default_dist->add_nodes({3}); + auto default_partitions = default_dist->get_partitions_copy(); + REQUIRE(default_partitions.size() == 4); + REQUIRE(default_partitions[1].intervals.size() == 1); + REQUIRE(default_partitions[1].intervals[0].start == + Catch::Approx(0.25f)); + + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + auto cutshift_rs = create_distributor_from_string("random_slicing", 0, 3); + auto* cutshift_dist = + dynamic_cast(cutshift_rs.get()); + REQUIRE(cutshift_dist != nullptr); + cutshift_dist->add_nodes({3}); + auto cutshift_partitions = cutshift_dist->get_partitions_copy(); + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + + REQUIRE(cutshift_partitions.size() == 4); + REQUIRE(cutshift_partitions[1].intervals.size() == 1); + REQUIRE(cutshift_partitions[1].intervals[0].start == + Catch::Approx(1.0f / 3.0f)); + REQUIRE(cutshift_partitions[1].intervals[0].end == + Catch::Approx(7.0f / 12.0f)); + REQUIRE(cutshift_partitions[3].host_id == 3); + REQUIRE(!cutshift_partitions[3].intervals.empty()); + for(const auto& interval : cutshift_partitions[3].intervals) { + REQUIRE(interval.host_id == 3); + } +} + +TEST_CASE("Pipeline: CutShift+Sorted produces valid partition updates", + "[pipeline][cutshift][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + + std::vector new_hosts = {3, 4}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(new_partitions.size() == 5); + + // All original hosts should have reduced capacity + for (size_t i = 0; i < 3; ++i) { + REQUIRE(new_partitions[i].total_capacity <= old_partitions[i].total_capacity); + } + + // New hosts should have positive capacity + for (size_t i = 3; i < 5; ++i) { + REQUIRE(new_partitions[i].total_capacity > 0); + REQUIRE(new_partitions[i].host_id == new_hosts[i - 3]); + REQUIRE(!new_partitions[i].intervals.empty()); + for (const auto& interval : new_partitions[i].intervals) { + REQUIRE(interval.host_id == new_partitions[i].host_id); + REQUIRE(interval.start < interval.end); + } + } +} + +TEST_CASE("Pipeline: CutShift+Sorted preserves full coverage without overlap", + "[pipeline][cutshift][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 4); + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + std::vector new_hosts = {4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + std::vector intervals; + for (const auto& partition : new_partitions) { + for (const auto& interval : partition.intervals) { + REQUIRE(interval.start >= 0.0f); + REQUIRE(interval.end <= 1.0f); + REQUIRE(interval.start < interval.end); + REQUIRE(interval.host_id == partition.host_id); + intervals.push_back(interval); + } + } + + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + + REQUIRE(!intervals.empty()); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for (size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i - 1].end <= intervals[i].start + 0.0001f); + } +} + +TEST_CASE("Pipeline: CutShift shrink preserves survivor ownership", + "[pipeline][cutshift][random_slicing][shrink]") { + std::vector old_partitions; + for(host_t host = 0; host < 4; ++host) { + const auto start = static_cast(host) / 4.0f; + const auto end = static_cast(host + 1) / 4.0f; + Partition partition; + partition.host_id = host; + partition.total_capacity = end - start; + partition.intervals.push_back({start, end, host}); + old_partitions.push_back(partition); + } + + auto new_partitions = shrink_with_cutshift(old_partitions, {3}); + + REQUIRE(new_partitions.size() == 3); + for(host_t host = 0; host < 3; ++host) { + REQUIRE(new_partitions[host].host_id == host); + REQUIRE(new_partitions[host].coverage() == Catch::Approx(1.0f / 3.0f)); + bool kept_original = false; + for(const auto& interval : new_partitions[host].intervals) { + REQUIRE(interval.host_id == host); + if(interval.start == Catch::Approx(static_cast(host) / 4.0f) && + interval.end == Catch::Approx(static_cast(host + 1) / 4.0f)) { + kept_original = true; + } + } + REQUIRE(kept_original); + } + + std::vector intervals; + for(const auto& partition : new_partitions) { + intervals.insert(intervals.end(), partition.intervals.begin(), + partition.intervals.end()); + } + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } +} + +TEST_CASE("Pipeline: RandomSlicing CutShift shrink only moves removed-node chunks", + "[pipeline][cutshift][random_slicing][shrink]") { + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + RandomSlicingDistributor old_dist(0, 4); + RandomSlicingDistributor new_dist(0, 4); + new_dist.remove_nodes({3}); + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + + REQUIRE(new_dist.hosts_size() == 3); + + constexpr uint64_t files = 128; + constexpr uint64_t chunks = 128; + uint64_t moved = 0; + uint64_t moved_from_survivors = 0; + std::unordered_set targets; + + for(uint64_t file = 0; file < files; ++file) { + const auto path = "/rs-shrink/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < chunks; ++chunk) { + const auto old_host = old_dist.locate_data(path, chunk, 0); + const auto new_host = new_dist.locate_data(path, chunk, 0); + REQUIRE(new_host < 3); + if(old_host != new_host) { + ++moved; + targets.insert(new_host); + if(old_host < 3) { + ++moved_from_survivors; + } + } + } + } + + REQUIRE(moved > 0); + REQUIRE(targets.size() == 3); + REQUIRE(moved_from_survivors == 0); +} + +TEST_CASE("Pipeline: RandomSlicing 4-to-2 preserves physical survivor ownership", + "[pipeline][cutshift][random_slicing][shrink]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + std::vector old_partitions; + for(host_t host = 0; host < 4; ++host) { + Partition partition; + partition.host_id = host; + partition.total_capacity = 0.25f; + partition.intervals.push_back( + {static_cast(host) / 4.0f, + static_cast(host + 1) / 4.0f, host}); + old_partitions.push_back(partition); + } + + auto new_partitions = shrink_with_cutshift(old_partitions, removed); + REQUIRE(new_partitions.size() == 2); + + std::vector survivors; + for(host_t host = 0; host < 4; ++host) { + if(std::find(removed.begin(), removed.end(), host) == removed.end()) { + survivors.push_back(host); + } + } + + for(size_t compact_id = 0; compact_id < survivors.size(); ++compact_id) { + REQUIRE(new_partitions[compact_id].host_id == compact_id); + bool original_range_preserved = false; + float preserved = 0.0f; + const auto old_start = static_cast(survivors[compact_id]) / 4.0f; + const auto old_end = static_cast(survivors[compact_id] + 1) / 4.0f; + for(const auto& interval : new_partitions[compact_id].intervals) { + REQUIRE(interval.host_id == compact_id); + preserved += std::max(0.0f, std::min(interval.end, old_end) - + std::max(interval.start, old_start)); + } + original_range_preserved = preserved == Catch::Approx(0.25f); + REQUIRE(original_range_preserved); + } + + std::vector intervals; + for(const auto& partition : new_partitions) { + intervals.insert(intervals.end(), partition.intervals.begin(), + partition.intervals.end()); + } + std::sort(intervals.begin(), intervals.end(), [](const auto& a, const auto& b) { + return a.start < b.start; + }); + REQUIRE(intervals.front().start == Catch::Approx(0.0f)); + REQUIRE(intervals.back().end == Catch::Approx(1.0f)); + for(size_t i = 1; i < intervals.size(); ++i) { + REQUIRE(intervals[i].start == Catch::Approx(intervals[i - 1].end)); + } + } +} + +TEST_CASE("Pipeline: RandomSlicing 4-to-2 hash ownership matches migration layout", + "[pipeline][cutshift][random_slicing][shrink][integrity]") { + const std::vector> removed_cases = { + {0, 1}, {0, 2}, {0, 3}, {1, 2}, {1, 3}, {2, 3}}; + + for(const auto& removed : removed_cases) { + CAPTURE(removed[0]); + CAPTURE(removed[1]); + + RandomSlicingDistributor old_dist(0, 4); + const auto old_partitions = old_dist.get_partitions_copy(); + const auto compact_partitions = + shrink_with_cutshift(old_partitions, removed); + + RandomSlicingDistributor new_dist(0, 2); + std::vector final_intervals; + for(const auto& partition : compact_partitions) { + final_intervals.insert(final_intervals.end(), + partition.intervals.begin(), + partition.intervals.end()); + } + REQUIRE(new_dist.set_intervals(final_intervals)); + + std::vector survivor_to_compact; + for(host_t old_host = 0; old_host < 4; ++old_host) { + if(std::find(removed.begin(), removed.end(), old_host) == + removed.end()) { + survivor_to_compact.push_back(old_host); + } + } + + for(uint64_t file = 0; file < 256; ++file) { + const auto path = "/integrity-4to2/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < 256; ++chunk) { + const auto old_owner = old_dist.locate_data(path, chunk, 0); + const auto new_owner = new_dist.locate_data(path, chunk, 0); + REQUIRE(new_owner < 2); + + if(std::find(removed.begin(), removed.end(), old_owner) == + removed.end()) { + const auto survivor_index = static_cast( + std::find(survivor_to_compact.begin(), + survivor_to_compact.end(), old_owner) - + survivor_to_compact.begin()); + CAPTURE(file, chunk, old_owner, new_owner, survivor_index); + REQUIRE(new_owner == survivor_index); + } + } + } + } +} + +TEST_CASE("Pipeline: DataMigrator detects migration, Executor executes it", + "[pipeline][migrator][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + + std::vector new_hosts = {3, 4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(DataMigrator::needs_migration(old_partitions, new_partitions)); + + DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_partitions, new_partitions, 256); + REQUIRE(stats.migrating_chunks > 0); +} + +TEST_CASE("Pipeline: full expansion cycle", + "[pipeline][random_slicing]") { + auto rs = create_distributor_from_string("random_slicing", 0, 3); + auto* dist = dynamic_cast(rs.get()); + REQUIRE(dist != nullptr); + + auto old_partitions = dist->get_partitions_copy(); + REQUIRE(old_partitions.size() == 3); + + std::vector new_hosts = {3, 4, 5}; + auto new_partitions = expand_with_cutshift(old_partitions, new_hosts, 1.0f, 1.0f); + + REQUIRE(new_partitions.size() == 6); + REQUIRE(DataMigrator::needs_migration(old_partitions, new_partitions)); + + DataMigrator migrator; + auto stats = migrator.get_migration_stats(old_partitions, new_partitions, 100); + + REQUIRE(stats.total_chunks == 100); + REQUIRE(stats.migrating_chunks > 0); + + auto result = migrator.compute_migrations(old_partitions, new_partitions, 100); + REQUIRE(!result.empty()); + + DataMigrationExecutor executor; + auto exec_result = executor.execute(result); + REQUIRE(exec_result == MigrationStatus::Success); + + auto exec_stats = executor.get_stats(); + REQUIRE(exec_stats.success_count == result.size()); +} + +TEST_CASE("Pipeline: Multiple expansion steps", + "[pipeline][random_slicing]") { + auto* dist = dynamic_cast( + create_distributor_from_string("random_slicing", 0, 2).get()); + REQUIRE(dist != nullptr); + + auto partitions = dist->get_partitions_copy(); + REQUIRE(partitions.size() == 2); + + std::vector step1 = {2, 3}; + partitions = expand_with_cutshift(partitions, step1, 1.0f, 1.0f); + REQUIRE(partitions.size() == 4); + + std::vector step2 = {4, 5}; + partitions = expand_with_cutshift(partitions, step2, 1.0f, 1.0f); + REQUIRE(partitions.size() == 6); + + std::vector step3 = {6, 7}; + partitions = expand_with_cutshift(partitions, step3, 1.0f, 1.0f); + REQUIRE(partitions.size() == 8); + + for (int i = 0; i < 2; ++i) { + REQUIRE(partitions[i].total_capacity < 1.0f); + } +} + +TEST_CASE("Pipeline: MigrationExecutor batched execution", + "[pipeline][migrator]") { + // Create simple migration scenario + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 50); + REQUIRE(!jobs.empty()); + + // Test batched execution + DataMigrationExecutor executor; + auto result = executor.execute_batched(jobs, 10); + REQUIRE(result == MigrationStatus::Success); + + auto stats = executor.get_stats(); + REQUIRE(stats.success_count == jobs.size()); +} + +TEST_CASE("Pipeline: Progress callback is invoked", + "[pipeline][migrator]") { + gkfs::rpc::Partition p1; + p1.host_id = 1; + p1.total_capacity = 1.0f; + gkfs::rpc::Interval iv1; + iv1.start = 0.0f; iv1.end = 1.0f; iv1.host_id = 1; + p1.intervals.push_back(iv1); + + gkfs::rpc::Partition p2; + p2.host_id = 2; + p2.total_capacity = 1.0f; + gkfs::rpc::Interval iv2; + iv2.start = 0.0f; iv2.end = 1.0f; iv2.host_id = 2; + p2.intervals.push_back(iv2); + + std::vector old_parts = {p1}; + std::vector new_parts = {p2}; + + DataMigrator migrator; + auto jobs = migrator.compute_migrations(old_parts, new_parts, 20); + REQUIRE(!jobs.empty()); + + size_t progress_calls = 0; + DataMigrationExecutor executor; + auto result = executor.execute(jobs, [&progress_calls](size_t, size_t) { + progress_calls++; + }); + + REQUIRE(result == MigrationStatus::Success); + REQUIRE(progress_calls == jobs.size()); +} + +TEST_CASE("Pipeline: DistributionConfig create_from_config", + "[pipeline][config]") { + DistributionConfig config; + config.set_strategy("random_slicing"); + + auto dist = create_from_config(config, 0, 5); + REQUIRE(dist != nullptr); + REQUIRE(dist->localhost() == 0); + REQUIRE(dist->hosts_size() == 5); +} + diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 5283f37e20aff591dc55a8848e844a210d650e60..e137179d80f414256e7dc28e231c138f26f98961 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -41,5 +41,13 @@ add_executable(gkfs_malleability malleability.cpp) target_link_libraries(gkfs_malleability PUBLIC gkfs_user_lib + gkfs_common CLI11::CLI11) -install(TARGETS gkfs_malleability RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) \ No newline at end of file +install(TARGETS gkfs_malleability RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + +add_executable(gkfs_malleability_sim malleability_simulator.cpp) +target_link_libraries(gkfs_malleability_sim + PUBLIC + distributor + CLI11::CLI11) +install(TARGETS gkfs_malleability_sim RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) \ No newline at end of file diff --git a/tools/malleability.cpp b/tools/malleability.cpp index aaccf55b3215bd716bf6d51328fe1bb2994c4d6f..3aab1d2eb06de345c8aa21a138386b5ffdf6c9f0 100644 --- a/tools/malleability.cpp +++ b/tools/malleability.cpp @@ -1,13 +1,70 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This software was partially supported by the + the European Union's Horizon 2020 JTI-EuroHPC research and + innovation programme, by the project ADMIRE (Project ID: 956748, + admire-eurohpc.eu) + + This project was partially promoted by the Ministry for Digital Transformation + and the Civil Service, within the framework of the Recovery, + Transformation and Resilience Plan - Funded by the European Union + -NextGenerationEU. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/** + * @brief Malleability CLI tool for GekkoFS cluster topology changes. + * + * This tool uses a marker-based single-hostfile approach: + * - Normal lines (no prefix) = active daemons + * - Lines with '-' prefix = to-be-removed daemons (user writes manually) + * - Lines with '+' prefix = to-be-added daemons (auto-written by daemons with GKFS_DAEMON_EXPAND=ON) + * + * The malleability CLI auto-discovers before/after state from markers in LIBGKFS_HOSTS_FILE. + * + * Usage: + * gkfs_malleability mutate start # Auto-discovers from LIBGKFS_HOSTS_FILE markers + * gkfs_malleability mutate status # Shows current mutate status + * gkfs_malleability mutate finalize # Rewrites hostfile clean, stops removed daemons + */ + #include #include #include #include +#include #include +#include +#include #include #include #include - +#include using namespace std; namespace fs = std::filesystem; @@ -16,55 +73,15 @@ struct cli_options { bool verbose = false; bool machine_readable = false; string action; - string subcommand; + bool show_status = false; }; -std::pair -get_expansion_host_num() { - // get hosts file and read how much should be expanded - auto hosts_file_path = std::getenv("LIBGKFS_HOSTS_FILE"); - if(!hosts_file_path) { - std::cerr - << "Error: LIBGKFS_HOSTS_FILE environment variable not set.\n"; - return {-1, -1}; - } - std::ifstream file(hosts_file_path); - if(!file) { - std::cerr << "Error: Unable to open file at " << hosts_file_path - << ".\n"; - return {-1, -1}; // Indicate an error - } - auto initialHostCount = 0; - auto finalHostCount = 0; - auto foundSeparator = false; - std::string line; - - while(std::getline(file, line)) { - if(line == "#FS_INSTANCE_END") { - if(foundSeparator) { - cerr << "marker was found twice. this is not allowed.\n"; - return {-1, -1}; - } - foundSeparator = true; - initialHostCount = finalHostCount; - continue; - } - if(!line.empty()) { - finalHostCount++; - } - } - if(!foundSeparator) { - initialHostCount = finalHostCount; - } - return {initialHostCount, finalHostCount}; -} - /** - * Count non-empty, non-comment lines in a hosts file. + * Count non-empty, non-comment, non-marker lines in a hosts file. * Returns -1 on error. */ int -count_hosts_in_file(const std::string& path) { +count_active_hosts_in_file(const std::string& path) { std::ifstream f(path); if(!f) { cerr << "Error: Unable to open file at " << path << ".\n"; @@ -73,15 +90,41 @@ count_hosts_in_file(const std::string& path) { int count = 0; std::string line; while(std::getline(f, line)) { - if(!line.empty() && line[0] != '#') - count++; + if(line.empty()) continue; + if(line[0] == '#') continue; + if(gkfs::malleable::is_marker_line(line)) continue; + count++; } return count; } +/** + * Extract the hostname (first field) from each non-comment line of a hosts file. + * Returns a set of hostnames for comparison. + */ +std::set +extract_hosts_in_file(const std::string& path) { + std::set hosts; + std::ifstream f(path); + if(!f) { + return hosts; + } + std::string line; + while(std::getline(f, line)) { + if(line.empty()) continue; + if(line[0] == '#') continue; + std::istringstream iss(line); + std::string host; + if(std::getline(iss, host, ' ')) { + hosts.insert(host); + } + } + return hosts; +} + int main(int argc, const char* argv[]) { - CLI::App desc{"Allowed options"}; + CLI::App desc{"GekkoFS Malleability Tool - Cluster topology changes"}; cli_options opts; // Global verbose flag @@ -90,31 +133,21 @@ main(int argc, const char* argv[]) { "machine-readable output"); - auto expand_args = - desc.add_subcommand("expand", "Expansion-related actions"); - expand_args->add_option("action", opts.action, "Action to perform") - ->required() - ->check(CLI::IsMember({"start", "status", "finalize"})); - + // Shared variables for mutate subcommand int old_nodes = -1; int new_nodes = -1; - string new_hosts_file; - auto shrink_args = - desc.add_subcommand("shrink", "Shrink-related actions"); - shrink_args->add_option("action", opts.action, "Action to perform") + + auto mutate_args = + desc.add_subcommand("mutate", "Mutate cluster topology (migrate nodes)"); + mutate_args->add_option("action", opts.action, "Action to perform") ->required() ->check(CLI::IsMember({"start", "status", "finalize"})); - shrink_args - ->add_option("--new-hosts-file", new_hosts_file, - "Path to new hosts file listing only the surviving " - "nodes (e.g. gkfs_hosts_new.txt)") - ->envname("LIBGKFS_HOSTS_FILE_NEW"); - shrink_args->add_option("--old-nodes", old_nodes, + mutate_args->add_option("--old-nodes", old_nodes, "Old number of nodes (auto-detected from " "LIBGKFS_HOSTS_FILE if not set)"); - shrink_args->add_option("--new-nodes", new_nodes, + mutate_args->add_option("--new-nodes", new_nodes, "New number of nodes (auto-detected from " - "--new-hosts-file if not set)"); + "LIBGKFS_HOSTS_FILE markers if not set)"); try { desc.parse(argc, argv); @@ -128,120 +161,137 @@ main(int argc, const char* argv[]) { int res; gkfs_init(); - if(expand_args->parsed()) { + if(mutate_args->parsed()) { if(opts.action == "start") { - auto [current_instance, expanded_instance] = - get_expansion_host_num(); - if(current_instance == -1 || expanded_instance == -1) { + // Auto-detect old_nodes and new_nodes from LIBGKFS_HOSTS_FILE markers + auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); + if(!hf) { + cerr << "Error: LIBGKFS_HOSTS_FILE is not set.\n"; return 1; } - res = gkfs::malleable::expand_start(current_instance, - expanded_instance); - if(res) { - cout << "Expand start failed. Exiting...\n"; - gkfs_end(); - cout.flush(); - return -1; - } else { - cerr << "Expansion process from " << current_instance - << " nodes to " << expanded_instance - << " nodes launched...\n"; - } - } else if(opts.action == "status") { - res = gkfs::malleable::expand_status(); - if(res > 0) { - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "Expansion in progress: " << res - << " nodes not finished.\n"; - } - } else { - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "No expansion running/finished.\n"; - } - } - } else if(opts.action == "finalize") { - res = gkfs::malleable::expand_finalize(); - if(opts.machine_readable) { - cerr << res; - } else { - cerr << "Expand finalize " << res << endl; - } - } - } else if(shrink_args->parsed()) { - if(opts.action == "start") { - // new_hosts_file is required for start - if(new_hosts_file.empty()) { - cerr << "Error: --new-hosts-file (or LIBGKFS_HOSTS_FILE_NEW) " - "is required for shrink start.\n"; - return 1; - } - // Auto-detect old_nodes from LIBGKFS_HOSTS_FILE if not given + string workspace_hostfile(hf); + if(old_nodes == -1) { - auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); - if(!hf) { - cerr << "Error: --old-nodes not set and LIBGKFS_HOSTS_FILE " - "is not available.\n"; + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + // old_nodes = active + removing (before state: all nodes before shrink) + old_nodes = static_cast(markers.active.size() + markers.removing.size()); + if(old_nodes <= 0) { + cerr << "Error: Could not determine old node count from '" + << workspace_hostfile << "'. At least one active node is required.\n"; return 1; } - old_nodes = count_hosts_in_file(hf); - if(old_nodes < 0) - return 1; } - // Auto-detect new_nodes from new_hosts_file if not given if(new_nodes == -1) { - new_nodes = count_hosts_in_file(new_hosts_file); - if(new_nodes < 0) + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + new_nodes = static_cast(markers.active.size() + markers.adding.size()); + if(new_nodes <= 0) { + cerr << "Error: Could not determine new node count from '" + << workspace_hostfile << "'.\n"; + return 1; + } + } + + if(old_nodes == new_nodes) { + // Same count: check if host identities actually differ + // For marker-based workflow, we need markers present + auto markers = gkfs::malleable::parse_hostfile_markers(workspace_hostfile); + if(!markers.has_markers()) { + cerr << "Error: Same node count (" << old_nodes << ") with no markers.\n" + << "Add '-' markers for nodes to remove or '+' markers for nodes to add.\n"; return 1; + } + if(markers.active.empty() && !markers.removing.empty() && + !markers.adding.empty()) { + cerr << "Note: Same node count (" << old_nodes + << ") with full replacement via '-' and '+' markers — proceeding with mutate.\n"; + } else { + auto old_hosts = extract_hosts_in_file(workspace_hostfile); + auto removing_hosts = std::set(); + std::ifstream f(workspace_hostfile); + string line; + while(std::getline(f, line)) { + if(gkfs::malleable::is_marker_line(line) && + gkfs::malleable::get_marker(line) == + gkfs::malleable::MARKER_REMOVE) { + std::istringstream iss( + gkfs::malleable::strip_marker(line)); + std::string host; + if(std::getline(iss, host, ' ')) { + removing_hosts.insert(host); + } + } + } + // Remove removing from old to get remaining + for(const auto& h : removing_hosts) { + old_hosts.erase(h); + } + if(old_hosts.empty()) { + cerr << "Warning: All hosts would be removed — aborting.\n"; + return 1; + } + if(old_hosts.size() == + extract_hosts_in_file(workspace_hostfile).size()) { + cerr << "Warning: No effective change detected — proceeding with mutate.\n"; + } else { + cerr << "Note: Same node count (" << old_nodes + << ") with different hosts via markers — proceeding with mutate.\n"; + } + } } - res = gkfs::malleable::shrink_start(old_nodes, new_nodes, - new_hosts_file); + + // For mutate, we use the workspace hostfile directly (no --new-hosts-file needed) + // The daemon parses markers to determine before/after state + res = gkfs::malleable::mutate_start(old_nodes, new_nodes, + workspace_hostfile); if(res) { - cout << "Shrink start failed. Exiting...\n"; + cout << "Mutate start failed. Exiting...\n"; gkfs_end(); cout.flush(); - return -1; + return 1; } else { - cerr << "Shrink process from " << old_nodes << " nodes to " - << new_nodes << " nodes launched...\n"; + cerr << "Mutate process from " << old_nodes + << " nodes to " << new_nodes + << " nodes launched (via LIBGKFS_HOSTS_FILE markers)...\n"; } } else if(opts.action == "status") { - res = gkfs::malleable::shrink_status(); + res = gkfs::malleable::mutate_status(); if(res > 0) { if(opts.machine_readable) { cerr << res; } else { - cerr << "Shrink in progress: " << res + cerr << "Mutate in progress: " << res << " nodes not finished.\n"; } } else { if(opts.machine_readable) { cerr << res; } else { - cerr << "No shrink running/finished.\n"; + cerr << "No mutate running/finished.\n"; } } } else if(opts.action == "finalize") { - res = gkfs::malleable::shrink_finalize(); + res = gkfs::malleable::mutate_finalize(); if(opts.machine_readable) { cerr << res; } else { - cerr << "Shrink finalize " << res << endl; + cerr << "Mutate finalize " << res << endl; } - // On success: move gkfs_hosts_new.txt -> gkfs_hosts.txt - if(res == 0 && !new_hosts_file.empty()) { + // On success: rewrite the workspace hostfile clean + if(res == 0) { auto hf = std::getenv("LIBGKFS_HOSTS_FILE"); if(hf) { + // Parse markers to determine after state + auto markers = gkfs::malleable::parse_hostfile_markers(hf); + auto rs_intervals = + gkfs::malleable::parse_rs_interval_comments(hf); + // Write clean hostfile (promote adding, remove removing) try { - fs::rename(new_hosts_file, hf); - cerr << "Hosts file updated: " << new_hosts_file - << " -> " << hf << "\n"; - } catch(const fs::filesystem_error& e) { - cerr << "Warning: Failed to rename hosts file: " - << e.what() << "\n"; + gkfs::malleable::write_clean_hostfile(hf, markers, + rs_intervals); + cerr << "Hosts file cleaned: " << hf + << " (removed markers, promoted adding to active)\n"; + } catch(const std::exception& e) { + cerr << "Warning: Failed to clean hosts file: " << e.what() << "\n"; } } } diff --git a/tools/malleability_simulator.cpp b/tools/malleability_simulator.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9d044641bd45b61a1092c40aae8039b312f7a7c1 --- /dev/null +++ b/tools/malleability_simulator.cpp @@ -0,0 +1,335 @@ +/* + Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using gkfs::rpc::chunkid_t; +using gkfs::rpc::Distributor; +using gkfs::rpc::host_t; +using gkfs::rpc::RandomSlicingDistributor; +using gkfs::rpc::SimpleHashDistributor; + +struct simulator_config { + unsigned int old_nodes = 4; + unsigned int new_nodes = 2; + std::string strategy = "simple_hash"; + bool cutshift = false; + uint64_t file_size = 1024 * 1024; + uint64_t chunk_size = gkfs::config::rpc::chunksize; + uint64_t files = 1; +}; + +struct host_stats { + uint64_t chunks = 0; + uint64_t bytes = 0; +}; + +struct simulation_result { + std::string strategy; + bool cutshift = false; + unsigned int old_nodes = 0; + unsigned int new_nodes = 0; + uint64_t files = 0; + uint64_t file_size = 0; + uint64_t chunk_size = 0; + uint64_t total_chunks = 0; + uint64_t total_bytes = 0; + uint64_t moved_chunks = 0; + uint64_t moved_bytes = 0; + uint64_t moved_from_removed_chunks = 0; + uint64_t moved_from_removed_bytes = 0; + uint64_t moved_from_survivor_chunks = 0; + uint64_t moved_from_survivor_bytes = 0; + double shrink_lower_bound_ratio = 0.0; + double extra_survivor_movement_ratio = 0.0; + double movement_ratio = 0.0; + double elapsed_ms = 0.0; + std::vector moved_from; + std::vector received_by; +}; + +std::unique_ptr +make_distributor(const std::string& strategy, unsigned int hosts) { + if(strategy == "simple_hash") { + return std::make_unique(0, hosts); + } + if(strategy == "random_slicing") { + return std::make_unique(0, hosts); + } + throw std::runtime_error("unsupported strategy: " + strategy); +} + +std::unique_ptr +make_final_distributor(const simulator_config& cfg) { + if(cfg.strategy == "simple_hash") { + return std::make_unique(0, cfg.new_nodes); + } + + if(cfg.strategy == "random_slicing") { + if(cfg.cutshift) { + setenv(gkfs::env::RANDOM_SLICING_CUTSHIFT, "ON", 1); + } else { + unsetenv(gkfs::env::RANDOM_SLICING_CUTSHIFT); + } + + auto rs = std::make_unique(0, cfg.old_nodes); + if(cfg.new_nodes > cfg.old_nodes) { + std::vector added; + for(unsigned int h = cfg.old_nodes; h < cfg.new_nodes; ++h) { + added.push_back(h); + } + rs->add_nodes(std::move(added)); + } else if(cfg.new_nodes < cfg.old_nodes) { + std::vector removed; + for(unsigned int h = cfg.new_nodes; h < cfg.old_nodes; ++h) { + removed.push_back(h); + } + rs->remove_nodes(std::move(removed)); + } + return rs; + } + + throw std::runtime_error("unsupported strategy: " + cfg.strategy); +} + +uint64_t +chunk_bytes(uint64_t file_size, uint64_t chunk_size, uint64_t chunk_id) { + const auto offset = chunk_id * chunk_size; + const auto remaining = file_size - offset; + return std::min(chunk_size, remaining); +} + +void +print_host_stats(const std::string& title, + const std::vector& stats) { + std::cout << title << "\n"; + for(size_t i = 0; i < stats.size(); ++i) { + std::cout << " host " << i << ": chunks=" << stats[i].chunks + << " bytes=" << stats[i].bytes << "\n"; + } +} + +simulation_result +simulate(simulator_config cfg) { + const auto old_dist = make_distributor(cfg.strategy, cfg.old_nodes); + const auto final_dist = make_final_distributor(cfg); + const auto chunks_per_file = + (cfg.file_size + cfg.chunk_size - 1) / cfg.chunk_size; + + simulation_result result; + result.strategy = cfg.strategy; + result.cutshift = cfg.cutshift; + result.old_nodes = cfg.old_nodes; + result.new_nodes = cfg.new_nodes; + result.files = cfg.files; + result.file_size = cfg.file_size; + result.chunk_size = cfg.chunk_size; + result.total_chunks = chunks_per_file * cfg.files; + result.total_bytes = cfg.file_size * cfg.files; + result.moved_from.resize(cfg.old_nodes); + result.received_by.resize(cfg.new_nodes); + + const auto t0 = std::chrono::steady_clock::now(); + for(uint64_t file = 0; file < cfg.files; ++file) { + const auto path = "/malleability-sim/file-" + std::to_string(file); + for(uint64_t chunk = 0; chunk < chunks_per_file; ++chunk) { + const auto chunk_id = static_cast(chunk); + const auto old_host = old_dist->locate_data(path, chunk_id, 0); + const auto new_host = final_dist->locate_data(path, chunk_id, 0); + if(old_host == new_host) { + continue; + } + + const auto bytes = + chunk_bytes(cfg.file_size, cfg.chunk_size, chunk); + ++result.moved_chunks; + result.moved_bytes += bytes; + if(cfg.new_nodes < cfg.old_nodes && old_host >= cfg.new_nodes) { + ++result.moved_from_removed_chunks; + result.moved_from_removed_bytes += bytes; + } else { + ++result.moved_from_survivor_chunks; + result.moved_from_survivor_bytes += bytes; + } + result.moved_from[old_host].chunks++; + result.moved_from[old_host].bytes += bytes; + result.received_by[new_host].chunks++; + result.received_by[new_host].bytes += bytes; + } + } + const auto t1 = std::chrono::steady_clock::now(); + result.elapsed_ms = + std::chrono::duration(t1 - t0).count(); + result.movement_ratio = + result.total_bytes == 0 + ? 0.0 + : static_cast(result.moved_bytes) / + static_cast(result.total_bytes); + if(cfg.new_nodes < cfg.old_nodes && result.total_bytes != 0) { + result.shrink_lower_bound_ratio = + static_cast(cfg.old_nodes - cfg.new_nodes) / + static_cast(cfg.old_nodes); + result.extra_survivor_movement_ratio = + static_cast(result.moved_from_survivor_bytes) / + static_cast(result.total_bytes); + } + return result; +} + +void +print_single_result(const simulation_result& result) { + std::cout << "strategy=" << result.strategy << "\n"; + std::cout << "cutshift=" << (result.cutshift ? "ON" : "OFF") << "\n"; + std::cout << "old_nodes=" << result.old_nodes << "\n"; + std::cout << "new_nodes=" << result.new_nodes << "\n"; + std::cout << "files=" << result.files << "\n"; + std::cout << "file_size=" << result.file_size << "\n"; + std::cout << "chunk_size=" << result.chunk_size << "\n"; + std::cout << "total_chunks=" << result.total_chunks << "\n"; + std::cout << "total_bytes=" << result.total_bytes << "\n"; + std::cout << "moved_chunks=" << result.moved_chunks << "\n"; + std::cout << "moved_bytes=" << result.moved_bytes << "\n"; + std::cout << "moved_from_removed_chunks=" + << result.moved_from_removed_chunks << "\n"; + std::cout << "moved_from_removed_bytes=" << result.moved_from_removed_bytes + << "\n"; + std::cout << "moved_from_survivor_chunks=" + << result.moved_from_survivor_chunks << "\n"; + std::cout << "moved_from_survivor_bytes=" + << result.moved_from_survivor_bytes << "\n"; + std::cout << std::fixed << std::setprecision(6) << "movement_ratio=" + << result.movement_ratio << "\n"; + if(result.new_nodes < result.old_nodes) { + std::cout << "shrink_lower_bound_ratio=" + << result.shrink_lower_bound_ratio << "\n"; + std::cout << "extra_survivor_movement_ratio=" + << result.extra_survivor_movement_ratio << "\n"; + std::cout + << "definition_moved_from_removed=chunks whose old owner is a removed daemon; unavoidable shrink movement\n"; + std::cout + << "definition_moved_from_survivor=chunks whose old owner is a surviving daemon; extra remap movement\n"; + } + std::cout << "planning_time_ms=" << result.elapsed_ms << "\n"; + print_host_stats("per_source_moved", result.moved_from); + print_host_stats("per_target_received", result.received_by); +} + +void +print_matrix(const std::vector& results) { + std::cout << "strategy comparison\n"; + std::cout << "old_nodes=" << results.front().old_nodes + << " new_nodes=" << results.front().new_nodes + << " files=" << results.front().files + << " file_size=" << results.front().file_size + << " chunk_size=" << results.front().chunk_size << "\n\n"; + + const bool is_shrink = results.front().new_nodes < results.front().old_nodes; + if(is_shrink) { + std::cout + << "removed_mib: moved data whose old owner is a removed daemon (unavoidable shrink lower-bound component)\n"; + std::cout + << "survivor_mib: moved data whose old owner is a surviving daemon (extra remap/rebalance movement)\n\n"; + } + + std::cout << std::left << std::setw(18) << "strategy" << std::setw(10) + << "cutshift" << std::right << std::setw(14) << "moved_bytes" + << std::setw(14) << (is_shrink ? "removed_mib" : "moved_mib") + << std::setw(14) << (is_shrink ? "survivor_mib" : "extra_mib") + << std::setw(14) << "moved_chunks" + << std::setw(12) << "ratio_%" + << std::setw(12) << "time_ms" << "\n"; + std::cout << std::string(108, '-') << "\n"; + + for(const auto& result : results) { + const auto moved_mib = static_cast(result.moved_bytes) / + (1024.0 * 1024.0); + const auto removed_mib = + static_cast(result.moved_from_removed_bytes) / + (1024.0 * 1024.0); + const auto survivor_mib = + static_cast(result.moved_from_survivor_bytes) / + (1024.0 * 1024.0); + std::cout << std::left << std::setw(18) << result.strategy + << std::setw(10) << (result.cutshift ? "ON" : "OFF") + << std::right << std::setw(14) << result.moved_bytes + << std::setw(14) << std::fixed << std::setprecision(3) + << (result.new_nodes < result.old_nodes ? removed_mib + : moved_mib) + << std::setw(14) << survivor_mib << std::setw(14) + << result.moved_chunks + << std::setw(12) << std::fixed << std::setprecision(3) + << (result.movement_ratio * 100.0) << std::setw(12) + << std::fixed << std::setprecision(3) << result.elapsed_ms + << "\n"; + } +} + +} // namespace + +int +main(int argc, char* argv[]) { + simulator_config cfg; + + CLI::App app{"GekkoFS theoretical malleability movement simulator"}; + app.add_option("--old-nodes", cfg.old_nodes, "node count before mutation") + ->check(CLI::PositiveNumber); + app.add_option("--new-nodes", cfg.new_nodes, "node count after mutation") + ->check(CLI::PositiveNumber); + auto strategy_opt = app.add_option( + "--strategy", cfg.strategy, + "distribution strategy: simple_hash|random_slicing. If omitted, run comparison matrix") + ->check(CLI::IsMember({"simple_hash", "random_slicing"})); + app.add_flag("--cutshift", cfg.cutshift, + "use Random Slicing CutShift for expand/shrink simulations"); + app.add_option("--file-size", cfg.file_size, "bytes per file") + ->check(CLI::PositiveNumber); + app.add_option("--chunk-size", cfg.chunk_size, "bytes per chunk") + ->check(CLI::PositiveNumber); + app.add_option("--files", cfg.files, "number of synthetic files") + ->check(CLI::PositiveNumber); + + CLI11_PARSE(app, argc, argv); + + if(strategy_opt->count() == 0) { + std::vector results; + cfg.strategy = "simple_hash"; + cfg.cutshift = false; + results.push_back(simulate(cfg)); + + cfg.strategy = "random_slicing"; + cfg.cutshift = false; + results.push_back(simulate(cfg)); + + cfg.cutshift = true; + results.push_back(simulate(cfg)); + + print_matrix(results); + return 0; + } + + print_single_result(simulate(cfg)); + + return 0; +}