test: expand resilience fault coverage

migration cancellation; and duplicate jobs. Each fault needs automated expected

recovery/data-availability/error assertions or a documented limitation.

**Status: Done for the supported injection scope.** Controlled coverage now

asserts daemon crash/restart data recovery, client disconnect survival,

endpoint lookup failure containment, hostfile race behavior, local storage

failure handling, and duplicate/conflicting mutation request semantics. Existing

RPC-disconnect, startup-error, malleability-error, and backend-cleanup tests

provide the remaining supported cases.

Packet delay/loss, true disk-full, short read/write, metadata-backend process

restart, and migration cancellation remain documented limitations: the current

test harness has no safe network/filesystem/backend fault-injection interface

for these cases. They must not be treated as covered until such hooks exist.

### 27. Add sanitizer, static-analysis, and ABI checks

Run AddressSanitizer and UndefinedBehaviorSanitizer on unit and selected

import time

import signal

def test_daemon_crash_recovery(gkfs_daemon, gkfs_client):

    """

    Test resilience: Write data, kill daemon, restart, verify data persistence.

    res = gkfs_client.write(persist_file, data, len(data))

    assert res.retval == len(data)

    try:

        # 2. Kill Daemon

        daemon_pid = gkfs_daemon._proc.pid

        print(f"\nKilling Daemon PID: {daemon_pid}")

        except ChildProcessError:

            pass

    # 3. Attempt Client Operation (Should Fail)

    # We don't verify here as it might depend on client-side retry logic

    # but the daemon is dead, so it should eventually fail.

    # 4. Restart Daemon

        # 3. Restart Daemon

        print("\nRestarting Daemon...")

    if gkfs_daemon._stdout: gkfs_daemon._stdout.close()

        if gkfs_daemon._stdout:

            gkfs_daemon._stdout.close()

            gkfs_daemon._stdout = None

        gkfs_daemon.run()

    # 5. Verify Persistence of Old Data

        # 4. Verify Persistence of Old Data

        res = gkfs_client.read(persist_file, len(data))

        assert res.retval == len(data)

        assert res.buf == data

    # 6. Verify New Ops Work

        # 5. Verify New Ops Work

        new_file = gkfs_daemon.mountdir / 'new_file'

        res = gkfs_client.open(new_file, os.O_CREAT | os.O_WRONLY)

        assert res.retval != -1

        res = gkfs_client.write(new_file, b"new data", 8)

        assert res.retval == 8

    finally:

        gkfs_daemon.shutdown()

"""Controlled resilience faults supported by the current test harness.

The tests in this module inject failures at process, configuration, and local

filesystem boundaries. Network packet loss, true disk exhaustion, short I/O,

backend-process restart, and migration cancellation require hooks that are not

available in the current harness; those limitations are recorded in WIP.md.

"""

import os

import threading

import time

from pathlib import Path

def _assert_daemon_alive(daemon):

    assert daemon._proc is not None

    assert daemon._proc.poll() is None, (

        f"daemon exited unexpectedly: {daemon._proc.returncode}"

    )

def test_endpoint_lookup_failure_does_not_kill_daemon(

    gkfwd_daemon_factory, gkfs_shell

):

    daemon = gkfwd_daemon_factory.create()

    try:

        invalid_hostfile = Path(daemon.hostfile).with_name("invalid-endpoint.txt")

        invalid_hostfile.write_text(

            "# valid syntax, but no daemon owns this endpoint\n"

            "127.0.0.1:1\n"

        )

        result = gkfs_shell.script(

            f'LIBGKFS_HOSTS_FILE="{invalid_hostfile}" '

            "gkfs_malleability mutate status",

            intercept_shell=False,

        )

        assert result.exit_code != 0

        _assert_daemon_alive(daemon)

        health = gkfs_shell.script("printf healthy", intercept_shell=False)

        assert health.exit_code == 0

        assert health.stdout == b"healthy"

    finally:

        daemon.shutdown()

def test_hostfile_race_never_publishes_partial_configuration(

    gkfwd_daemon_factory, gkfs_shell

):

    daemon = gkfwd_daemon_factory.create()

    hostfile = Path(daemon.hostfile)

    original = hostfile.read_text()

    invalid = "127.0.0.1:1\n"

    stop = threading.Event()

    def mutate_hostfile():

        while not stop.is_set():

            temporary = hostfile.with_suffix(".tmp")

            temporary.write_text(original)

            os.replace(temporary, hostfile)

            temporary.write_text(invalid)

            os.replace(temporary, hostfile)

    writer = threading.Thread(target=mutate_hostfile)

    writer.start()

    failures = 0

    try:

        for _ in range(20):

            result = gkfs_shell.script(

                f'LIBGKFS_HOSTS_FILE="{hostfile}" '

                "gkfs_malleability mutate status",

                intercept_shell=False,

            )

            if result.exit_code != 0:

                failures += 1

            _assert_daemon_alive(daemon)

    finally:

        stop.set()

        writer.join(timeout=5)

        hostfile.write_text(original)

        daemon.shutdown()

    assert failures >= 1, "race did not exercise an invalid hostfile snapshot"

    ${CMAKE_CURRENT_LIST_DIR}/test_env_util.cpp

    ${CMAKE_CURRENT_LIST_DIR}/test_stats.cpp

    ${CMAKE_CURRENT_LIST_DIR}/test_unique_fd.cpp

    ${CMAKE_SOURCE_DIR}/src/common/hostfile_management.cpp)

    ${CMAKE_CURRENT_LIST_DIR}/test_fs_data_faults.cpp

    ${CMAKE_SOURCE_DIR}/src/common/hostfile_management.cpp

    ${CMAKE_SOURCE_DIR}/src/daemon/classes/fs_data.cpp)

if (GKFS_TESTS_GUIDED_DISTRIBUTION)

    target_sources(unit_tests PRIVATE ${CMAKE_CURRENT_LIST_DIR}/test_guided_distributor.cpp)

/*

 * Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain

 * Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany

 *

 * This file is part of GekkoFS.

 *

 * SPDX-License-Identifier: GPL-3.0-or-later

 */

#include <daemon/classes/fs_data.hpp>

#include <catch2/catch_all.hpp>

#include <stdexcept>

TEST_CASE("Duplicate mutation requests are idempotent",

          "[resilience][mutation]") {

    auto* data = gkfs::daemon::FsData::getInstance();

    data->end_mutate_start();

    REQUIRE_FALSE(data->begin_mutate_start(1, 2, "/tmp/hosts-a"));

    REQUIRE(data->mutate_start_active());

    REQUIRE(data->begin_mutate_start(1, 2, "/tmp/hosts-a"));

    data->end_mutate_start();

    REQUIRE_FALSE(data->mutate_start_active());

}

TEST_CASE("Conflicting mutation requests are rejected",

          "[resilience][mutation]") {

    auto* data = gkfs::daemon::FsData::getInstance();

    data->end_mutate_start();

    REQUIRE_FALSE(data->begin_mutate_start(1, 2, "/tmp/hosts-a"));

    REQUIRE_THROWS_WITH(

            data->begin_mutate_start(1, 3, "/tmp/hosts-b"),

            "A different mutate operation is already active");

    data->end_mutate_start();

}

 No newline at end of file