testing: establish deterministic local workflow

Add a one-backend smoke test, deterministic random seeds, and stale

build/install-artifact detection.

Status: done. `scripts/test_workflow.py` provides the documented workflow,

manifest/artifact checks, cleanup, coverage entry point, and the one-backend

smoke test. The integration harness records deterministic seed, paths,

environment, hostfile, daemon address, and workspace metadata.

### 26. Expand fault-injection and resilience coverage

Add controlled faults for daemon crash/restart; client disconnect during every

## Integration and functionality tests

### Deterministic local workflow

Use the repository workflow for a clean, reproducible local run. It uses the

prepared dependency prefix `/home/rnou/iodeps` by default, writes build and

install output below `builds/p2.25` and `out/install/p2.25`, and refuses to run

the smoke test when the install manifest or required artifacts are missing.

```console

$ python3 scripts/test_workflow.py configure

$ python3 scripts/test_workflow.py build

$ python3 scripts/test_workflow.py install

$ python3 scripts/test_workflow.py check

$ python3 scripts/test_workflow.py unit

$ python3 scripts/test_workflow.py smoke --seed 424242

$ python3 scripts/test_workflow.py integration

$ python3 scripts/test_workflow.py coverage

$ python3 scripts/test_workflow.py clean

```

Set `GKFS_TEST_SEED` or pass `--seed` to reproduce pseudo-random test data and

port selection. Each integration test workspace logs exact binary and library

search paths, seed, hostfile, patched environment, daemon address, and the

workspace path in `run-metadata.json`. The one-backend smoke test runs the

existing `test_mkdir` case with RocksDB only.

GekkoFS provides an automated testing harness to simplify writing and running

integration and functional tests for the file system. For simplicity and ease of

development, tests are written in Python (3.6+ required). The harness itself

#!/usr/bin/env python3

"""Deterministic local configure, build, install, and test workflow."""

import argparse

import os

import shutil

import subprocess

import sys

from pathlib import Path

DEFAULT_BUILD = "builds/p2.25"

DEFAULT_INSTALL = "out/install/p2.25"

DEFAULT_SEED = "424242"

REQUIRED_ARTIFACTS = (

    "bin/gkfs_daemon",

    "lib64/libgkfs_intercept.so",

)

def run(command, cwd):

    print("$ " + " ".join(str(part) for part in command), flush=True)

    subprocess.run(command, cwd=cwd, check=True)

def artifact_paths(install_dir):

    paths = [install_dir / "bin/gkfs_daemon"]

    library = install_dir / "lib64/libgkfs_intercept.so"

    alternative = install_dir / "lib/libgkfs_intercept.so"

    if not library.is_file() and alternative.is_file():

        library = alternative

    paths.append(library)

    return paths

def check_install(build_dir, install_dir):

    manifest = build_dir / "install_manifest.txt"

    if not manifest.exists():

        return ["missing install manifest: {}".format(manifest)]

    return [str(path) for path in artifact_paths(install_dir)

            if not path.is_file()]

def configure(args):

    args.build_dir.parent.mkdir(parents=True, exist_ok=True)

    args.install_dir.parent.mkdir(parents=True, exist_ok=True)

    run([

        "cmake", "-S", str(args.source_dir), "-B", str(args.build_dir),

        "-G", "Ninja", "-DCMAKE_BUILD_TYPE=Debug",

        "-DCMAKE_PREFIX_PATH={}".format(args.deps_dir),

        "-DCMAKE_INSTALL_PREFIX={}".format(args.install_dir),

        "-DGKFS_BUILD_TESTS=ON", "-DGKFS_INSTALL_TESTS=ON",

        "-DGKFS_BUILD_TOOLS=ON", "-DGKFS_ENABLE_PARALLAX=OFF",

    ], args.source_dir)

def main(argv=None):

    parser = argparse.ArgumentParser(description=__doc__)

    parser.add_argument("command", choices=(

        "configure", "build", "install", "unit", "integration",

        "coverage", "smoke", "check", "clean"))

    parser.add_argument("--source-dir", type=Path,

                        default=Path(__file__).resolve().parent.parent)

    parser.add_argument("--build-dir", type=Path, default=Path(DEFAULT_BUILD))

    parser.add_argument("--install-dir", type=Path,

                        default=Path(DEFAULT_INSTALL))

    parser.add_argument("--deps-dir", type=Path, default=Path("/home/rnou/iodeps"))

    parser.add_argument("--seed", default=os.environ.get("GKFS_TEST_SEED", DEFAULT_SEED))

    parser.add_argument("--jobs", type=int, default=os.cpu_count() or 1)

    args = parser.parse_args(argv)

    args.source_dir = args.source_dir.resolve()

    args.build_dir = args.build_dir.resolve()

    args.install_dir = args.install_dir.resolve()

    args.deps_dir = args.deps_dir.resolve()

    os.environ["GKFS_TEST_SEED"] = str(args.seed)

    if args.command == "clean":

        for path in (args.build_dir, args.install_dir):

            if path.exists():

                print("removing {}".format(path))

                shutil.rmtree(path)

        return 0

    if args.command == "check":

        stale = check_install(args.build_dir, args.install_dir)

        if stale:

            print("stale or incomplete install artifacts:", file=sys.stderr)

            print("\n".join("- " + item for item in stale), file=sys.stderr)

            return 1

        print("install artifacts are complete: {}".format(args.install_dir))

        return 0

    if args.command == "configure":

        configure(args)

    elif args.command == "build":

        run(["cmake", "--build", str(args.build_dir), "--parallel", str(args.jobs)], args.source_dir)

    elif args.command == "install":

        run(["cmake", "--install", str(args.build_dir)], args.source_dir)

    elif args.command == "unit":

        run(["ctest", "--test-dir", str(args.build_dir), "--output-on-failure", "-L", "unit::all"], args.source_dir)

    elif args.command == "integration":

        run(["ctest", "--test-dir", str(args.build_dir), "--output-on-failure"], args.source_dir)

    elif args.command == "coverage":

        run(["cmake", "--build", str(args.build_dir), "--target", "coverage-summary"], args.source_dir)

    elif args.command == "smoke":

        if check_install(args.build_dir, args.install_dir):

            raise SystemExit("run configure, build, and install before smoke")

        env = os.environ.copy()

        env["GKFS_TEST_SEED"] = str(args.seed)

        env["INTEGRATION_TESTS_BIN_PATH"] = str(args.install_dir / "bin")

        env["GKFS_TEST_WORKFLOW"] = "p2.25-smoke"

        command = [

            sys.executable, "-m", "pytest", "-s", "-v",

            "tests/integration/directories/test_directories.py",

            "-k", "test_mkdir", "--interface=lo",

            "--bin-dir={}".format(args.install_dir / "bin"),

            "--bin-dir={}".format(args.build_dir / "tests/integration/harness"),

            "--lib-dir={}".format(args.install_dir / "lib64"),

            "--lib-dir={}".format(args.install_dir / "lib"),

        ]

        print("$ " + " ".join(command), flush=True)

        subprocess.run(command, cwd=args.source_dir, env=env, check=True)

    else:

        raise AssertionError("unhandled command")

    return 0

if __name__ == "__main__":

    raise SystemExit(main())

 No newline at end of file

This directory contains GekkoFS unit, functional, and integration tests. Please

refer to the wiki page about [testing GekkoFS](../-/wikis/Testing) for more

information.

For a documented deterministic configure/build/install/unit/integration/coverage

workflow, run `python3 scripts/test_workflow.py --help` from the repository root.

    Some configurations for our particular usage of pytest

    """

    set_default_log_formatter(config, "%(message)s")

    if config.getoption('--seed') is not None:

        import os

        from harness.run_metadata import set_test_seed

        from harness.gkfs import set_port_seed

        os.environ['GKFS_TEST_SEED'] = str(config.getoption('--seed'))

        set_test_seed(config.getoption('--seed'))

        set_port_seed(config.getoption('--seed'))

def pytest_assertion_pass(item, lineno, orig, expl):