Commit b3c7e3fb authored by Ramon Nou's avatar Ramon Nou
Browse files

ci: split integration tests into separate pipeline job 

Move malleability, forwarding, concurrency, and resilience tests from
gkfs:integration-2 into a new gkfs:integration-3 job with dedicated
JUnit and coverage output. Update the coverage report dependencies to
include the new integration job, reducing load and isolating slower test
groups in CI.
parent 1cedd02a

.gitlab-ci.yml

+51 −11
Original line number Diff line number Diff line
@@ -172,12 +172,6 @@ gkfs:integration-2: 
  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - export GKFS_MALLEABILITY_CI_FAST=ON

    - export GKFS_PERF_REPETITIONS=1

    - export GKFS_PERF_NUM_FILES=1

    - export GKFS_PERF_FILE_SIZE=$((1024 * 1024))

    - export GKFS_MALLEABILITY_OLD_NODES=2

    - export GKFS_MALLEABILITY_NEW_NODES=1

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)
@@ -187,10 +181,6 @@ gkfs:integration-2: 
          ${INTEGRATION_TESTS_BIN_PATH}/rename

          ${INTEGRATION_TESTS_BIN_PATH}/position

          ${INTEGRATION_TESTS_BIN_PATH}/status

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency

          ${INTEGRATION_TESTS_BIN_PATH}/resilience

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report-2.xml
@@ -218,6 +208,56 @@ gkfs:integration-2: 
    reports:

      junit: ${BUILD_PATH}/tests/integration/report-2.xml





gkfs:integration-3:

  stage: test

  image: ${TESTING}

  interruptible: true

  needs: ['gkfs']



  script:

    ## run tests

    - export PATH=${PATH}:/usr/local/bin

    - export GKFS_MALLEABILITY_CI_FAST=ON

    - export GKFS_PERF_REPETITIONS=1

    - export GKFS_PERF_NUM_FILES=1

    - export GKFS_PERF_FILE_SIZE=$((1024 * 1024))

    - export GKFS_MALLEABILITY_OLD_NODES=2

    - export GKFS_MALLEABILITY_NEW_NODES=1

    - mkdir -p ${BUILD_PATH}/tests/run

    - cd ${BUILD_PATH}/tests/integration

    - ${PYTEST} -v -n $(nproc)

          ${INTEGRATION_TESTS_BIN_PATH}/malleability

          ${INTEGRATION_TESTS_BIN_PATH}/forwarding

          ${INTEGRATION_TESTS_BIN_PATH}/concurrency

          ${INTEGRATION_TESTS_BIN_PATH}/resilience

          --basetemp=${BUILD_PATH}/tests/run/

          --junit-xml=report-3.xml



    ## capture coverage information

    - cd ${CI_PROJECT_DIR}

    - /usr/sbin/update-ccache-symlinks

    - export PATH="/usr/lib/ccache:$PATH"

    - cmake --preset ci-coverage

          -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}

          -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration3.info

    - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;

    - cmake --build ${BUILD_PATH} --target coverage-capture



  after_script:

    - perl -i.orig

          -pe 's%file="(.*?)"%file="tests/integration/$1"%g;'

          -pe 's%(../)+install/share/gkfs/%%g;'

          ${BUILD_PATH}/tests/integration/report-3.xml



  artifacts:

    expire_in: 1 day

    when: always

    paths:

      - ${BUILD_PATH}

    reports:

      junit: ${BUILD_PATH}/tests/integration/report-3.xml



## == integration tests for gkfs ===========

gkfs:integration:

  stage: test
@@ -572,7 +612,7 @@ coverage: 
  stage: report

  image: ${TESTING}

  #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']

  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:integration-3', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]

  script:

      # use ccache

    - ccache --zero-stats

README.md

+2 −0
Original line number Diff line number Diff line
@@ -566,6 +566,8 @@ For `random_slicing`, `mutate start` writes the chosen interval table into the w 


CutShift is applied for expand-only Random Slicing operations (`+` entries present, no `-` entries). Other Random Slicing mutate cases currently fall back to the final equal RS layout.



Reference: Random Slicing is based on "Random slicing: Efficient and scalable data placement for large-scale storage systems" by Alberto Miranda, Sascha Effert, Yangwook Kang, Ethan L. Miller, Ivan Popov, Andre Brinkmann, Tom Friedetzky, and Toni Cortes, published in ACM Transactions on Storage 10(3), 2014. See the [Google Scholar entry](https://scholar.google.com/citations?view_op=view_citation&hl=en&user=hK-ogNAAAAAJ&cstart=20&pagesize=80&sortby=pubdate&citation_for_view=hK-ogNAAAAAJ:RGFaLdJalmkC).



### Manual `gkfs_malleability` workflow



Use this when you manage daemon startup and hostfile markers yourself.

tests/apps/lockfile.sh

+1 −1
Original line number Diff line number Diff line
@@ -46,7 +46,7 @@ export LIBGKFS_LOG_OUTPUT=/builds/gitlab/hpc/gekkofs/gkfs/build/tests/run/lockfi 
export LIBGKFS_LOG=all

export LIBGKFS_LOG_SYSCALL_FILTER=epoll_wait,epoll_create



LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex &

LIBGKFS_ENABLE_METRICS=ON LIBGKFS_METRICS_FLUSH_INTERVAL=1 LIBGKFS_PROTECT_FILES_GENERATOR=1 LIBGKFS_PROTECT_FD=1 LD_PRELOAD=$GKFS $APP $MNT/syscall/filex &

GENERATOR=$!

sleep 1

OUTPUTLS=`LIBGKFS_PROTECT_FILES_CONSUMER=1 LD_PRELOAD=$GKFS_LIBC ls $MNT/syscall/`

tests/integration/directories/test_packing_order.py

+32 −27
Original line number Diff line number Diff line 


import pytest

import logging

import re

import time

from harness.gkfs import Daemon, ShellClient, Client, find_command



from harness.gkfs import Client, Daemon, ShellClient, find_command



log = logging.getLogger(__name__)





def test_packing_order_all_fields(test_workspace, request):

    """

    Comprehensive test to verify the packing order of compressed directory entries.
@@ -16,11 +17,13 @@ def test_packing_order_all_fields(test_workspace, request): 
        "GKFS_DAEMON_LOG_LEVEL": "info",

        "GKFS_USE_DIRENTS_COMPRESSION": "ON"

    }

    daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env)

    daemon = Daemon(request.config.getoption('--interface'), "rocksdb",

                    test_workspace, env=daemon_env)

    daemon.run()



    try:

        client = ShellClient(test_workspace)

        shell = ShellClient(test_workspace)

        io_client = Client(test_workspace)

        mount_dir = test_workspace.mountdir

        client_env = {"GKFS_USE_DIRENTS_COMPRESSION": "ON"}
@@ -34,9 +37,12 @@ def test_packing_order_all_fields(test_workspace, request): 
        file_b = mount_dir / "file_b"

        file_c = mount_dir / "file_c"



        # Use dd to create files with specific sizes

        client.run("dd", "if=/dev/zero", f"of={file_a}", "bs=100", "count=1", env=client_env)

        client.run("dd", "if=/dev/zero", f"of={file_b}", "bs=200", "count=1", env=client_env)

        ret = io_client.run("write_sequential", "--pathname", str(file_a),

                            "--count", "1", "--size", "100", env=client_env)

        assert ret.retval == 0

        ret = io_client.run("write_sequential", "--pathname", str(file_b),

                            "--count", "1", "--size", "200", env=client_env)

        assert ret.retval == 0



        log.info("Waiting 2 seconds to distinguish ctimes...")

        time.sleep(2)
@@ -47,14 +53,15 @@ def test_packing_order_all_fields(test_workspace, request): 
        timestamp_file.touch()



        time.sleep(1)

        client.run("dd", "if=/dev/zero", f"of={file_c}", "bs=100", "count=1", env=client_env)

        ret = io_client.run("write_sequential", "--pathname", str(file_c),

                            "--count", "1", "--size", "100", env=client_env)

        assert ret.retval == 0



        sfind_bin = find_command("sfind", test_workspace.bindirs)

        assert sfind_bin, "sfind binary not found"



        common_args = ["-M", str(mount_dir), "-S", "1", "--server-side"]



        import re

        def check_match(ret, expected_matches, expected_total=3):

            output = ret.stdout.decode()

            log.info(f"SFIND OUTPUT:\n{output}")
@@ -66,54 +73,52 @@ def test_packing_order_all_fields(test_workspace, request): 
            total_checked = int(match.group(2))



            log.info(f"Actual matches: {actual_matches}, Total checked: {total_checked}")

            assert actual_matches == expected_matches, f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}"

            assert total_checked == expected_total, f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}"

            assert actual_matches == expected_matches, \

                f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}"

            assert total_checked == expected_total, \

                f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}"



        # Scenario 1: Filter by Name

        log.info("Testing -name filter...")

        ret = client.run(str(sfind_bin), str(mount_dir), "-name", "file_a", *common_args, env=client_env)

        ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "file_a",

                        *common_args, env=client_env)

        assert ret.exit_code == 0

        check_match(ret, 1)



        # Scenario 2: Filter by Size (100c)

        log.info("Testing -size filter...")

        # Should match file_a and file_c (both 100 bytes)

        ret = client.run(str(sfind_bin), str(mount_dir), "-size", "100c", *common_args, env=client_env)

        ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "100c",

                        *common_args, env=client_env)

        assert ret.exit_code == 0

        check_match(ret, 2)



        # Scenario 3: Filter by Size (200c)

        log.info("Testing -size filter (200c)...")

        # Should match file_b

        ret = client.run(str(sfind_bin), str(mount_dir), "-size", "200c", *common_args, env=client_env)

        ret = shell.run(str(sfind_bin), str(mount_dir), "-size", "200c",

                        *common_args, env=client_env)

        assert ret.exit_code == 0

        check_match(ret, 1)



        # Scenario 4: Filter by Newer (ctime)

        log.info("Testing -newer filter...")

        # Should match file_c (created after timestamp_file)

        ret = client.run(str(sfind_bin), str(mount_dir), "-newer", str(timestamp_file), *common_args, env=client_env)

        ret = shell.run(str(sfind_bin), str(mount_dir), "-newer",

                        str(timestamp_file), *common_args, env=client_env)

        assert ret.exit_code == 0

        check_match(ret, 1)



        # Scenario 5: Combined Filter (Name *file* AND Size 100c AND Newer)

        log.info("Testing combined filter...")

        # Should match only file_c

        ret = client.run(str(sfind_bin), str(mount_dir), "-name", "*file*", "-size", "100c", "-newer", str(timestamp_file), *common_args, env=client_env)

        ret = shell.run(str(sfind_bin), str(mount_dir), "-name", "*file*",

                        "-size", "100c", "-newer", str(timestamp_file),

                        *common_args, env=client_env)

        assert ret.exit_code == 0

        check_match(ret, 1)



        log.info("Packing order verification successful!")



    finally:

        # Dump daemon log to inspect scanned keys and ctime

       # daemon_log = test_workspace.logdir / "daemon.log"

       # if daemon_log.exists():

       ##     log.info("DUMPING DAEMON LOG DEBUG ENTRIES:")

        #    with open(daemon_log, 'r') as f:

        #        for line in f:

        #            if "DEBUG" in line:

        #                print(line.strip())

        #else:

        #    log.warning(f"Daemon log not found at {daemon_log}")

        daemon.shutdown()

tests/integration/directories/test_sfind.py

+25 −35
Original line number Diff line number Diff line 


import pytest

import logging

from harness.gkfs import Daemon, ShellClient, Client, find_command

import os

import time



import pytest



from harness.gkfs import Client, Daemon, ShellClient, find_command



log = logging.getLogger(__name__)



SHELL_CHECK_TIMEOUT = 180
@@ -23,7 +23,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 
    1. Populate with Safe Mode (Comp=OFF)

    2. Restart Daemon with Target Mode

    3. Run sfind with Target Client Mode

    4. Run ls with Target Client Mode

    4. Run readdir with Target Client Mode

    """



    # 1. Safe Population (Daemon Comp=OFF)
@@ -32,7 +32,8 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 
        "GKFS_DAEMON_LOG_LEVEL": "info",

        "GKFS_USE_DIRENTS_COMPRESSION": "OFF"

    }

    daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=pop_daemon_env)

    daemon_pop = Daemon(request.config.getoption('--interface'), "rocksdb",

                        test_workspace, env=pop_daemon_env)

    daemon_pop.run()



    try:
@@ -43,18 +44,15 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 
            "LIBGKFS_LOG": "info"

        }



        client = ShellClient(test_workspace)

        shell = ShellClient(test_workspace)

        io_client = Client(test_workspace)

        mount_dir = test_workspace.mountdir

        test_dir = mount_dir / "testdir"



        # Ensure directory exists

        client.run("mkdir", "-p", str(test_dir))

        ret = io_client.mkdir(str(test_dir), 0o755, env=pop_client_env)

        assert ret.retval == 0, f"mkdir failed: {ret.errno}"



        # Populate using create_n_files (faster)

        io_client = Client(test_workspace)

        # Create 2000 files

        ret = io_client.create_n_files(str(test_dir), 2000, env=pop_client_env)

        

        assert ret.retval == 0, f"Population failed: {ret.errno}"

        assert ret.files_created == 2000, f"Expected 2000 files, created {ret.files_created}"

        log.info("Population complete.")
@@ -70,7 +68,8 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 
        "GKFS_DAEMON_LOG_LEVEL": "info",

        "GKFS_USE_DIRENTS_COMPRESSION": conf["compress"]

    }

    daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=test_daemon_env)

    daemon_test = Daemon(request.config.getoption('--interface'), "rocksdb",

                         test_workspace, env=test_daemon_env)

    daemon_test.run()



    try:
@@ -86,40 +85,31 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): 
        assert sfind_bin, "sfind binary not found"



        # --- sfind Check ---

        log.info(f"Running sfind...")

        log.info("Running sfind...")

        # sfind <target_dir> -S 1 -M <mount_dir>

        # Run sfind directly instead of wrapping it in `bash -c`: ShellClient

        # preloads the executed process, and preloading both bash and sfind can

        # make the shell crash during teardown after sfind already printed a

        # successful MATCHED line.

        ret = client.run(

        ret = shell.run(

            str(sfind_bin), str(test_dir), "-S", "1", "-M", str(mount_dir),

            timeout=SHELL_CHECK_TIMEOUT, env=test_client_env)



        sfind_stderr = ret.stderr.decode() if ret.stderr else ""

        sfind_stdout = ret.stdout.decode() if ret.stdout else ""



        assert ret.exit_code == 0, f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}"

        assert "MATCHED 2000/2000" in sfind_stdout, f"sfind did not match 2000/2000. Output:\n{sfind_stdout}"

        assert ret.exit_code == 0, \

            f"sfind failed with {ret.exit_code}\nStderr: {sfind_stderr}\nStdout: {sfind_stdout}"

        assert "MATCHED 2000/2000" in sfind_stdout, \

            f"sfind did not match 2000/2000. Output:\n{sfind_stdout}"

        log.info("sfind verification successful.")



        # --- ls Check ---

        log.info(f"Running ls check...")

        # Avoid `ls -l` here: long format stats every entry and is the first

        # thing to time out on overloaded CI runners.

        ret_ls = client.run(

            "ls", "-1", str(test_dir), timeout=SHELL_CHECK_TIMEOUT,

            env=test_client_env)



        ls_stderr = ret_ls.stderr.decode() if ret_ls.stderr else ""

        ls_stdout = ret_ls.stdout.decode() if ret_ls.stdout else ""



        assert ret_ls.exit_code == 0, f"ls check failed with {ret_ls.exit_code}\nStderr: {ls_stderr}"



        count = sum(1 for line in ls_stdout.splitlines()

                    if line.startswith("file_"))

        assert count == 2000, f"ls count expected 2000, got '{count}'"

        log.info("ls verification successful.")

        log.info("Running readdir check...")

        ret_ls = io_client.readdir(str(test_dir), env=test_client_env)

        assert ret_ls.errno == 0, f"readdir failed: {ret_ls.errno}"

        count = sum(1 for entry in ret_ls.dirents if entry.d_name.startswith("file_"))

        assert count == 2000, f"readdir count expected 2000, got '{count}'"

        log.info("readdir verification successful.")



    finally:

        daemon_test.shutdown()