From 56cb4024d713029dafd307006f7a96e5b8a44804 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:30:05 +0200 Subject: [PATCH 1/9] separated tests as manual in ci --- .gitlab-ci.yml | 51 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cdaa7cd9c..bb8849a91 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -95,6 +95,53 @@ scripts: reports: junit: ${BUILD_PATH}/tests/scripts/report.xml +## == integration tests for gkfs =========== +gkfs:allintegration: + stage: test + image: gekkofs/testing:0.9.4-dev + interruptible: true + needs: ['gkfs'] + + script: + ## run tests + - export PATH=${PATH}:/usr/local/bin + - mkdir -p ${BUILD_PATH}/tests/run + - cd ${BUILD_PATH}/tests/integration + - ${PYTEST} -v -n $(nproc) + ${INTEGRATION_TESTS_BIN_PATH}/ + --basetemp=${BUILD_PATH}/tests/run/ + --junit-xml=report.xml + + ## capture coverage information for this test and write it to + ## $COVERAGE_PATH/$SUBTEST.info + - cd ${CI_PROJECT_DIR} + # use ccache + - /usr/sbin/update-ccache-symlinks + - export PATH="/usr/lib/ccache:$PATH" + - cmake --preset ci-coverage + -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/*.info + ## Since the pipeline recreates the source tree, the access times for .gcno + ## files are newer than those of .gcda files. This makes gcov emit a + ## warning for each file which slows it down. Updating the timestamps + ## avoids this + - find ${BUILD_PATH} -name "*.gcno" -exec touch {} \; + - cmake --build ${BUILD_PATH} --target coverage-capture + + # fix relative paths so that GitLab can find the correct files + after_script: + - perl -i.orig + -pe 's%file="(.*?)"%file="tests/integration/$1"%;' + -pe 's%(../)+install/share/gkfs/%%g;' + ${BUILD_PATH}/tests/integration/report.xml + + artifacts: + expire_in: 1 week + when: always + paths: + - ${BUILD_PATH} + reports: + junit: ${BUILD_PATH}/tests/integration/report.xml ## == integration tests for gkfs =========== gkfs:integration: @@ -105,7 +152,9 @@ gkfs:integration: parallel: matrix: - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename ] - + rules: + - if: '$CI_MERGE_REQUEST_ID != ""' + when: manual script: ## run tests - export PATH=${PATH}:/usr/local/bin -- GitLab From 610d5238e9347f1fe8e73c362030126cb89f978a Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:33:03 +0200 Subject: [PATCH 2/9] Test statx >= --- tests/integration/status/test_status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/status/test_status.py b/tests/integration/status/test_status.py index dc0cd0290..6f6fe720e 100644 --- a/tests/integration/status/test_status.py +++ b/tests/integration/status/test_status.py @@ -99,10 +99,10 @@ def test_statx(gkfs_daemon, gkfs_client): # if greater zero, it is activated in config.hpp if ret.statbuf.stx_mtime.tv_sec > 0: - assert (ret.statbuf.stx_mtime.tv_sec > ts) + assert (ret.statbuf.stx_mtime.tv_sec >= ts) if ret.statbuf.stx_ctime.tv_sec > 0: - assert (ret.statbuf.stx_ctime.tv_sec > ts) + assert (ret.statbuf.stx_ctime.tv_sec >= ts) return -- GitLab From 4bc4171dc0ec4e88ef4de69e37470c25c0475eef Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:38:46 +0200 Subject: [PATCH 3/9] typo --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb8849a91..37c2b01ad 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,8 +153,7 @@ gkfs:integration: matrix: - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename ] rules: - - if: '$CI_MERGE_REQUEST_ID != ""' - when: manual + - when: manual script: ## run tests - export PATH=${PATH}:/usr/local/bin -- GitLab From ee8ebf344661f629a407dec29b132ea37c6f6e2e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:41:42 +0200 Subject: [PATCH 4/9] typo --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 37c2b01ad..97f035bc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -357,9 +357,8 @@ coverage:baseline: coverage: stage: report image: gekkofs/testing:0.9.4-dev -# needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration', -# 'gkfs:unit' ] - needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] + #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] + needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfwd:integration'] script: # use ccache - ccache --zero-stats -- GitLab From b3866875235faec8f460fe36066e8e53d106a11f Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:51:01 +0200 Subject: [PATCH 5/9] changed coverage name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97f035bc6..3c95fb695 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -120,7 +120,7 @@ gkfs:allintegration: - export PATH="/usr/lib/ccache:$PATH" - cmake --preset ci-coverage -DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH} - -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/*.info + -DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration.info ## Since the pipeline recreates the source tree, the access times for .gcno ## files are newer than those of .gcda files. This makes gcov emit a ## warning for each file which slows it down. Updating the timestamps -- GitLab From ecd40bc4cc8a7efcfed27f373c1318f5aea1804d Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 11:55:28 +0200 Subject: [PATCH 6/9] removed manal from integration --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c95fb695..cf3d008f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ gkfs:integration: stage: test image: gekkofs/testing:0.9.4-dev interruptible: true - needs: ['gkfs'] + needs: [] # we need to remove gkfs dependencies on manual parallel: matrix: - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename ] -- GitLab From f7599a492a63f72c2d3a0acf590a565bcd4cdd00 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 12:07:13 +0200 Subject: [PATCH 7/9] remove gkfs dependency, manual runnint --- .gitlab-ci.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cf3d008f8..3fb0ce6de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,7 +153,11 @@ gkfs:integration: matrix: - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename ] rules: - - when: manual + - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"' + when: never + - if: '$CI_MERGE_REQUEST_ID != ""' + when: manual + allow_failure: true script: ## run tests - export PATH=${PATH}:/usr/local/bin @@ -201,12 +205,16 @@ gkfwd:integration: stage: test image: gekkofs/testing:0.9.4-dev interruptible: true - needs: ['gkfs'] + needs: [] parallel: matrix: - SUBTEST: [ forwarding ] - # rules: - # - when: never + rules: + - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"' + when: never + - if: '$CI_MERGE_REQUEST_ID != ""' + when: manual + allow_failure: true script: ## run tests @@ -358,7 +366,7 @@ coverage: stage: report image: gekkofs/testing:0.9.4-dev #needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration'] - needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfwd:integration'] + needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit'] script: # use ccache - ccache --zero-stats -- GitLab From 6540d51d6f3cd011b232645d31c8b6a504315e4b Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 12:16:12 +0200 Subject: [PATCH 8/9] conflict files in forwarding test --- tests/integration/forwarding/test_map.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/integration/forwarding/test_map.py b/tests/integration/forwarding/test_map.py index 4bc63e41d..616e5c942 100644 --- a/tests/integration/forwarding/test_map.py +++ b/tests/integration/forwarding/test_map.py @@ -41,6 +41,7 @@ from harness.logger import logger nonexisting = "nonexisting" +# tests can be run in parallel, so it is not safe to have the same file name def test_two_io_nodes(gkfwd_daemon_factory, gkfwd_client_factory): """Write files from two clients using two daemons""" @@ -143,10 +144,10 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): d00 = gkfwd_daemon_factory.create() d01 = gkfwd_daemon_factory.create() - c00 = gkfwd_client_factory.create('c-0') - c01 = gkfwd_client_factory.create('c-1') + c00 = gkfwd_client_factory.create('rc-0') + c01 = gkfwd_client_factory.create('rc-1') - file = d00.mountdir / "file-c00-1" + file = d00.mountdir / "file-rc00-1" # create a file in gekkofs ret = c00.open(file, @@ -176,7 +177,7 @@ def test_two_io_nodes_remap(gkfwd_daemon_factory, gkfwd_client_factory): # we need to wait for at least the number of seconds between remap calls time.sleep(10) - file = d00.mountdir / "file-c00-2" + file = d00.mountdir / "file-rc00-2" # open the file to write ret = c00.open(file, @@ -207,10 +208,10 @@ def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): d00 = gkfwd_daemon_factory.create() d01 = gkfwd_daemon_factory.create() - c00 = gkfwd_client_factory.create('c-0') - c01 = gkfwd_client_factory.create('c-1') + c00 = gkfwd_client_factory.create('oc-0') + c01 = gkfwd_client_factory.create('oc-1') - file = d00.mountdir / "file-c00" + file = d00.mountdir / "file-oc00" # create a file in gekkofs ret = c00.open(file, @@ -256,7 +257,7 @@ def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): assert len(ret.dirents) == 1 - assert ret.dirents[0].d_name == 'file-c00' + assert ret.dirents[0].d_name == 'file-oc00' assert ret.dirents[0].d_type == 8 # DT_REG # the file should be there and accessible by the two clients @@ -264,7 +265,7 @@ def test_two_io_nodes_operations(gkfwd_daemon_factory, gkfwd_client_factory): assert len(ret.dirents) == 1 - assert ret.dirents[0].d_name == 'file-c00' + assert ret.dirents[0].d_name == 'file-oc00' assert ret.dirents[0].d_type == 8 # DT_REG with open(c00.log) as f: -- GitLab From 9f1e6917121441fc5867dd444cff4cedcec306aa Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Oct 2024 12:45:04 +0200 Subject: [PATCH 9/9] correct perl expression typo --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3fb0ce6de..1f8471653 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -131,7 +131,7 @@ gkfs:allintegration: # fix relative paths so that GitLab can find the correct files after_script: - perl -i.orig - -pe 's%file="(.*?)"%file="tests/integration/$1"%;' + -pe 's%file="(.*?)"%file="tests/integration/$1"%g;' -pe 's%(../)+install/share/gkfs/%%g;' ${BUILD_PATH}/tests/integration/report.xml @@ -148,7 +148,7 @@ gkfs:integration: stage: test image: gekkofs/testing:0.9.4-dev interruptible: true - needs: [] # we need to remove gkfs dependencies on manual + needs: ['gkfs'] # we need to remove gkfs dependencies on manual parallel: matrix: - SUBTEST: [ data, status, syscalls, directories, operations, position, shell, rename ] @@ -187,7 +187,7 @@ gkfs:integration: # fix relative paths so that GitLab can find the correct files after_script: - perl -i.orig - -pe 's%file="(.*?)"%file="tests/integration/$1"%;' + -pe 's%file="(.*?)"%file="tests/integration/$1"%g;' -pe 's%(../)+install/share/gkfs/%%g;' ${BUILD_PATH}/tests/integration/report.xml @@ -205,7 +205,7 @@ gkfwd:integration: stage: test image: gekkofs/testing:0.9.4-dev interruptible: true - needs: [] + needs: ['gkfs'] parallel: matrix: - SUBTEST: [ forwarding ] @@ -244,7 +244,7 @@ gkfwd:integration: # fix relative paths so that GitLab can find the correct files after_script: - perl -i.orig - -pe 's%file="(.*?)"%file="tests/integration/$1"%;' + -pe 's%file="(.*?)"%file="tests/integration/$1"%g;' -pe 's%(../)+install/share/gkfs/%%g;' ${BUILD_PATH}/tests/integration/report.xml -- GitLab