Merge branch 'rnou/382-test-fails-sometimes' into 'master' (d412bf3d) · Commits · hpc / gekkofs

.gitlab-ci.yml

+96 −34

Original line number	Diff line number	Diff line
		@@ -7,6 +7,7 @@ stages:
		- deploy

		variables:
		VERSION: "0.9.6"
		SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts"
		CI_SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts/ci"
		BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build"
		@@ -23,19 +24,28 @@ variables:
		LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
		GIT_SUBMODULE_STRATEGY: recursive
		CCACHE_DIR: "${CI_PROJECT_DIR}/ccache"
		# SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar" # Defines the location of the analysis task cache
		# GIT_DEPTH: "0" # Tells git to fetch all the branches of the project, required by the analysis task

		IMAGE_PREFIX: "${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/"
		#IMAGE_PREFIX: "gekkofs/"

		CORE: "${IMAGE_PREFIX}core:${VERSION}"
		LINTER: "${IMAGE_PREFIX}linter:${VERSION}"
		DEPS: "${IMAGE_PREFIX}deps:${VERSION}"
		TESTING: "${IMAGE_PREFIX}testing:${VERSION}"
		APPS: "${IMAGE_PREFIX}apps:${VERSION}"
		JAVA: "${IMAGE_PREFIX}java:${VERSION}"
		DOCS: "${IMAGE_PREFIX}docs:${VERSION}"


		# base image
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/core:0.9.6
		image: ${CORE}

		################################################################################
		## Validating
		################################################################################
		check format:
		stage: lint
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/linter:0.9.6
		image: ${LINTER}
		needs: []
		script:
		- ${SCRIPTS_DIR}/check_format.sh
		@@ -49,7 +59,7 @@ check format:
		################################################################################
		gkfs:
		stage: build
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/deps:0.9.6
		image: ${DEPS}
		interruptible: true
		needs: []
		script:
		@@ -88,7 +98,7 @@ gkfs:
		## == tests for scripts ====================
		scripts:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		needs: []
		script:
		- mkdir -p ${BUILD_PATH}/tests/scripts
		@@ -100,10 +110,10 @@ scripts:
		reports:
		junit: ${BUILD_PATH}/tests/scripts/report.xml

		## == integration tests for gkfs ===========
		gkfs:allintegration:
		## == integration tests for gkfs (group 1: I/O, FUSE, shell) ================
		gkfs:integration-1:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']

		@@ -113,32 +123,30 @@ gkfs:allintegration:
		- mkdir -p ${BUILD_PATH}/tests/run
		- cd ${BUILD_PATH}/tests/integration
		- ${PYTEST} -v -n $(nproc)
		${INTEGRATION_TESTS_BIN_PATH}/
		${INTEGRATION_TESTS_BIN_PATH}/data
		${INTEGRATION_TESTS_BIN_PATH}/fuse
		${INTEGRATION_TESTS_BIN_PATH}/shell
		${INTEGRATION_TESTS_BIN_PATH}/compatibility
		${INTEGRATION_TESTS_BIN_PATH}/startup
		${INTEGRATION_TESTS_BIN_PATH}/error_handling
		--basetemp=${BUILD_PATH}/tests/run/
		--junit-xml=report.xml
		--junit-xml=report-1.xml

		## capture coverage information for this test and write it to
		## $COVERAGE_PATH/$SUBTEST.info
		## capture coverage information
		- cd ${CI_PROJECT_DIR}
		# use ccache
		- /usr/sbin/update-ccache-symlinks
		- export PATH="/usr/lib/ccache:$PATH"
		- cmake --preset ci-coverage
		-DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
		-DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration.info
		## Since the pipeline recreates the source tree, the access times for .gcno
		## files are newer than those of .gcda files. This makes gcov emit a
		## warning for each file which slows it down. Updating the timestamps
		## avoids this
		-DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration1.info
		- find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;
		- cmake --build ${BUILD_PATH} --target coverage-capture

		# fix relative paths so that GitLab can find the correct files
		after_script:
		- perl -i.orig
		-pe 's%file="(.*?)"%file="tests/integration/$1"%g;'
		-pe 's%(../)+install/share/gkfs/%%g;'
		${BUILD_PATH}/tests/integration/report.xml
		${BUILD_PATH}/tests/integration/report-1.xml

		artifacts:
		expire_in: 1 day
		@@ -146,12 +154,62 @@ gkfs:allintegration:
		paths:
		- ${BUILD_PATH}
		reports:
		junit: ${BUILD_PATH}/tests/integration/report.xml
		junit: ${BUILD_PATH}/tests/integration/report-1.xml


		gkfs:integration-2:
		stage: test
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']

		script:
		## run tests
		- export PATH=${PATH}:/usr/local/bin
		- mkdir -p ${BUILD_PATH}/tests/run
		- cd ${BUILD_PATH}/tests/integration
		- ${PYTEST} -v -n $(nproc)
		${INTEGRATION_TESTS_BIN_PATH}/syscalls
		${INTEGRATION_TESTS_BIN_PATH}/directories
		${INTEGRATION_TESTS_BIN_PATH}/operations
		${INTEGRATION_TESTS_BIN_PATH}/rename
		${INTEGRATION_TESTS_BIN_PATH}/position
		${INTEGRATION_TESTS_BIN_PATH}/status
		${INTEGRATION_TESTS_BIN_PATH}/malleability
		${INTEGRATION_TESTS_BIN_PATH}/forwarding
		${INTEGRATION_TESTS_BIN_PATH}/concurrency
		${INTEGRATION_TESTS_BIN_PATH}/resilience
		--basetemp=${BUILD_PATH}/tests/run/
		--junit-xml=report-2.xml

		## capture coverage information
		- cd ${CI_PROJECT_DIR}
		- /usr/sbin/update-ccache-symlinks
		- export PATH="/usr/lib/ccache:$PATH"
		- cmake --preset ci-coverage
		-DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
		-DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration2.info
		- find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;
		- cmake --build ${BUILD_PATH} --target coverage-capture

		after_script:
		- perl -i.orig
		-pe 's%file="(.*?)"%file="tests/integration/$1"%g;'
		-pe 's%(../)+install/share/gkfs/%%g;'
		${BUILD_PATH}/tests/integration/report-2.xml

		artifacts:
		expire_in: 1 day
		when: always
		paths:
		- ${BUILD_PATH}
		reports:
		junit: ${BUILD_PATH}/tests/integration/report-2.xml

		## == integration tests for gkfs ===========
		gkfs:integration:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs'] # we need to remove gkfs dependencies on manual
		parallel:
		@@ -209,7 +267,7 @@ gkfs:integration:
		## == integration tests for gkfwd ==========
		gkfwd:integration:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']
		parallel:
		@@ -267,7 +325,8 @@ gkfwd:integration:
		## == unit tests for gkfs ==================
		gkfs:unit:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']
		script:
		## Add path to mkfs.kreon
		@@ -307,7 +366,8 @@ gkfs:unit:
		## == unit tests for gkfs ==================
		gkfs:app:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/apps:0.9.6
		image: ${APPS}
		interruptible: true
		needs: ['gkfs']
		script:
		## Add path to mkfs.kreon
		@@ -348,7 +408,8 @@ gkfs:app:
		## == java tests for gkfs ==================
		gkfs:java:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/java:0.9.6
		image: ${JAVA}
		interruptible: true
		needs: ['gkfs']
		script:
		## Add path to mkfs.kreon
		@@ -389,7 +450,8 @@ gkfs:java:
		## == python tests for gkfs ==================
		gkfs:python:
		stage: test
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']
		script:
		## Add path to mkfs.kreon
		@@ -432,7 +494,7 @@ gkfs:python:
		################################################################################
		documentation:
		stage: docs
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/docs:0.9.6
		image: ${DOCS}
		needs: []
		rules:
		# we only build the documentation automatically if we are on the
		@@ -465,7 +527,7 @@ documentation:
		## == coverage baseline ====================
		coverage:baseline:
		stage: report
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']

		@@ -496,9 +558,9 @@ coverage:baseline:

		coverage:
		stage: report
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		#needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
		needs: [ 'coverage:baseline', 'gkfs:allintegration', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
		needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
		script:
		# use ccache
		- ccache --zero-stats
		@@ -528,7 +590,7 @@ coverage:

		cppcheck:
		stage: report
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/gekkofs/testing:0.9.6
		image: ${TESTING}
		needs: [ 'gkfs']
		script:
		- cd ${CI_PROJECT_DIR}
		@@ -554,7 +616,7 @@ cppcheck:
		## for DEPLOY_KEY_FILE, DEPLOY_USERNAME, DEPLOY_GROUP, DEPLOY_SERVER and
		## DEPLOY_PATH must be defined as protected variables.
		deploy:
		image: ${CI_DEPENDENCY_PROXY_GROUP_IMAGE_PREFIX}/bscstorage/deployer
		image: bscstorage/deployer
		stage: deploy
		needs: [ 'documentation' ]
		only:

docker/0.9.6/deps/Dockerfile

+2 −1

Original line number	Diff line number	Diff line
		@@ -23,7 +23,8 @@ RUN apt-get update && \
		python3-setuptools \
		libnuma-dev libyaml-dev libcurl4-openssl-dev \
		libfuse3-dev fuse3 \
		procps && \
		procps \
		kmod && \
		rm -rf /var/lib/apt/lists/* && \
		apt-get clean && apt-get autoclean

include/client/open_file_map.hpp

+1 −1

Original line number	Diff line number	Diff line
		@@ -66,7 +66,7 @@ enum class OpenFile_flags {
		flag_count // this is purely used as a size variable of this enum class
		};

		enum class FileType { regular, directory };
		enum class FileType { regular, directory, symlink };

		class OpenFile {
		protected:

include/client/rpc/forward_metadata.hpp

+4 −3

Original line number	Diff line number	Diff line
		@@ -101,8 +101,8 @@ forward_get_metadentry_size(const std::string& path, const int copy);
		std::pair<int, std::shared_ptr<gkfs::filemap::OpenDir>>
		forward_get_dirents(const std::string& path);

		std::pair<int, std::unique_ptr<std::vector<
		std::tuple<const std::string, bool, size_t, time_t>>>>
		std::pair<int, std::unique_ptr<std::vector<std::tuple<
		const std::string, unsigned char, size_t, time_t>>>>
		forward_get_dirents_single(const std::string& path, int server,
		const std::string& start_key = "",
		bool get_all = true);
		@@ -143,7 +143,8 @@ forward_read_inline(const std::string& path, void* buf, off64_t offset,
		size_t read_size);

		std::tuple<int,
		std::vector<std::tuple<const std::string, bool, size_t, time_t>>,
		std::vector<std::tuple<const std::string, unsigned char, size_t,
		time_t>>,
		uint64_t, std::string>
		forward_get_dirents_filtered(const std::string& path, int server,
		const std::string& start_key,

include/client/rpc/forward_metadata_proxy.hpp

+2 −2

Original line number	Diff line number	Diff line
		@@ -52,8 +52,8 @@ forward_update_metadentry_size_proxy(const std::string& path, const size_t size,
		std::pair<int, off64_t>
		forward_get_metadentry_size_proxy(const std::string& path);

		std::pair<int, std::unique_ptr<std::vector<
		std::tuple<const std::string, bool, size_t, time_t>>>>
		std::pair<int, std::unique_ptr<std::vector<std::tuple<
		const std::string, unsigned char, size_t, time_t>>>>
		forward_get_dirents_single_proxy_v2(const std::string& path, int server);

		} // namespace gkfs::rpc