Merge branch 'rnou/rslice' into 'master' (aa661713) · Commits · hpc / gekkofs

.gitlab-ci.yml

+51 −5

Original line number	Diff line number	Diff line
		@@ -181,10 +181,6 @@ gkfs:integration-2:
		${INTEGRATION_TESTS_BIN_PATH}/rename
		${INTEGRATION_TESTS_BIN_PATH}/position
		${INTEGRATION_TESTS_BIN_PATH}/status
		${INTEGRATION_TESTS_BIN_PATH}/malleability
		${INTEGRATION_TESTS_BIN_PATH}/forwarding
		${INTEGRATION_TESTS_BIN_PATH}/concurrency
		${INTEGRATION_TESTS_BIN_PATH}/resilience
		--basetemp=${BUILD_PATH}/tests/run/
		--junit-xml=report-2.xml

		@@ -212,6 +208,56 @@ gkfs:integration-2:
		reports:
		junit: ${BUILD_PATH}/tests/integration/report-2.xml


		gkfs:integration-3:
		stage: test
		image: ${TESTING}
		interruptible: true
		needs: ['gkfs']

		script:
		## run tests
		- export PATH=${PATH}:/usr/local/bin
		- export GKFS_MALLEABILITY_CI_FAST=ON
		- export GKFS_PERF_REPETITIONS=1
		- export GKFS_PERF_NUM_FILES=1
		- export GKFS_PERF_FILE_SIZE=$((1024 * 1024))
		- export GKFS_MALLEABILITY_OLD_NODES=2
		- export GKFS_MALLEABILITY_NEW_NODES=1
		- mkdir -p ${BUILD_PATH}/tests/run
		- cd ${BUILD_PATH}/tests/integration
		- ${PYTEST} -v -n $(nproc)
		${INTEGRATION_TESTS_BIN_PATH}/malleability
		${INTEGRATION_TESTS_BIN_PATH}/forwarding
		${INTEGRATION_TESTS_BIN_PATH}/concurrency
		${INTEGRATION_TESTS_BIN_PATH}/resilience
		--basetemp=${BUILD_PATH}/tests/run/
		--junit-xml=report-3.xml

		## capture coverage information
		- cd ${CI_PROJECT_DIR}
		- /usr/sbin/update-ccache-symlinks
		- export PATH="/usr/lib/ccache:$PATH"
		- cmake --preset ci-coverage
		-DCOVERAGE_OUTPUT_DIR=${COVERAGE_PATH}
		-DCOVERAGE_CAPTURE_TRACEFILE=${COVERAGE_PATH}/integration3.info
		- find ${BUILD_PATH} -name "*.gcno" -exec touch {} \;
		- cmake --build ${BUILD_PATH} --target coverage-capture

		after_script:
		- perl -i.orig
		-pe 's%file="(.*?)"%file="tests/integration/$1"%g;'
		-pe 's%(../)+install/share/gkfs/%%g;'
		${BUILD_PATH}/tests/integration/report-3.xml

		artifacts:
		expire_in: 1 day
		when: always
		paths:
		- ${BUILD_PATH}
		reports:
		junit: ${BUILD_PATH}/tests/integration/report-3.xml

		## == integration tests for gkfs ===========
		gkfs:integration:
		stage: test
		@@ -566,7 +612,7 @@ coverage:
		stage: report
		image: ${TESTING}
		#needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
		needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
		needs: [ 'coverage:baseline', 'gkfs:integration-1', 'gkfs:integration-2', 'gkfs:integration-3', 'gkfs:unit', 'gkfs:app', 'gkfs:java', 'gkfs:python' ]
		script:
		# use ccache
		- ccache --zero-stats

CHANGELOG.md

+8 −2

Original line number	Diff line number	Diff line
		@@ -47,8 +47,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
		- Client interface and hostfile removel ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313))
		- Demon option to avoid removing hostfile on exit : ENV variable , GKFS_KEEP_HOSTS_FILE or option --keep-hosts
		- Client Interface selection avoiding loopback setup : LIBGKFS_OFI_INTERFACE=ib0


		- Random Slicing + cutshift ([!316](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/316))
		- Added marker-based expand, shrink, and mixed mutate workflow using one shared hostfile.
		- Added Random Slicing placement with CutShift for expand-only resizes to reduce inter-node data movement.
		- Added optional expand-on-demand data movement for pure expand operations.
		- Added malleability integration coverage and user documentation.


		### Changed
		@@ -60,6 +63,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
		- LIBGKFS/ GKFS _SYMLINK_SUPPORT, _RENAME_SUPPORT and _CREATE_CHECK_PARENTS.
		- Now all the performance options are in config.hpp and env variables.
		- Added DIRECT_IO on server side to increase performance ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))
		- Shared metrics/message routing variables now use the common `GKFS_` prefix.


		### Fixed
		- SYS_lstat does not exists on some architectures, change to newfstatat ([!269](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/269))
		@@ -71,6 +76,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
		- Fix remove chunk bug ([!294](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/294))
		- Fix decompress_and_parse_entries_standard() Unexpected end of buffer while parsing name bug ([!312](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/312))
		- Fix client dissapearing on malleability ends with an error. ([!313](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/313))
		- Fixed buffer-size validation issues in filtered directory listing paths.


		## [0.9.5] - 2025-08

README.md

+335 −102

File changed.

Preview size limit exceeded, changes collapsed.

examples/gfind/sfind.cpp

+12 −4

Original line number	Diff line number	Diff line
		@@ -302,11 +302,15 @@ worker_routine(void* arg) {

		if(n > 0 && entries && !data->opt->just_count) {
		char* ptr = reinterpret_cast<char*>(entries);
		int bytes_processed = 0;
		size_t bytes_processed = 0;
		while(bytes_processed < n) {
		struct dirent_extended* temp =
		reinterpret_cast<struct dirent_extended*>(ptr);
		if(temp->d_reclen == 0)
		const auto remaining =
		static_cast<size_t>(n) - bytes_processed;
		if(temp->d_reclen <
		offsetof(struct dirent_extended, d_name) + 1 \|\|
		temp->d_reclen > remaining)
		break;

		local_found++;
		@@ -360,11 +364,15 @@ worker_routine(void* arg) {
		}

		char* ptr = reinterpret_cast<char*>(entries);
		int bytes_processed = 0;
		size_t bytes_processed = 0;
		while(bytes_processed < n) {
		struct dirent_extended* temp =
		reinterpret_cast<struct dirent_extended*>(ptr);
		if(temp->d_reclen == 0)
		const auto remaining =
		static_cast<size_t>(n) - bytes_processed;
		if(temp->d_reclen <
		offsetof(struct dirent_extended, d_name) + 1 \|\|
		temp->d_reclen > remaining)
		break;

		if(temp->d_type != 1) {

include/client/env.hpp

+5 −6

Original line number	Diff line number	Diff line
		@@ -83,8 +83,6 @@ static constexpr auto PROTECT_FILES_CONSUMER =
		ADD_PREFIX("PROTECT_FILES_CONSUMER");
		static constexpr auto RANGE_FD = ADD_PREFIX("RANGE_FD");
		static constexpr auto DIRENTS_BUFF_SIZE = ADD_PREFIX("DIRENTS_BUFF_SIZE");
		static constexpr auto USE_DIRENTS_COMPRESSION =
		ADD_PREFIX("USE_DIRENTS_COMPRESSION");

		static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");
		static constexpr auto PROXY_PID_FILE = ADD_PREFIX("PROXY_PID_FILE");
		@@ -95,10 +93,6 @@ static constexpr auto WRITE_SIZE_THRESHOLD =
		ADD_PREFIX("WRITE_SIZE_CACHE_THRESHOLD");
		} // namespace cache

		static constexpr auto CREATE_CHECK_PARENTS = ADD_PREFIX("CREATE_CHECK_PARENTS");
		static constexpr auto SYMLINK_SUPPORT = ADD_PREFIX("SYMLINK_SUPPORT");
		static constexpr auto RENAME_SUPPORT = ADD_PREFIX("RENAME_SUPPORT");
		static constexpr auto USE_INLINE_DATA = ADD_PREFIX("USE_INLINE_DATA");
		static constexpr auto CREATE_WRITE_OPTIMIZATION =
		ADD_PREFIX("CREATE_WRITE_OPTIMIZATION");
		static constexpr auto READ_INLINE_PREFETCH = ADD_PREFIX("READ_INLINE_PREFETCH");
		@@ -107,6 +101,11 @@ static constexpr auto METADATA_BATCH = ADD_PREFIX("METADATA_BATCH");
		static constexpr auto METADATA_BATCH_THRESHOLD =
		ADD_PREFIX("METADATA_BATCH_THRESHOLD");
		static constexpr auto ASYNC_WRITE = ADD_PREFIX("ASYNC_WRITE");
		static constexpr auto FUSE_ENTRY_TIMEOUT = ADD_PREFIX("FUSE_ENTRY_TIMEOUT");
		static constexpr auto FUSE_ATTR_TIMEOUT = ADD_PREFIX("FUSE_ATTR_TIMEOUT");
		static constexpr auto FUSE_NEGATIVE_TIMEOUT =
		ADD_PREFIX("FUSE_NEGATIVE_TIMEOUT");
		static constexpr auto FUSE_WRITEBACK = ADD_PREFIX("FUSE_WRITEBACK");

		// Libfabric interface pinning (consumed by libfabric at HG_init() time)
		// OFI_INTERFACE is used with the GKFS_ prefix (e.g., LIBGKFS_OFI_INTERFACE)