MR2: Add integration tests and harness (62004d96) · Commits · hpc / gekkofs

tests/CMakeLists.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -42,6 +42,8 @@ add_custom_target(check

		# app tests
		add_subdirectory(apps)
		# integration tests
		add_subdirectory(integration)

		# integration tests
		add_subdirectory(integration)

tests/integration/CMakeLists.txt

+46 −0

Original line number	Diff line number	Diff line
		@@ -134,6 +134,27 @@ gkfs_add_python_test(
		SOURCE syscalls/
		)

		gkfs_add_python_test(
		NAME test_malleability
		PYTHON_VERSION 3.6
		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
		SOURCE malleability/
		)

		gkfs_add_python_test(
		NAME test_startup
		PYTHON_VERSION 3.6
		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
		SOURCE startup/
		)

		gkfs_add_python_test(
		NAME test_error_handling
		PYTHON_VERSION 3.6
		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
		SOURCE error_handling/
		)

		if (GKFS_RENAME_SUPPORT)
		gkfs_add_python_test(
		NAME test_rename
		@@ -239,6 +260,31 @@ if (GKFS_INSTALL_TESTS)
		PATTERN ".pytest_cache" EXCLUDE
		)
		endif ()

		install(DIRECTORY malleability
		DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
		FILES_MATCHING
		REGEX ".*\\.py"
		PATTERN "__pycache__" EXCLUDE
		PATTERN ".pytest_cache" EXCLUDE
		)

		install(DIRECTORY startup
		DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
		FILES_MATCHING
		REGEX ".*\\.py"
		PATTERN "__pycache__" EXCLUDE
		PATTERN ".pytest_cache" EXCLUDE
		)

		install(DIRECTORY error_handling
		DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
		FILES_MATCHING
		REGEX ".*\\.py"
		PATTERN "__pycache__" EXCLUDE
		PATTERN ".pytest_cache" EXCLUDE
		)

		endif ()

tests/integration/compatibility/test_compat.py

0 → 100644

+33 −0

Original line number	Diff line number	Diff line
		import pytest

		def test_compat_cp_mv(gkfs_daemon, gkfs_shell):
		"""
		Test cp and mv compatibility.
		"""
		cmd = gkfs_shell.script(
		f"""
		mkdir -p {gkfs_daemon.mountdir / 'compat'}
		echo "data" > {gkfs_daemon.mountdir / 'compat/file1'}
		cp {gkfs_daemon.mountdir / 'compat/file1'} {gkfs_daemon.mountdir / 'compat/file2'}
		mv {gkfs_daemon.mountdir / 'compat/file2'} {gkfs_daemon.mountdir / 'compat/file3'}

		diff {gkfs_daemon.mountdir / 'compat/file1'} {gkfs_daemon.mountdir / 'compat/file3'}
		""")
		if cmd.exit_code != 0:
		import sys
		sys.stderr.write(f"compat_cp_mv failed. stdout: {cmd.stdout.decode()} stderr: {cmd.stderr.decode()}")
		assert cmd.exit_code == 0

		def test_compat_grep(gkfs_daemon, gkfs_shell):
		cmd = gkfs_shell.script(
		f"""
		mkdir -p {gkfs_daemon.mountdir / 'grep_dir'}
		echo "hello world" > {gkfs_daemon.mountdir / 'grep_dir/f1'}
		echo "goodbye" > {gkfs_daemon.mountdir / 'grep_dir/f2'}

		grep "hello" {gkfs_daemon.mountdir / 'grep_dir/f1'}
		if [ $? -ne 0 ]; then exit 1; fi

		grep "world" {gkfs_daemon.mountdir / 'grep_dir'}/*
		""")
		assert cmd.exit_code == 0

tests/integration/compatibility/test_standard_tools.py

0 → 100644

+69 −0

Original line number	Diff line number	Diff line
		import pytest
		import hashlib

		@pytest.mark.parametrize("shell_fixture", ["gkfs_shell", "gkfs_shellLibc"])
		def test_tar_extract(gkfs_daemon, shell_fixture, test_workspace, request):
		"""
		Test tar extraction onto GekkoFS.
		"""
		gkfs_shell = request.getfixturevalue(shell_fixture)
		# Create a local tar file (not in GekkoFS)
		local_tar = test_workspace.twd / "payload.tar"
		cmd = gkfs_shell.script(
		f"""
		mkdir -p /tmp/payload_src/subdir
		echo "stuff" > /tmp/payload_src/file1
		echo "more" > /tmp/payload_src/subdir/file2
		tar -cf {local_tar} -C /tmp/payload_src .
		rm -rf /tmp/payload_src
		""", intercept_shell=False) # Run natively
		assert cmd.exit_code == 0

		# Extract into GekkoFS
		cmd = gkfs_shell.script(
		f"""
		mkdir -p {gkfs_daemon.mountdir / 'tar_target'}
		tar -xf {local_tar} -C {gkfs_daemon.mountdir / 'tar_target'}
		exit $?
		""")
		assert cmd.exit_code == 0

		# Verify content
		cmd = gkfs_shell.script(
		f"""
		cat {gkfs_daemon.mountdir / 'tar_target/file1'}
		cat {gkfs_daemon.mountdir / 'tar_target/subdir/file2'}
		""")
		assert "stuff" in cmd.stdout.decode()
		assert "more" in cmd.stdout.decode()

		@pytest.mark.parametrize("shell_fixture", ["gkfs_shell", "gkfs_shellLibc"])
		def test_rm_recursive(gkfs_daemon, shell_fixture, request):
		"""
		Test rm -rf directories.
		"""
		gkfs_shell = request.getfixturevalue(shell_fixture)
		cmd = gkfs_shell.script(
		f"""
		mkdir -p {gkfs_daemon.mountdir / 'delete_me/nested'}
		echo "val" > {gkfs_daemon.mountdir / 'delete_me/nested/f'}
		rm -rf {gkfs_daemon.mountdir / 'delete_me'}
		if [ -e {gkfs_daemon.mountdir / 'delete_me'} ]; then exit 1; fi
		exit 0
		""")
		assert cmd.exit_code == 0

		@pytest.mark.parametrize("shell_fixture", ["gkfs_shell", "gkfs_shellLibc"])
		def test_md5sum(gkfs_daemon, shell_fixture, request):
		"""
		Test reading files with checksum tools.
		"""
		gkfs_shell = request.getfixturevalue(shell_fixture)
		cmd = gkfs_shell.script(
		f"""
		echo "checksum_this" > {gkfs_daemon.mountdir / 'chk_file'}
		md5sum {gkfs_daemon.mountdir / 'chk_file'}
		""")
		assert cmd.exit_code == 0
		# md5sum of "checksum_this\n" is usually ...
		# We just check exit code here, ensuring read works.

tests/integration/concurrency/test_concurrency.py

0 → 100644

+75 −0

Original line number	Diff line number	Diff line
		import pytest
		from harness.logger import logger
		import time

		def test_concurrent_create(gkfs_daemon, gkfs_shell):
		"""
		Test concurrent file creation in the same directory.
		Using background processes via shell script.
		"""
		cmd = gkfs_shell.script(
		f"""
		GKFS_LOG=info mkdir -p {gkfs_daemon.mountdir / 'concurrent_dir'} > /tmp/mkdir_check.log 2>&1
		ls -ld {gkfs_daemon.mountdir / 'concurrent_dir'} >> /tmp/mkdir_check.log 2>&1

		# Start 10 background processes creating files via nested bash with env vars SET BEFORE EXEC
		# Use touch to ensure openat is used correctly
		GKFS_LOG=info GKFS_LOG_OUTPUT=/tmp/gkfs_client_conc.log bash -c "echo 'STARTING LOOP'; for i in \$(seq 1 10); do touch \\\"{gkfs_daemon.mountdir / 'concurrent_dir'}/file_\$i\\\" & done; echo 'WAITING'; wait" > /tmp/loop_output.log 2>&1


		# Wait for all background jobs
		wait

		# Verify count
		ls -1 "{gkfs_daemon.mountdir / 'concurrent_dir'}" \| wc -l
		""")

		if cmd.exit_code != 0:
		import sys
		sys.stderr.write(f"concurrent_create failed. stdout: {cmd.stdout.decode()} stderr: {cmd.stderr.decode()}")

		assert cmd.exit_code == 0
		assert int(cmd.stdout.decode().strip()) == 10

		def test_concurrent_write_shared_file(gkfs_daemon, gkfs_shell):
		"""
		Test concurrent writes to the SAME file (append).
		Note: GekkoFS might not support atomic append perfectly but shouldn't crash.
		"""
		cmd = gkfs_shell.script(
		f"""
		echo "" > {gkfs_daemon.mountdir / 'shared_file'}

		for i in $(seq 1 10); do
		echo "line_$i" >> "{gkfs_daemon.mountdir / 'shared_file'}" &
		done

		wait

		wc -l < "{gkfs_daemon.mountdir / 'shared_file'}"
		""")
		assert cmd.exit_code == 0
		# We expect 10 lines (plus initial empty line = 11? or just 10 if echo "" creates 1 line)
		# echo "" creates newline. echo "line" >> appends.
		# Total 11 lines.
		lines = int(cmd.stdout.decode().strip())
		assert lines == 11

		def test_concurrent_read(gkfs_daemon, gkfs_shell):
		"""
		Test concurrent reads from the same file.
		"""
		cmd = gkfs_shell.script(
		f"""
		# Create 1MB file
		head -c 1048576 /dev/urandom > {gkfs_daemon.mountdir / 'read_file'}

		# 5 readers
		for i in $(seq 1 5); do
		cat {gkfs_daemon.mountdir / 'read_file'} > /dev/null &
		done

		wait
		exit 0
		""")
		assert cmd.exit_code == 0