Skip to content
Commits on Source (51)
Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 1c4528b4
......@@ -5,22 +5,25 @@ stages:
- test
variables:
DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src"
DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install"
DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit"
BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build"
INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install"
TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build"
LOG_PATH: "${CI_PROJECT_DIR}/logs"
LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64"
DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src"
DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install"
DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit"
BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build"
INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install"
INTEGRATION_TESTS_BIN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration"
INTEGRATION_TESTS_RUN_PATH: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/run"
TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build"
PYTEST: "${CI_PROJECT_DIR}/gkfs/install/share/gkfs/tests/integration/pytest-venv/bin/py.test"
LOG_PATH: "${CI_PROJECT_DIR}/logs"
LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64"
# Configuration variables
GKFS_LOG_LEVEL: "100"
GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log"
LIBGKFS_LOG: "all"
LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
GIT_SUBMODULE_STRATEGY: recursive
GKFS_LOG_LEVEL: "100"
GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log"
LIBGKFS_LOG: "all"
LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
GIT_SUBMODULE_STRATEGY: recursive
image: gekkofs/gekkofs:build_env
image: gekkofs/gekkofs:build_env-0.8.0
compile dependencies:
stage: build deps
......@@ -51,6 +54,8 @@ compile GekkoFS:
-Wdev
-Wdeprecate
-DCMAKE_BUILD_TYPE=Debug
-DGKFS_BUILD_TESTS:BOOL=ON
-DGKFS_INSTALL_TESTS:BOOL=ON
-DRPC_PROTOCOL="ofi+sockets"
-DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH}
-DCMAKE_INSTALL_PREFIX=${INSTALL_PATH}
......@@ -72,6 +77,17 @@ compile tests:
paths:
- ${TESTS_BUILD_PATH}
integration tests:
stage: test
script:
- mkdir -p ${INTEGRATION_TESTS_RUN_PATH}
- cd ${INTEGRATION_TESTS_BIN_PATH}
- TMPDIR=${INTEGRATION_TESTS_RUN_PATH} unbuffer ${PYTEST} -v | tee ${INTEGRATION_TESTS_RUN_PATH}/session.log
artifacts:
when: on_failure
paths:
- "${INTEGRATION_TESTS_RUN_PATH}"
test wr:
stage: test
script:
......
CMake/GkfsPythonTesting.cmake 0 → 100644
View file @ 1c4528b4
include(CMakeParseArguments)
function(gkfs_enable_python_testing)
# Parse arguments
set(MULTI BINARY_DIRECTORIES LIBRARY_PREFIX_DIRECTORIES)
cmake_parse_arguments(PYTEST "${OPTION}" "${SINGLE}" "${MULTI}" ${ARGN})
if(PYTEST_UNPARSED_ARGUMENTS)
message(WARNING "Unparsed arguments in gkfs_enable_python_testing: This often indicates typos!")
endif()
if(PYTEST_BINARY_DIRECTORIES)
set(GKFS_PYTEST_BINARY_DIRECTORIES ${PYTEST_BINARY_DIRECTORIES} PARENT_SCOPE)
endif()
if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)
set(GKFS_PYTEST_LIBRARY_PREFIX_DIRECTORIES ${PYTEST_LIBRARY_PREFIX_DIRECTORIES} PARENT_SCOPE)
endif()
set(PYTEST_BINDIR_ARGS, "")
if(PYTEST_BINARY_DIRECTORIES)
foreach(dir IN LISTS PYTEST_BINARY_DIRECTORIES)
list(APPEND PYTEST_BINDIR_ARGS "--bin-dir=${dir}")
endforeach()
endif()
set(PYTEST_LIBDIR_ARGS, "")
if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)
foreach(dir IN LISTS PYTEST_LIBRARY_PREFIX_DIRECTORIES)
if(NOT IS_ABSOLUTE ${dir})
set(dir ${CMAKE_BINARY_DIR}/${dir})
endif()
file(TO_CMAKE_PATH "${dir}/lib" libdir)
file(TO_CMAKE_PATH "${dir}/lib64" lib64dir)
if(EXISTS ${libdir})
list(APPEND PYTEST_LIBDIR_ARGS "--lib-dir=${libdir}")
endif()
if(EXISTS ${lib64dir})
list(APPEND PYTEST_LIBDIR_ARGS "--lib-dir=${lib64dir}")
endif()
endforeach()
endif()
# convert path lists to space separated arguments
string(REPLACE ";" " " PYTEST_BINDIR_ARGS "${PYTEST_BINDIR_ARGS}")
string(REPLACE ";" " " PYTEST_BINDIR_ARGS "${PYTEST_BINDIR_ARGS}")
configure_file(pytest.ini.in pytest.ini @ONLY)
configure_file(conftest.py.in conftest.py @ONLY)
configure_file(harness/cli.py harness/cli.py COPYONLY)
if(GKFS_INSTALL_TESTS)
configure_file(pytest.install.ini.in pytest.install.ini @ONLY)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/pytest.install.ini
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
RENAME pytest.ini
)
install(FILES conftest.py
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
)
if(NOT PYTEST_VIRTUALENV)
set(PYTEST_VIRTUALENV ${CMAKE_INSTALL_FULL_DATAROOTDIR}/gkfs/tests/integration/pytest-venv)
endif()
# Python's virtual environments are not relocatable, we need to
# recreate the virtualenv at the appropriate install location
# find an appropriate python interpreter
find_package(Python3
3.6
REQUIRED
COMPONENTS Interpreter)
if(NOT Python3_FOUND)
message(FATAL_ERROR "Unable to find Python 3")
endif()
install(
CODE "message(\"Install pytest virtual environment...\")"
CODE "message(\"-- Create virtual environment: ${PYTEST_VIRTUALENV}\")"
CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -m venv ${PYTEST_VIRTUALENV})"
CODE "message(\"-- Installing packages...\")"
CODE "execute_process(COMMAND ${PYTEST_VIRTUALENV}/bin/pip install --upgrade pip -v)"
CODE "execute_process(COMMAND ${PYTEST_VIRTUALENV}/bin/pip install -r ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt --upgrade -v)"
)
endif()
# enable testing
set(GKFS_PYTHON_TESTING_ENABLED ON PARENT_SCOPE)
endfunction()
function(gkfs_add_python_test)
# ignore call if testing is not enabled
if(NOT CMAKE_TESTING_ENABLED OR NOT GKFS_PYTHON_TESTING_ENABLED)
return()
endif()
# Parse arguments
set(OPTION)
set(SINGLE NAME PYTHON_VERSION WORKING_DIRECTORY VIRTUALENV)
set(MULTI SOURCE BINARY_DIRECTORIES LIBRARY_PREFIX_DIRECTORIES)
cmake_parse_arguments(PYTEST "${OPTION}" "${SINGLE}" "${MULTI}" ${ARGN})
if(PYTEST_UNPARSED_ARGUMENTS)
message(WARNING "Unparsed arguments in gkfs_add_python_test: This often indicates typos!")
endif()
if(NOT PYTEST_NAME)
message(FATAL_ERROR "gkfs_add_python_test requires a NAME argument")
endif()
# set default values for arguments not provided
if(NOT PYTEST_PYTHON_VERSION)
set(PYTEST_PYTHON_VERSION 3.0)
endif()
if(NOT PYTEST_WORKING_DIRECTORY)
set(PYTEST_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
endif()
if(NOT PYTEST_VIRTUALENV)
set(PYTEST_VIRTUALENV ${CMAKE_CURRENT_BINARY_DIR}/pytest-venv)
endif()
# if the test doesn't provide a list of binary or library prefix
# directories, use the one set on gkfs_enable_python_testing()
if(NOT PYTEST_BINARY_DIRECTORIES)
set(PYTEST_BINARY_DIRECTORIES ${GKFS_PYTEST_BINARY_DIRECTORIES})
endif()
if(NOT PYTEST_LIBRARY_PREFIX_DIRECTORIES)
set(PYTEST_LIBRARY_PREFIX_DIRECTORIES ${GKFS_PYTEST_LIBRARY_PREFIX_DIRECTORIES})
endif()
set(PYTEST_COMMAND_ARGS, "")
if(PYTEST_BINARY_DIRECTORIES)
foreach(dir IN LISTS PYTEST_BINARY_DIRECTORIES)
list(APPEND PYTEST_COMMAND_ARGS "--bin-dir=${dir}")
endforeach()
endif()
if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)
foreach(dir IN LISTS PYTEST_LIBRARY_PREFIX_DIRECTORIES)
if(NOT IS_ABSOLUTE ${dir})
set(dir ${CMAKE_BINARY_DIR}/${dir})
endif()
file(TO_CMAKE_PATH "${dir}/lib" libdir)
file(TO_CMAKE_PATH "${dir}/lib64" lib64dir)
if(EXISTS "${dir}/lib")
list(APPEND PYTEST_COMMAND_ARGS "--lib-dir=${libdir}")
endif()
if(EXISTS "${dir}/lib64")
list(APPEND PYTEST_COMMAND_ARGS "--lib-dir=${lib64dir}")
endif()
endforeach()
endif()
# Extend the given virtualenv to be a full path.
if(NOT IS_ABSOLUTE ${PYTEST_VIRTUALENV})
set(PYTEST_VIRTUALENV ${CMAKE_BINARY_DIR}/${PYTEST_VIRTUALENV})
endif()
# find an appropriate python interpreter
find_package(Python3
${PYTEST_PYTHON_VERSION}
REQUIRED
COMPONENTS Interpreter)
set(PYTEST_VIRTUALENV_PIP ${PYTEST_VIRTUALENV}/bin/pip)
set(PYTEST_VIRTUALENV_INTERPRETER ${PYTEST_VIRTUALENV}/bin/python)
# create a virtual environment to run the test
configure_file(requirements.txt.in requirements.txt @ONLY)
add_custom_command(
OUTPUT ${PYTEST_VIRTUALENV}
COMMENT "Creating virtual environment ${PYTEST_VIRTUALENV}"
COMMAND Python3::Interpreter -m venv "${PYTEST_VIRTUALENV}"
COMMAND ${PYTEST_VIRTUALENV_PIP} install --upgrade pip -q
COMMAND ${PYTEST_VIRTUALENV_PIP} install -r requirements.txt --upgrade -q
)
if(NOT TARGET venv)
# ensure that the virtual environment is created by the build process
# (this is required because we can't add dependencies between
# "test targets" and "normal targets"
add_custom_target(venv
ALL
DEPENDS ${PYTEST_VIRTUALENV}
DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt)
endif()
add_test(NAME ${PYTEST_NAME}
COMMAND ${PYTEST_VIRTUALENV_INTERPRETER}
-m pytest -v -s
${PYTEST_COMMAND_ARGS}
${PYTEST_SOURCE}
WORKING_DIRECTORY ${PYTEST_WORKING_DIRECTORY})
# instruct Python to not create __pycache__ directories,
# otherwise they will pollute ${PYTEST_WORKING_DIRECTORY} which
# is typically ${PROJECT_SOURCE_DIR}
set_tests_properties(${PYTEST_NAME} PROPERTIES
ENVIRONMENT PYTHONDONTWRITEBYTECODE=1)
endfunction()
CMakeLists.txt
View file @ 1c4528b4
......@@ -5,6 +5,7 @@ project(
VERSION 0.7.0
)
enable_testing()
if(NOT CMAKE_COMPILER_IS_GNUCC)
message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
......@@ -209,3 +210,17 @@ add_subdirectory(src/global)
add_subdirectory(src/daemon)
# Client library
add_subdirectory(src/client)
option(GKFS_BUILD_TESTS "Build GekkoFS self tests" OFF)
include(CMakeDependentOption)
cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
if(GKFS_BUILD_TESTS)
message(STATUS "[gekkofs] Preparing tests...")
set(GKFS_TESTS_INTERFACE "lo" CACHE STRING "Network interface to use when running tests (default: lo)")
message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
add_subdirectory(tests)
else()
unset(GKFS_TESTS_INTERFACE CACHE)
endif()
README.md
View file @ 1c4528b4
......@@ -115,6 +115,21 @@ make
make install
```
In order to build self-tests, the *optional* GKFS_BUILD_TESTS CMake option needs
to be enabled when building. Once that is done, tests can be run by running
`make test` in the `build` directory:
```bash
mkdir build && cd build
cmake -DGKFS_BUILD_TESTS=ON -DCMAKE_BUILD_TYPE=Release -DRPC_PROTOCOL='ofi+sockets' ..
make
make test
make install
```
**IMPORTANT:** Please note that the testing framework requires Python 3.6 as an
additional dependency in order to run.
## Run GekkoFS
First on each node a daemon has to be started. This can be done in two ways using the `gkfs_daemon` binary directly or
......
docker/debian_build_env.docker
View file @ 1c4528b4
......@@ -35,14 +35,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libboost-program-options-dev \
valgrind \
uuid-dev \
python3 \
python3-dev \
python3-venv \
expect \
# Clean apt cache to reduce image layer size
&& rm -rf /var/lib/apt/lists/*
# Download dependencies source
COPY scripts/dl_dep.sh $SCRIPTS_PATH/
RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all
# Compile dependencies
COPY scripts/compile_dep.sh $SCRIPTS_PATH/
COPY scripts/patches $SCRIPTS_PATH/patches
RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH
## COPY scripts/dl_dep.sh $SCRIPTS_PATH/
## COPY scripts/compile_dep.sh $SCRIPTS_PATH/
## COPY scripts/patches $SCRIPTS_PATH/patches
##
## # Download dependencies source
## RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all
##
## # Compile dependencies
## RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH
test/README.md 0 → 100644
View file @ 1c4528b4
# README
This directory contains old/deprecated GekkoFS tests. It is kept here until all
tests have been migrated to the new testing framework.
***
**IMPORTANT:**
Some of these tests are still active in the CI scripts.
***
tests/CMakeLists.txt 0 → 100644
View file @ 1c4528b4
include(GkfsPythonTesting)
add_custom_target(check
COMMAND ${CMAKE_CTEST_COMMAND}
--force-new-ctest-process
--verbose
--output-on-failure
)
# integration tests
add_subdirectory(integration)
# unit tests
add_subdirectory(unit)
tests/README.md 0 → 100644
View file @ 1c4528b4
# README
This directory contains GekkoFS unit, functional, and integration tests. Please
refer to the wiki page about [testing GekkoFS](../-/wikis/Testing) for more
information.
tests/integration/CMakeLists.txt 0 → 100644
View file @ 1c4528b4
# ensure helper programs in the testing harness get built
add_subdirectory(harness)
gkfs_enable_python_testing(
BINARY_DIRECTORIES ${CMAKE_BINARY_DIR}/src/daemon/
${CMAKE_BINARY_DIR}/src/client/
${CMAKE_BINARY_DIR}/tests/integration/harness/
LIBRARY_PREFIX_DIRECTORIES ${CMAKE_PREFIX_PATH}
)
# define CTest tests for functional test groups
gkfs_add_python_test(
NAME test_directories
PYTHON_VERSION 3.6
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
SOURCE directories/test_directories.py
)
gkfs_add_python_test(
NAME test_shell
PYTHON_VERSION 3.6
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
SOURCE shell/
)
if(GKFS_INSTALL_TESTS)
install(DIRECTORY harness
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
FILES_MATCHING
REGEX ".*\\.py"
PATTERN "__pycache__" EXCLUDE
PATTERN ".pytest_cache" EXCLUDE
PATTERN "gkfs.io" EXCLUDE
)
install(DIRECTORY directories
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
FILES_MATCHING
REGEX ".*\\.py"
PATTERN "__pycache__" EXCLUDE
PATTERN ".pytest_cache" EXCLUDE
)
install(DIRECTORY shell
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
FILES_MATCHING
REGEX ".*\\.py"
PATTERN "__pycache__" EXCLUDE
PATTERN ".pytest_cache" EXCLUDE
)
endif()
tests/integration/conftest.py 0 → 100644
View file @ 1c4528b4
################################################################################
# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
import pytest
import logging
from collections import namedtuple
from _pytest.logging import caplog as _caplog
from pathlib import Path
from harness.logger import logger, initialize_logging, finalize_logging
from harness.cli import add_cli_options, set_default_log_formatter
from harness.workspace import Workspace, FileCreator
from harness.gkfs import Daemon, Client, ShellClient
from harness.reporter import report_test_status, report_test_headline, report_assertion_pass
def pytest_configure(config):
"""
Some configurations for our particular usage of pytest
"""
set_default_log_formatter(config, "%(message)s")
def pytest_assertion_pass(item, lineno, orig, expl):
location = namedtuple(
'Location', ['path', 'module', 'function', 'lineno'])(
str(item.parent.fspath), item.parent.name, item.name, lineno)
report_assertion_pass(logger, location, orig, expl)
def pytest_addoption(parser):
"""
Adds extra options from the GKFS harness to the py.test CLI.
"""
add_cli_options(parser)
@pytest.fixture(autouse=True)
def caplog(test_workspace, request, _caplog):
# we don't need to see the logs from sh
_caplog.set_level(logging.CRITICAL, 'sh.command')
_caplog.set_level(logging.CRITICAL, 'sh.command.process')
_caplog.set_level(logging.CRITICAL, 'sh.command.process.streamreader')
_caplog.set_level(logging.CRITICAL, 'sh.stream_bufferer')
_caplog.set_level(logging.CRITICAL, 'sh.streamreader')
test_log_path = test_workspace.logdir / (request.node.name + ".log")
h = initialize_logging(logger, test_log_path)
report_test_headline(logger, request.node.nodeid, request.config, test_workspace)
yield _caplog
finalize_logging(logger, h)
def pytest_runtest_logreport(report):
"""
Pytest hook called after a test phase (setup, call, teardownd)
has completed.
"""
report_test_status(logger, report)
@pytest.fixture
def test_workspace(tmp_path, request):
"""
Initializes a test workspace by creating a temporary directory for it.
"""
yield Workspace(tmp_path,
request.config.getoption('--bin-dir'),
request.config.getoption('--lib-dir'))
@pytest.fixture
def gkfs_daemon(test_workspace, request):
"""
Initializes a local gekkofs daemon
"""
interface = request.config.getoption('--interface')
daemon = Daemon(interface, test_workspace)
yield daemon.run()
daemon.shutdown()
@pytest.fixture
def gkfs_client(test_workspace):
"""
Sets up a gekkofs client environment so that
operations (system calls, library calls, ...) can
be requested from a co-running daemon.
"""
return Client(test_workspace)
@pytest.fixture
def gkfs_shell(test_workspace):
"""
Sets up a gekkofs environment so that shell commands
(stat, ls, mkdir, etc.) can be issued to a co-running daemon.
"""
return ShellClient(test_workspace)
@pytest.fixture
def file_factory(test_workspace):
"""
Returns a factory that can create custom input files
in the test workspace.
"""
return FileCreator(test_workspace)
tests/integration/conftest.py.in 0 → 100644
View file @ 1c4528b4
################################################################################
# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
from harness.cli import add_cli_options, set_default_log_formatter
from pathlib import Path
def pytest_configure(config):
"""
Some configurations for our particularusage of pytest
"""
set_default_log_formatter(config, "%(message)s")
def pytest_addoption(parser):
"""
Adds extra options from the GKFS harness to the py.test CLI.
"""
add_cli_options(parser)
tests/integration/directories/README.md 0 → 100644
View file @ 1c4528b4
# README
This directory contains functional tests for any directory-related
functionalities in GekkoFS.
tests/integration/directories/test_directories.py 0 → 100644
View file @ 1c4528b4
################################################################################
# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
import harness
from pathlib import Path
import errno
import stat
import os
import ctypes
import sh
import sys
import pytest
from harness.logger import logger
nonexisting = "nonexisting"
#@pytest.mark.xfail(reason="invalid errno returned on success")
def test_mkdir(gkfs_daemon, gkfs_client):
"""Create a new directory in the FS's root"""
topdir = gkfs_daemon.mountdir / "top"
longer = Path(topdir.parent, topdir.name + "_plus")
dir_a = topdir / "dir_a"
dir_b = topdir / "dir_b"
file_a = topdir / "file_a"
subdir_a = dir_a / "subdir_a"
# create topdir
ret = gkfs_client.mkdir(
topdir,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval == 0
assert ret.errno == 115 #FIXME: Should be 0!
# test stat on existing dir
ret = gkfs_client.stat(topdir)
assert ret.retval == 0
assert ret.errno == 115 #FIXME: Should be 0!
assert stat.S_ISDIR(ret.statbuf.st_mode)
# open topdir
ret = gkfs_client.open(topdir, os.O_DIRECTORY)
assert ret.retval != -1
assert ret.errno == 115 #FIXME: Should be 0!
# read and write should be impossible on directories
ret = gkfs_client.read(topdir, 1)
assert ret.buf is None
assert ret.retval == -1
assert ret.errno == errno.EISDIR
# buf = bytes('42', sys.stdout.encoding)
# print(buf.hex())
buf = b'42'
ret = gkfs_client.write(topdir, buf, 1)
assert ret.retval == -1
assert ret.errno == errno.EISDIR
# read top directory that is empty
ret = gkfs_client.opendir(topdir)
assert ret.dirp is not None
assert ret.errno == 115 #FIXME: Should be 0!
ret = gkfs_client.readdir(topdir)
# XXX: This might change in the future if we add '.' and '..'
assert len(ret.dirents) == 0
assert ret.errno == 115 #FIXME: Should be 0!
# close directory
# TODO: disabled for now because we have no way to keep DIR* alive
# between gkfs.io executions
# ret = gkfs_client.opendir(XXX)
# assert ret.errno == 115 #FIXME: Should be 0!
# populate top directory
for d in [dir_a, dir_b]:
ret = gkfs_client.mkdir(
d,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval == 0
assert ret.errno == 115 #FIXME: Should be 0!
ret = gkfs_client.open(file_a,
os.O_CREAT,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval != -1
assert ret.errno == 115 #FIXME: Should be 0!
ret = gkfs_client.readdir(gkfs_daemon.mountdir)
# XXX: This might change in the future if we add '.' and '..'
assert len(ret.dirents) == 1
assert ret.dirents[0].d_name == 'top'
assert ret.dirents[0].d_type == 4 # DT_DIR
assert ret.errno == 115 #FIXME: Should be 0!
expected = [
( dir_a.name, 4 ), # DT_DIR
( dir_b.name, 4 ),
( file_a.name, 8 ) # DT_REG
]
ret = gkfs_client.readdir(topdir)
assert len(ret.dirents) == len(expected)
assert ret.errno == 115 #FIXME: Should be 0!
for d,e in zip(ret.dirents, expected):
assert d.d_name == e[0]
assert d.d_type == e[1]
# remove file using rmdir should produce an error
ret = gkfs_client.rmdir(file_a)
assert ret.retval == -1
assert ret.errno == errno.ENOTDIR
# create a directory with the same prefix as topdir but longer name
ret = gkfs_client.mkdir(
longer,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval == 0
assert ret.errno == 115 #FIXME: Should be 0!
expected = [
( topdir.name, 4 ), # DT_DIR
( longer.name, 4 ), # DT_DIR
]
ret = gkfs_client.readdir(gkfs_daemon.mountdir)
assert len(ret.dirents) == len(expected)
assert ret.errno == 115 #FIXME: Should be 0!
for d,e in zip(ret.dirents, expected):
assert d.d_name == e[0]
assert d.d_type == e[1]
# create 2nd level subdir and check it's not included in readdir()
ret = gkfs_client.mkdir(
subdir_a,
stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval == 0
assert ret.errno == 115 #FIXME: Should be 0!
expected = [
( topdir.name, 4 ), # DT_DIR
( longer.name, 4 ), # DT_DIR
]
ret = gkfs_client.readdir(gkfs_daemon.mountdir)
assert len(ret.dirents) == len(expected)
assert ret.errno == 115 #FIXME: Should be 0!
for d,e in zip(ret.dirents, expected):
assert d.d_name == e[0]
assert d.d_type == e[1]
expected = [
( subdir_a.name, 4 ), # DT_DIR
]
ret = gkfs_client.readdir(dir_a)
assert len(ret.dirents) == len(expected)
assert ret.errno == 115 #FIXME: Should be 0!
for d,e in zip(ret.dirents, expected):
assert d.d_name == e[0]
assert d.d_type == e[1]
return
@pytest.mark.skip(reason="invalid errno returned on success")
@pytest.mark.parametrize("directory_path",
[ nonexisting ])
def test_opendir(gkfs_daemon, gkfs_client, directory_path):
ret = gkfs_client.opendir(gkfs_daemon.mountdir / directory_path)
assert ret.dirp is None
assert ret.errno == errno.ENOENT
# def test_stat(gkfs_daemon):
# pass
#
# def test_rmdir(gkfs_daemon):
# pass
#
# def test_closedir(gkfs_daemon):
# pass
tests/integration/harness/CMakeLists.txt 0 → 100644
View file @ 1c4528b4
cmake_minimum_required(VERSION 3.11)
project(gkfs.io
VERSION 0.1
LANGUAGES CXX
)
add_executable(gkfs.io
gkfs.io/main.cpp
gkfs.io/commands.hpp
gkfs.io/mkdir.cpp
gkfs.io/open.cpp
gkfs.io/opendir.cpp
gkfs.io/read.cpp
gkfs.io/readdir.cpp
gkfs.io/reflection.hpp
gkfs.io/rmdir.cpp
gkfs.io/serialize.hpp
gkfs.io/stat.cpp
gkfs.io/write.cpp
)
include(FetchContent)
set(FETCHCONTENT_QUIET OFF)
FetchContent_Declare(cli11
GIT_REPOSITORY https://github.com/CLIUtils/CLI11.git
GIT_TAG dd0d8e4fe729e5b1110232c7a5c9566dad884686 # v1.9.0
GIT_SHALLOW ON
GIT_PROGRESS ON
)
FetchContent_Declare(nlohmann_json
GIT_REPOSITORY https://github.com/nlohmann/json
GIT_TAG e7b3b40b5a95bc74b9a7f662830a27c49ffc01b4 # v3.7.3
GIT_SHALLOW ON
GIT_PROGRESS ON
)
FetchContent_GetProperties(cli11)
if(NOT cli11_POPULATED)
FetchContent_Populate(cli11)
message(STATUS "[gkfs.io] CLI11 source dir: ${cli11_SOURCE_DIR}")
message(STATUS "[gkfs.io] CLI11 binary dir: ${cli11_BINARY_DIR}")
add_subdirectory(${cli11_SOURCE_DIR} ${cli11_BINARY_DIR})
endif()
FetchContent_GetProperties(nlohmann_json)
if(NOT nlohmann_json_POPULATED)
FetchContent_Populate(nlohmann_json)
message(STATUS "[gkfs.io] Nlohmann JSON source dir: ${nlohmann_json_SOURCE_DIR}")
message(STATUS "[gkfs.io] Nlohmann JSON binary dir: ${nlohmann_json_BINARY_DIR}")
# we don't really care so much about a third party library's tests to be
# run from our own project's code
set(JSON_BuildTests OFF CACHE INTERNAL "")
# we also don't need to install it when our main project gets installed
set(JSON_Install OFF CACHE INTERNAL "")
add_subdirectory(${nlohmann_json_SOURCE_DIR} ${nlohmann_json_BINARY_DIR})
endif()
target_include_directories(gkfs.io PRIVATE
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/gkfs.io>
)
target_link_libraries(gkfs.io
CLI11::CLI11
nlohmann_json::nlohmann_json
fmt::fmt
)
if(GKFS_INSTALL_TESTS)
install(TARGETS gkfs.io
DESTINATION ${CMAKE_INSTALL_BINDIR}
)
endif()
tests/integration/harness/README.md 0 → 100644
View file @ 1c4528b4
# README
This directory contains the code for the testing harness used for running
functional and integration tests. Please refer to the wiki page about [testing
GekkoFS](../-/wikis/Testing) for more information.
tests/integration/harness/cli.py 0 → 100644
View file @ 1c4528b4
################################################################################
# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
import _pytest
import logging
from pathlib import Path
### This code is meant to be included automatically by CMake in the build
### directory's top-level conftest.py as well as the source directory's
### conftest.py, so that tests can be correctly run from both directories
def add_cli_options(parser):
"""
Adds extra options to the py.test CLI so that we can provide
search directories for libraries and helper programs.
"""
try:
parser.addoption(
'--interface',
action='store',
type=str,
default='lo',
help="network interface used for communications (default: 'lo')."
)
parser.addoption(
"--bin-dir",
action='append',
default=[Path.cwd()],
help="directory that should be considered when searching "
"for programs (multi-allowed)."
)
parser.addoption(
"--lib-dir",
action='append',
default=[Path.cwd()],
help="directory that should be considered when searching "
"for libraries (multi-allowed)."
)
except ValueError:
# if the CLI args have already been added, we have been called both
# from the build directory's conftest.py and from the source
# directory's conftest.py through automatic finding, ignore the error
pass
def set_default_log_formatter(config, fmt):
plugin_class = config.pluginmanager.get_plugin('logging').LoggingPlugin
if not isinstance(plugin_class, LoggingPlugin):
config.pluginmanager.get_plugin('logging').LoggingPlugin = LoggingPlugin
class LoggingPlugin(_pytest.logging.LoggingPlugin):
"""
Replacement logging plugin that rewrites py.test default log formatter
"""
def _create_formatter(self, log_format,
log_date_format, auto_indent) -> logging.Formatter:
"""
Patch pytest default logger to always return our formatter
Returns:
logging.Formatter: Our formatter
"""
# since we use loguru for formatting, we just want the message
return logging.Formatter("%(message)s")
tests/integration/harness/cmd.py 0 → 100644
View file @ 1c4528b4
################################################################################
# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain #
# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
# #
# This software was partially supported by the #
# ADA-FS project under the SPPEXA project funded by the DFG. #
# #
# SPDX-License-Identifier: MIT #
################################################################################
from collections import namedtuple
class Md5sumOutputSchema:
"""
Schema to deserialize the results of a md5sum command:
$ md5sum foobar
7f45c62700402ce5f9abe5b8d70d2844 foobar
"""
_field_names = [ 'digest', 'filename' ]
def loads(self, input):
values = input.split()
return namedtuple('md5sumOutput', self._field_names)(*values)
class StatOutputSchema:
"""
Schema to deserialize the results of a stat --terse command:
$ stat --terse foobar
foobar 913 8 81b4 1000 1000 10308 7343758 1 0 0 1583160824 1583160634 1583160634 0 4096
Output for the command follows the format below:
%n %s %b %f %u %g %D %i %h %t %T %X %Y %Z %W %o %C
%n: file name
%s: total size, in bytes
%b: number of blocks
%f: raw mode in hex
%u: owner UID
%g: owner GID
%D: device numer in hex
%i: inode number
%h: number of hard links
%t: major device in hex
%T: minor device in hex
%X: time of last access, seconds since Epoch
%Y: time of last data modification, seconds since Epoch
%Z: time of last status change, seconds since Epoch
%W: time of file birth, seconds since Epoch; 0 if unknown
%o: optimal I/O transfer size hint
"""
_field_names = [
'filename', 'size', 'blocks', 'raw_mode', 'uid', 'gid', 'device',
'inode', 'hard_links', 'major', 'minor', 'last_access',
'last_modification', 'last_status_change', 'creation',
'transfer_size' ]
_field_types = [
str, int, int, str, int, int, str, int, int, str, str, int, int, int,
int, int, str ]
def loads(self, input):
values = [ t(s) for t,s in zip(self._field_types, input.split()) ]
return namedtuple('statOutput', self._field_names)(*values)
class CommandParser:
"""
A helper parser to transform the output of some shell commands to native
Python objects.
"""
OutputSchemas = {
'md5sum' : Md5sumOutputSchema(),
'stat' : StatOutputSchema(),
}
def parse(self, command, output):
if command not in self.OutputSchemas:
raise NotImplementedError(
f"Output parser for '{command}' not implemented")
return self.OutputSchemas[command].loads(output)
tests/integration/harness/gkfs.io/binary_buffer.hpp 0 → 100644
View file @ 1c4528b4
/*
Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef IO_BINARY_BUFFER_HPP
#define IO_BINARY_BUFFER_HPP
#include <nlohmann/json.hpp>
#include <fmt/format.h>
namespace io {
struct buffer {
buffer(::size_t size = 0) :
m_data(size) { }
buffer(nullptr_t p) { }
buffer(const std::string& s) {
m_data.clear();
std::copy(s.begin(), s.end(), std::back_inserter(m_data));
}
bool
operator==(nullptr_t) const {
return m_data.size() == 0;
}
bool
operator!=(nullptr_t) const {
return m_data.size() != 0;
}
auto
data() {
return m_data.data();
}
auto
storage() const {
return m_data;
}
std::size_t
size() const {
return m_data.size();
}
std::vector<uint8_t> m_data;
};
inline void
to_json(nlohmann::json& record,
const buffer& out) {
if(out == nullptr) {
record = nullptr;
}
else {
// record = fmt::format("x{:2x}", fmt::join(out, "x"));
record = out.storage();
}
}
} // namespace io
#endif // IO_BINARY_BUFFER_HPP
tests/integration/harness/gkfs.io/command.hpp 0 → 100644
View file @ 1c4528b4
/*
Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef GKFS_IO_COMMAND_HPP
#define GKFS_IO_COMMAND_HPP
struct command {
std::string name;
std::string alt_name;
};
#endif // GKFS_IO_COMMAND_HPP
tests/integration/harness/gkfs.io/commands.hpp 0 → 100644
View file @ 1c4528b4
/*
Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
SPDX-License-Identifier: MIT
*/
#ifndef IO_COMMANDS_HPP
#define IO_COMMANDS_HPP
// forward declare CLI::App
namespace CLI { struct App; }
void
mkdir_init(CLI::App& app);
void
open_init(CLI::App& app);
void
opendir_init(CLI::App& app);
void
read_init(CLI::App& app);
void
readdir_init(CLI::App& app);
void
rmdir_init(CLI::App& app);
void
stat_init(CLI::App& app);
void
write_init(CLI::App& app);
#endif // IO_COMMANDS_HPP