diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65a9707051d771fefdb04feeab32c22a599a70dd..00a5d74b471ab41f67c7c1baf7d558b969b24ab6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,9 @@ stages: - lint - build - test + - docs - report + - deploy variables: SCRIPTS_DIR: "${CI_PROJECT_DIR}/scripts" @@ -233,6 +235,42 @@ gkfs:unit: junit: ${BUILD_PATH}/tests/unit/report.xml +################################################################################ +## Generation of documentation +################################################################################ +documentation: + stage: docs + image: gekkofs/docs:0.9.0 + needs: [] + rules: + # we only build the documentation automatically if we are on the + # `master` branch, but since we also would like to test the documentation + # against our CI, we allow developers to also build it manually + - if: '$CI_MERGE_REQUEST_EVENT_TYPE == "detached"' + when: never + - if: '$CI_MERGE_REQUEST_ID != ""' + when: manual + allow_failure: true + - if: '$CI_COMMIT_REF_SLUG == "master"' + when: always + + script: + - mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH} + - cmake + -Wdev + -Wdeprecate + -DCMAKE_BUILD_TYPE=Debug + -DGKFS_BUILD_DOCUMENTATION:BOOL=ON + -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} + -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} + ${CI_PROJECT_DIR} + - make docs + artifacts: + paths: + - ${BUILD_PATH}/docs + expire_in: 2 weeks + + ################################################################################ ## Generation of code coverage reports ################################################################################ @@ -254,3 +292,27 @@ coverage: paths: - ${BUILD_PATH}/.coverage expire_in: 2 weeks + + +################################################################################ +## Deployment of documentation and reports +################################################################################ +# +## for the deploy stage to work as expected, we need to run rsync with the +## appropriate credentials provided by sysadmins. For that, the specific values +## for DEPLOY_KEY_FILE, DEPLOY_USERNAME, DEPLOY_GROUP, DEPLOY_SERVER and +## DEPLOY_PATH must be defined as protected variables. +deploy: + image: bscstorage/deployer + stage: deploy + needs: [ 'documentation' ] + only: + - master + script: + - chmod 400 ${DEPLOY_KEY_FILE} + - rsync -e "ssh -i ${DEPLOY_KEY_FILE}" + -avzr + --delete + --chown=${DEPLOY_USERNAME}:${DEPLOY_GROUP} + ${BUILD_PATH}/docs/sphinx/sphinx_docs/ + ${DEPLOY_USERNAME}@${DEPLOY_SERVER}:${DEPLOY_PATH} diff --git a/CMake/FindSphinx.cmake b/CMake/FindSphinx.cmake new file mode 100644 index 0000000000000000000000000000000000000000..9a3c67cbd31675dfd190bddce338a54dd328c44a --- /dev/null +++ b/CMake/FindSphinx.cmake @@ -0,0 +1,629 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +# Heavily based on FindSphinx.cmake (https://github.com/k0ekk0ek/cmake-sphinx) + +#[=======================================================================[.rst: +FindSphinx +---------- + +Sphinx is a documentatino generation tool (see https://www.sphinx-doc.org). +This module looks for Sphinx and some optional extensions in order to connect +it to Doxygen documentation. These tools are enabled as components in the +:command:`find_package` command: + +``breathe`` + `Breathe `_ A utility that provides a bridge + between the Sphinx and Doxygen documentation systems. + +Examples: + +.. code-block:: cmake + + find_package(Sphinx + REQUIRED breathe + OPTIONAL_COMPONENTS exhale) + +The following variables are defined by this module: + +.. variable:: SPHINX_FOUND + + True if the ``sphinx`` executable was found. + +.. variable:: SPHINX_BUILD_VERSION + + The version reported by ``sphinx-build --version``. + + +Functions +^^^^^^^^^ + +.. command:: sphinx_add_docs + + This function is intended as a convenience for adding a target for generating + documentation with Sphinx. It aims to provide sensible defaults so that + projects can generally just provide the input files and directories and that + will be sufficient to give sensible results. + + :: + + sphinx_add_docs(targetName + [filesOrDirs...] + [ALL] + [BUILDER builder] + [SOURCE_DIRECTORY dir] + [OUTPUT_DIRECTORY dir] + [BREATHE_PROJECTS doxygenTargetName] + [COMMENT comment]) + + The function defines a custom target that runs Sphinx on all plain-text + document sources found in ``SOURCE_DIRECTORY``, including its subdirectories. + The documentation generated will be placed in the directory defined by + the ``OUTPUT_DIRECTORY`` option (or in + `${CMAKE_CURRENT_BINARY_DIR}/targetName`, if the option is not provided). + + The ``CONF_TEMPLATE`` option allows users to provide a template for Sphinx's + configuration file ``conf.py``. This template will be copied into + ``${CMAKE_CURRENT_BINARY_DIR}/targetName.cache/_build`` using CMake's + `` configure_file()``, which means that any CMake variable can be used + in the template and will be replaced by with its corresponding value + at the time of invocation (see the `configure_file() documentation + `_) for more + details). + + .. variable:: SPHINX_EXTENSIONS + .. variable:: SPHINX_EXTENSIONS + + The ``BUILDER`` option allows controlling the Sphinx builder that should be + used when producing the documentation, e.g. html or man, among others. Please + refer to Sphinx's documentation for the full list. + + If ``BREATHE_PROJECTS`` is set, the function will use the Breathe extension to + include already-generated Doxygen documentation into the produced Sphinx + documentation. ```` must thus be a target that produces + such documentation, such as the one created with ``doxygen_add_docs()`` in the + ``FindDoxygen.cmake`` module. + + If provided, the optional ``comment`` will be passed as the ``COMMENT`` for + the :command:`add_custom_target` command used to create the custom target + internally. + + If ``ALL`` is set, the target will be added to the default build target. + + The contents of the generated ``conf.py`` for Sphinx can be customized by + setting CMake variables before calling ``sphinx_add_docs()``. Each of the + following will be explictly set unless the variable already has a value + before ``sphinx_add_docs()`` is called: + + .. variable:: EXHALE_CONTAINMENT_FOLDER + + This variable overrides the ``containmentFolder`` key in the + ``exhale_args`` dictionary (see the `Exhale documentation + `_ for + details). Set to ``api`` by this module. + + .. variable:: EXHALE_ROOT_FILE_NAME + + This variable overrides the ``rootFileName`` key in the + ``exhale_args`` dictionary (see the `Exhale documentation + `_ for + details). + Set to ``api.rst`` by this module. + + .. variable:: EXHALE_ROOT_FILE_TITLE + + This variable overrides the ``rootFileTitle`` key in the + ``exhale_args`` dictionary (see the `Exhale documentation + `_ for + details). + Set to ``${PROJECT_NAME} Reference`` by this module. + + To change any of these defaults, set relevant variables before calling + ``sphinx_add_docs()``. For example: + + .. code-block:: cmake + + set(EXHALE_CONTAINMENT_FOLDER api) + set(EXHALE_ROOT_FILE_NAME reference.rst) + set(EXHALE_ROOT_FILE_TITLE "Reference") + + sphinx_add_docs( + sphinx_docs + CONF_TEMPLATE "${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in" + BREATHE_PROJECTS doxygen_docs + BUILDER html + SOURCE_DIRECTORY ${CMAKE_SOURCE_DIR}/docs/sphinx + COMMENT "Generating Sphinx+Breathe documentation" + ) + + +#]=======================================================================] + +include(FindPackageHandleStandardArgs) + +macro(_Sphinx_find_executable _exe) + string(TOUPPER "${_exe}" _uc) + # sphinx-(build|quickstart)-3 x.x.x + # FIXME: This works on Fedora (and probably most other UNIX like targets). + # Windows targets and PIP installs might need some work. + find_program( + SPHINX_${_uc}_EXECUTABLE + NAMES "sphinx-${_exe}-3" "sphinx-${_exe}" "sphinx-${_exe}.exe") + + if(SPHINX_${_uc}_EXECUTABLE) + execute_process( + COMMAND "${SPHINX_${_uc}_EXECUTABLE}" --version + RESULT_VARIABLE _result + OUTPUT_VARIABLE _output + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(_result EQUAL 0 AND _output MATCHES " v?([0-9]+\\.[0-9]+\\.[0-9]+)$") + set(SPHINX_${_uc}_VERSION "${CMAKE_MATCH_1}") + endif() + + if(NOT TARGET Sphinx::${_exe}) + add_executable(Sphinx::${_exe} IMPORTED GLOBAL) + set_target_properties(Sphinx::${_exe} PROPERTIES + IMPORTED_LOCATION "${SPHINX_${_uc}_EXECUTABLE}") + endif() + set(Sphinx_${_exe}_FOUND TRUE) + else() + set(Sphinx_${_exe}_FOUND FALSE) + endif() + unset(_uc) +endmacro() + +macro(_Sphinx_find_module _name _module) + string(TOUPPER "${_name}" _Sphinx_uc) + if(SPHINX_PYTHON_EXECUTABLE) + execute_process( + COMMAND ${SPHINX_PYTHON_EXECUTABLE} -m ${_module} --version + RESULT_VARIABLE _result + OUTPUT_VARIABLE _output + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_QUIET) + if(_result EQUAL 0) + if(_output MATCHES " v?([0-9]+\\.[0-9]+\\.[0-9]+)$") + set(SPHINX_${_Sphinx_uc}_VERSION "${CMAKE_MATCH_1}") + endif() + + if(NOT TARGET Sphinx::${_name}) + set(SPHINX_${_Sphinx_uc}_EXECUTABLE "${SPHINX_PYTHON_EXECUTABLE} -m ${_module}") + add_executable(Sphinx::${_name} IMPORTED GLOBAL) + set_target_properties(Sphinx::${_name} PROPERTIES + IMPORTED_LOCATION "${SPHINX_PYTHON_EXECUTABLE}") + endif() + set(Sphinx_${_name}_ARGS -m ${_module}) + set(Sphinx_${_name}_FOUND TRUE) + else() + set(Sphinx_${_name}_FOUND FALSE) + endif() + else() + set(Sphinx_${_name}_FOUND FALSE) + endif() + unset(_Sphinx_uc) +endmacro() + +macro(_Sphinx_find_extension _ext) + if(SPHINX_PYTHON_EXECUTABLE) + execute_process( + COMMAND ${SPHINX_PYTHON_EXECUTABLE} -c "import ${_ext}" + RESULT_VARIABLE _result) + if(_result EQUAL 0) + set(Sphinx_${_ext}_FOUND TRUE) + else() + set(Sphinx_${_ext}_FOUND FALSE) + endif() + endif() +endmacro() + +# +# Find sphinx-build and sphinx-quickstart. +# + +# Find sphinx-build shim. +_Sphinx_find_executable(build) + +if(SPHINX_BUILD_EXECUTABLE) + # Find sphinx-quickstart shim. + _Sphinx_find_executable(quickstart) + + # Locate Python executable + if(CMAKE_HOST_WIN32) + # script-build on Windows located under (when PIP is used): + # C:/Program Files/PythonXX/Scripts + # C:/Users/username/AppData/Roaming/Python/PythonXX/Sripts + # + # Python modules are installed under: + # C:/Program Files/PythonXX/Lib + # C:/Users/username/AppData/Roaming/Python/PythonXX/site-packages + # + # To verify a given module is installed, use the Python base directory + # and test if either Lib/module.py or site-packages/module.py exists. + get_filename_component(_Sphinx_directory "${SPHINX_BUILD_EXECUTABLE}" DIRECTORY) + get_filename_component(_Sphinx_directory "${_Sphinx_directory}" DIRECTORY) + if(EXISTS "${_Sphinx_directory}/python.exe") + set(SPHINX_PYTHON_EXECUTABLE "${_Sphinx_directory}/python.exe") + endif() + unset(_Sphinx_directory) + else() + file(READ "${SPHINX_BUILD_EXECUTABLE}" _Sphinx_script) + if(_Sphinx_script MATCHES "^#!([^\n]+)") + string(STRIP "${CMAKE_MATCH_1}" _Sphinx_shebang) + if(EXISTS "${_Sphinx_shebang}") + set(SPHINX_PYTHON_EXECUTABLE "${_Sphinx_shebang}") + endif() + endif() + unset(_Sphinx_script) + unset(_Sphinx_shebang) + endif() +endif() + +if(NOT SPHINX_PYTHON_EXECUTABLE) + # Python executable cannot be extracted from shim shebang or path if e.g. + # virtual environments are used, fallback to find package. Assume the + # correct installation is found, the setup is probably broken in more ways + # than one otherwise. + find_package(Python3 QUIET COMPONENTS Interpreter) + if(TARGET Python3::Interpreter) + set(SPHINX_PYTHON_EXECUTABLE ${Python3_EXECUTABLE}) + # Revert to "python -m sphinx" if shim cannot be found. + if(NOT SPHINX_BUILD_EXECUTABLE) + _Sphinx_find_module(build sphinx) + _Sphinx_find_module(quickstart sphinx.cmd.quickstart) + endif() + endif() +endif() + +# +# Verify components are available. +# +if(SPHINX_BUILD_VERSION) + + # Breathe is required for Exhale + if("exhale" IN_LIST Sphinx_FIND_COMPONENTS AND NOT + "breathe" IN_LIST Sphinx_FIND_COMPONENTS) + list(APPEND Sphinx_FIND_COMPONENTS "breathe") + endif() + + # Find all requested components for Sphinx... + foreach(_Sphinx_component IN LISTS Sphinx_FIND_COMPONENTS) + if(_Sphinx_component STREQUAL "build") + # Do nothing, sphinx-build is always required. + continue() + elseif(_Sphinx_component STREQUAL "quickstart") + # Do nothing, sphinx-quickstart is optional, but looked up by default. + continue() + endif() + _Sphinx_find_extension(${_Sphinx_component}) + endforeach() + unset(_Sphinx_component) + + # + # Verify both executables are part of the Sphinx distribution. + # + if(SPHINX_QUICKSTART_VERSION AND NOT SPHINX_BUILD_VERSION STREQUAL SPHINX_QUICKSTART_VERSION) + message(FATAL_ERROR "Versions for sphinx-build (${SPHINX_BUILD_VERSION}) " + "and sphinx-quickstart (${SPHINX_QUICKSTART_VERSION}) " + "do not match") + endif() +endif() + +find_package_handle_standard_args( + Sphinx + VERSION_VAR SPHINX_BUILD_VERSION + REQUIRED_VARS SPHINX_BUILD_EXECUTABLE SPHINX_BUILD_VERSION + HANDLE_COMPONENTS) + +# Generate a conf.py template file using sphinx-quickstart. +# +# sphinx-quickstart allows for quiet operation and a lot of settings can be +# specified as command line arguments, therefore its not required to parse the +# generated conf.py. +function(_Sphinx_generate_confpy _target _cachedir) + if(NOT TARGET Sphinx::quickstart) + message(FATAL_ERROR "sphinx-quickstart is not available, needed by" + "sphinx_add_docs for target ${_target}") + endif() + + if(NOT DEFINED SPHINX_PROJECT) + set(SPHINX_PROJECT ${PROJECT_NAME}) + endif() + + if(NOT DEFINED SPHINX_AUTHOR) + set(SPHINX_AUTHOR "${SPHINX_PROJECT} committers") + endif() + + if(NOT DEFINED SPHINX_COPYRIGHT) + string(TIMESTAMP "%Y, ${SPHINX_AUTHOR}" SPHINX_COPYRIGHT) + endif() + + if(NOT DEFINED SPHINX_VERSION) + set(SPHINX_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}") + endif() + + if(NOT DEFINED SPHINX_RELEASE) + set(SPHINX_RELEASE "${PROJECT_VERSION}") + endif() + + if(NOT DEFINED SPHINX_LANGUAGE) + set(SPHINX_LANGUAGE "en") + endif() + + if(NOT DEFINED SPHINX_MASTER) + set(SPHINX_MASTER "index") + endif() + + set(_known_exts autodoc doctest intersphinx todo coverage imgmath mathjax + ifconfig viewcode githubpages) + + if(DEFINED SPHINX_EXTENSIONS) + foreach(_ext ${SPHINX_EXTENSIONS}) + set(_is_known_ext FALSE) + foreach(_known_ext ${_known_exsts}) + if(_ext STREQUAL _known_ext) + set(_opts "${opts} --ext-${_ext}") + set(_is_known_ext TRUE) + break() + endif() + endforeach() + if(NOT _is_known_ext) + if(_exts) + set(_exts "${_exts},${_ext}") + else() + set(_exts "${_ext}") + endif() + endif() + endforeach() + endif() + + if(_exts) + set(_exts "--extensions=${_exts}") + endif() + + set(_templatedir "${CMAKE_CURRENT_BINARY_DIR}/${_target}.template") + file(MAKE_DIRECTORY "${_templatedir}") + string(REPLACE " " ";" _Sphinx_executable ${SPHINX_QUICKSTART_EXECUTABLE}) + execute_process( + COMMAND ${_Sphinx_executable} + -q --no-makefile --no-batchfile + -p "${SPHINX_PROJECT}" + -a "${SPHINX_AUTHOR}" + -v "${SPHINX_VERSION}" + -r "${SPHINX_RELEASE}" + -l "${SPHINX_LANGUAGE}" + --master "${SPHINX_MASTER}" + ${_opts} ${_exts} "${_templatedir}" + RESULT_VARIABLE _result + OUTPUT_QUIET) + unset(_Sphinx_executable) + + if(_result EQUAL 0 AND EXISTS "${_templatedir}/conf.py") + file(COPY "${_templatedir}/conf.py" DESTINATION "${_cachedir}") + endif() + + file(REMOVE_RECURSE "${_templatedir}") + + if(NOT _result EQUAL 0 OR NOT EXISTS "${_cachedir}/conf.py") + message(FATAL_ERROR "Sphinx configuration file not generated for " + "target ${_target}") + endif() +endfunction() + +function(sphinx_add_docs _target) + set(_opts ALL) + set(_single_opts CONF_TEMPLATE BUILDER OUTPUT_DIRECTORY SOURCE_DIRECTORY COMMENT) + set(_multi_opts BREATHE_PROJECTS) + cmake_parse_arguments(_args "${_opts}" "${_single_opts}" "${_multi_opts}" ${ARGN}) + + unset(SPHINX_BREATHE_PROJECTS) + + if(NOT _args_COMMENT) + set(_args_COMMENT "Generating Sphinx API documentation for ${_target}") + endif() + + if(_args_CONF_TEMPLATE AND NOT EXISTS ${_args_CONF_TEMPLATE}) + message(FATAL_ERROR "CONF_TEMPLATE file '${_args_CONF_TEMPLATE}' does not exist") + endif() + + if(NOT EXHALE_CONTAINMENT_FOLDER) + set(EXHALE_CONTAINMENT_FOLDER "api") + endif() + + if(NOT EXHALE_ROOT_FILE_NAME) + set(EXHALE_ROOT_FILE_NAME "api.rst") + endif() + + if(NOT EXHALE_ROOT_FILE_TITLE) + set(EXHALE_ROOT_FILE_TITLE "${PROJECT_NAME} Reference") + endif() + + if(NOT _args_BUILDER) + message(FATAL_ERROR "Sphinx builder not specified for target ${_target}") + elseif(NOT _args_SOURCE_DIRECTORY) + message(FATAL_ERROR "Sphinx source directory not specified for target ${_target}") + else() + if(NOT IS_ABSOLUTE "${_args_SOURCE_DIRECTORY}") + get_filename_component(_sourcedir "${_args_SOURCE_DIRECTORY}" ABSOLUTE) + else() + set(_sourcedir "${_args_SOURCE_DIRECTORY}") + endif() + if(NOT IS_DIRECTORY "${_sourcedir}") + message(FATAL_ERROR "Sphinx source directory '${_sourcedir}' for" + "target ${_target} does not exist") + endif() + endif() + + set(_builder "${_args_BUILDER}") + if(_args_OUTPUT_DIRECTORY) + set(_outputdir "${_args_OUTPUT_DIRECTORY}") + else() + set(_outputdir "${CMAKE_CURRENT_BINARY_DIR}/${_target}") + endif() + + if(_args_BREATHE_PROJECTS) + if(NOT Sphinx_breathe_FOUND) + message(FATAL_ERROR "Sphinx extension 'breathe' is not available. Needed" + "by sphinx_add_docs for target ${_target}") + endif() + list(APPEND SPHINX_EXTENSIONS breathe) + + foreach(_doxygen_target ${_args_BREATHE_PROJECTS}) + if(TARGET ${_doxygen_target}) + list(APPEND _depends ${_doxygen_target}) + + # Doxygen targets are supported. Verify that a Doxyfile exists. + get_target_property(_dir ${_doxygen_target} BINARY_DIR) + set(_doxyfile "${_dir}/Doxyfile.${_doxygen_target}") + if(NOT EXISTS "${_doxyfile}") + message(FATAL_ERROR "Target ${_doxygen_target} is not a Doxygen" + "target, needed by sphinx_add_docs for target" + "${_target}") + endif() + + # Read the Doxyfile, verify XML generation is enabled and retrieve the + # output directory. + file(READ "${_doxyfile}" _contents) + if(NOT _contents MATCHES "GENERATE_XML *= *YES") + message(FATAL_ERROR "Doxygen target ${_doxygen_target} does not" + "generate XML, needed by sphinx_add_docs for" + "target ${_target}") + elseif(_contents MATCHES "OUTPUT_DIRECTORY *= *([^ ][^\n]*)") + string(STRIP "${CMAKE_MATCH_1}" _dir) + set(_name "${_doxygen_target}") + set(_dir "${_dir}/xml") + else() + message(FATAL_ERROR "Cannot parse Doxyfile generated by Doxygen" + "target ${_doxygen_target}, needed by" + "sphinx_add_docs for target ${_target}") + endif() + elseif(_doxygen_target MATCHES "([^: ]+) *: *(.*)") + set(_name "${CMAKE_MATCH_1}") + string(STRIP "${CMAKE_MATCH_2}" _dir) + endif() + + if(_name AND _dir) + if(_breathe_projects) + set(_breathe_projects "${_breathe_projects}, \"${_name}\": \"${_dir}\"") + else() + set(_breathe_projects "\"${_name}\": \"${_dir}\"") + endif() + if(NOT _breathe_default_project) + set(_breathe_default_project "${_name}") + endif() + endif() + endforeach() + endif() + + if(Sphinx_exhale_FOUND) + if(NOT Sphinx_breathe_FOUND) + message(FATAL_ERROR "Sphinx extension 'breathe' is not available, but is " + "required by exhale. Needed by sphinx_add_docs for " + "target ${_target}") + endif() + list(APPEND SPHINX_EXTENSIONS exhale) + endif() + + set(_cachedir "${CMAKE_CURRENT_BINARY_DIR}/${_target}.cache") + set(_cached_sourcesdir "${_cachedir}/_sources") + set(_cached_builddir "${_cachedir}/_build") + file(MAKE_DIRECTORY "${_cachedir}") + file(MAKE_DIRECTORY "${_cached_sourcesdir}") + file(MAKE_DIRECTORY "${_cached_builddir}") + file(MAKE_DIRECTORY "${_cached_builddir}/_static") + + # prepare Sphinx configuration file + set(_target_sphinx_conf "${_cached_builddir}/conf.py") + if(_args_CONF_TEMPLATE) + set(_sphinx_conf_template ${_args_CONF_TEMPLATE}) + + list(TRANSFORM SPHINX_EXTENSIONS REPLACE "^(.+)$" "'\\1'" OUTPUT_VARIABLE _foo) + list(JOIN _foo ",\n " SPHINX_EXTENSIONS) + unset(_foo) + + configure_file("${_sphinx_conf_template}" "${_target_sphinx_conf}" @ONLY) + else() + _Sphinx_generate_confpy(${_target} "${_cached_builddir}") + endif() + + if(_breathe_projects) + file(APPEND "${_target_sphinx_conf}" + "\n# Setup the breathe extension" + "\nbreathe_projects = { ${_breathe_projects} }" + "\nbreathe_default_project = '${_breathe_default_project}'" + "\n") + endif() + + if(Sphinx_exhale_FOUND) + set(_exhale_containment_folder "${_cached_sourcesdir}/${EXHALE_CONTAINMENT_FOLDER}") + set(_exhale_root_file_name ${EXHALE_ROOT_FILE_NAME}) + set(_exhale_root_file_title "${EXHALE_ROOT_FILE_TITLE}") + + file(APPEND "${_target_sphinx_conf}" + "\n# Setup the exhale extension" + "\nexhale_args = {" + "\n 'containmentFolder': '${_exhale_containment_folder}'," + "\n 'rootFileName': '${_exhale_root_file_name}'," + "\n 'rootFileTitle': '${_exhale_root_file_title}'," + "\n 'doxygenStripFromPath': '${CMAKE_SOURCE_DIR}'," + "\n 'createTreeView': True" + "\n}") + endif() + + unset(_all) + if(${_args_ALL}) + set(_all ALL) + endif() + + # Propagate some variables + set(SPHINX_OUTPUT_DIRECTORY "${_outputdir}" PARENT_SCOPE) + + string(REPLACE " " ";" _Sphinx_executable ${SPHINX_BUILD_EXECUTABLE}) + add_custom_target( + ${_target} ${_all} + COMMAND ${CMAKE_COMMAND} -E copy_directory ${_sourcedir} ${_cached_sourcesdir} + COMMAND ${_Sphinx_executable} + -b ${_builder} + -d "${_cached_builddir}/_doctrees" + -c "${_cached_builddir}" + "${_cached_sourcesdir}" + "${_outputdir}" + DEPENDS "${_target_sphinx_conf}" ${_depends} + COMMENT "${_args_COMMENT}") + unset(_Sphinx_executable) + + list(APPEND + _sphinx_generated_files + ${_cached_builddir}/_doctrees + ${_cached_builddir}/_static + ${_cached_sourcesdir} + ) + + set_target_properties(${_target} + PROPERTIES ADDITIONAL_CLEAN_FILES "${_sphinx_generated_files}") + +endfunction() diff --git a/CMakeLists.txt b/CMakeLists.txt index 45669d7f6fcf08d55b071e49ae114014be0ff765..5a69c62aa3eba4b1521562411eca18bef4cb74b1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,6 +140,14 @@ find_package(Date REQUIRED) # Import some convenience functions include(gkfs-utils) +################################################################################ +## Build GekkoFS documentation +################################################################################ +option(GKFS_BUILD_DOCUMENTATION "Build documentation" OFF) +if(GKFS_BUILD_DOCUMENTATION) + add_subdirectory(docs) +endif() + option(CREATE_CHECK_PARENTS "Check parent directory existance before creating child node" ON) message(STATUS "[gekkofs] Create checks parents: ${CREATE_CHECK_PARENTS}") diff --git a/docker/0.8.0/docs/Dockerfile b/docker/0.8.0/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..f8ff5ba94bbb19c5c4d5b758cc8d40f1eb4cee6c --- /dev/null +++ b/docker/0.8.0/docs/Dockerfile @@ -0,0 +1,33 @@ +FROM gekkofs/deps:0.8.0 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.31.0' \ + 'exhale==0.2.4' \ + 'sphinx-copybutton==0.4.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.15.1' && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.8.0/docs/Makefile b/docker/0.8.0/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..81025201fd8fc8187f26469f5f3c4cf8020d021e --- /dev/null +++ b/docker/0.8.0/docs/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/docs:0.8.0 . diff --git a/docker/0.9.0/docs/Dockerfile b/docker/0.9.0/docs/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..d657d8767039887f2d91016731b8c2965dd6c7bd --- /dev/null +++ b/docker/0.9.0/docs/Dockerfile @@ -0,0 +1,33 @@ +FROM gekkofs/deps:0.9.0 + +LABEL Description="Debian-based environment suitable to build GekkoFS' documentation" + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + # install dependencies for Doxygen + python \ + flex \ + bison \ + graphviz && \ + # install doxygen (repo version is kind of old) + cd /tmp && curl -OL https://www.doxygen.nl/files/doxygen-1.9.2.src.tar.gz && \ + tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \ + mkdir -p /tmp/doxygen-1.9.2/build && \ + cd /tmp/doxygen-1.9.2/build && \ + cmake -G "Unix Makefiles" .. && \ + make -j8 install && \ + # install sphinx, breathe and exhale + pip3 install \ + 'sphinx==4.4.0' \ + sphinx_rtd_theme \ + 'breathe==4.31.0' \ + 'exhale==0.2.4' \ + 'sphinx-copybutton==0.4.0' \ + 'sphinx-multiversion==0.2.4' \ + 'myst_parser==0.15.1' && \ + # Clean apt cache to reduce image layer size + rm -rf /var/lib/apt/lists/* && \ + rm -rf /tmp/doxygen-1.9.2 && \ + rm /tmp/doxygen-1.9.2.src.tar.gz && \ + # Clean apt caches of packages + apt-get clean && apt-get autoclean diff --git a/docker/0.9.0/docs/Makefile b/docker/0.9.0/docs/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..7f0bd2c05fd4a97e9846a898638da80205173293 --- /dev/null +++ b/docker/0.9.0/docs/Makefile @@ -0,0 +1,4 @@ +.PHONY: all + +all: + docker build -t gekkofs/docs:0.9.0 . diff --git a/docs/CMakeLists.txt b/docs/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..fc6d036443d5160f5511a32244b9a6efb26469fe --- /dev/null +++ b/docs/CMakeLists.txt @@ -0,0 +1,36 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +add_subdirectory(doxygen) +add_subdirectory(sphinx) + +# Define a convenience target to build the documentation: +# gkfs_sphinx already depends on gkfs_doxygen, no need to +# add it explicitly +add_custom_target(docs + DEPENDS sphinx_docs) diff --git a/docs/doxygen/CMakeLists.txt b/docs/doxygen/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..22009c5980ef5abbd79339b83c76033f7924ac7b --- /dev/null +++ b/docs/doxygen/CMakeLists.txt @@ -0,0 +1,92 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +find_package(Doxygen REQUIRED dot) + +set(DOXYGEN_FULL_PATH_NAMES YES) +set(DOXYGEN_WARN_IF_UNDOCUMENTED NO) + +# Sphinx needs the Doxygen's XML output, but doesn't care about its HTML output +set(DOXYGEN_GENERATE_HTML NO) +set(DOXYGEN_GENERATE_XML YES) + +set(DOXYGEN_GENERATE_TREEVIEW NO) +set(DOXYGEN_GENERATE_TODOLIST YES) +set(DOXYGEN_GENERATE_BUGLIST YES) +set(DOXYGEN_GENERATE_HTMLHELP NO) +set(DOXYGEN_GENERATE_LATEX NO) +set(DOXYGEN_GENERATE_MAN NO) +set(DOXYGEN_GENERATE_RTF NO) + +set(DOXYGEN_ENABLE_PREPROCESSING YES) +set(DOXYGEN_MACRO_EXPANSION YES) + +set(DOXYGEN_SEARCH_INCLUDES YES) +set(DOXYGEN_EXPAND_ONLY_PREDEF YES) + +# extra defs for exhale to help with building the _right_ version of the docs +list(APPEND DOXYGEN_PREDEFINED + DOXYGEN_DOCUMENTATION_BUILD + DOXYGEN_SHOULD_SKIP_THIS) +set(DOXYGEN_CREATE_SUBDIRS NO) + +# Allow for rst directives and advanced functions e.g. grid tables +list(APPEND DOXYGEN_ALIASES + "\"rst=\\verbatim embed:rst:leading-asterisk\"" + "\"endrst=\\endverbatim\"") + +set(DOXYGEN_ENABLE_PREPROCESSING YES) +set(DOXYGEN_MACRO_EXPANSION YES) +set(DOXYGEN_EXPAND_ONLY_PREDEF NO) + +set(DOXYGEN_STRIP_FROM_PATH ${CMAKE_SOURCE_DIR}) + +list(APPEND + DOXYGEN_EXCLUDE_PATTERNS + "${PROJECT_SOURCE_DIR}/*/rpc_types.*" + "${PROJECT_SOURCE_DIR}/*/detail/*" + "${PROJECT_SOURCE_DIR}/tests/*/test_*.cpp" + "${PROJECT_SOURCE_DIR}/tests/integration/*" + "${PROJECT_SOURCE_DIR}/*/README.md") + +doxygen_add_docs( + doxygen_docs + ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/tests + COMMENT "Generating Doxygen documentation" + ) + +## ensure Doxygen files are cleaned with 'make clean' +## (this should be done by doxygen_add_docs() but...) +list(APPEND + _doxygen_generated_files + ${CMAKE_CURRENT_BINARY_DIR}/xml + ) +set_target_properties(doxygen_docs + PROPERTIES ADDITIONAL_CLEAN_FILES "${_doxygen_generated_files}") diff --git a/docs/sphinx/CMakeLists.txt b/docs/sphinx/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d01c10a6ba95669419b455e43e7bbee0f0739829 --- /dev/null +++ b/docs/sphinx/CMakeLists.txt @@ -0,0 +1,50 @@ +################################################################################ +# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# This file is part of GekkoFS. # +# # +# GekkoFS is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# GekkoFS is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with GekkoFS. If not, see . # +# # +# SPDX-License-Identifier: GPL-3.0-or-later # +################################################################################ + +find_package(Sphinx REQUIRED breathe exhale) + +set(EXHALE_CONTAINMENT_FOLDER api) +set(EXHALE_ROOT_FILE_NAME reference.rst) +set(EXHALE_ROOT_FILE_TITLE "Reference") + +sphinx_add_docs( + sphinx_docs + CONF_TEMPLATE "${CMAKE_CURRENT_SOURCE_DIR}/conf.py.in" + BREATHE_PROJECTS doxygen_docs + BUILDER html + SOURCE_DIRECTORY ${CMAKE_SOURCE_DIR}/docs/sphinx + COMMENT "Generating Sphinx+Breathe documentation" + ) + +# Add an install target to install the docs +# (the trailing slash is important to install the contents inside +# SPHINX_OUTPUT_DIRECTORY without including the directory itself) +include(GNUInstallDirs) +install(DIRECTORY "${SPHINX_OUTPUT_DIRECTORY}/" + OPTIONAL + DESTINATION ${CMAKE_INSTALL_DOCDIR}) diff --git a/docs/sphinx/_templates/footer.html b/docs/sphinx/_templates/footer.html new file mode 100644 index 0000000000000000000000000000000000000000..81b1e10786ad2fbf38d20624ad75787ce16b3d08 --- /dev/null +++ b/docs/sphinx/_templates/footer.html @@ -0,0 +1,68 @@ +
+ {%- if (theme_prev_next_buttons_location == 'bottom' or theme_prev_next_buttons_location == 'both') and (next or prev) %} + + {%- endif %} + +
+ +
+ {%- block contentinfo %} +

+ {%- if show_copyright %} + {%- if hasdoc('copyright') %} + {% trans path=pathto('copyright'), copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} + {%- else %} + {%- if copyright is string %} + {% trans copyright=copyright|e %}© Copyright {{ copyright }}.{% endtrans %} + {%- else %} + {%- for copyright_holder in copyright %} + {% trans copyright_holder=copyright_holder|e %}© Copyright {{ copyright_holder }}.
{% endtrans %} + {%- endfor %} + {%- endif %} + {%- endif %} + {%- endif %} + + {%- if build_id and build_url %} + + {# Translators: Build is a noun, not a verb #} + {% trans %}Build{% endtrans %} + {{ build_id }}. + + {%- elif commit %} + + {# Translators: the phrase "revision" comes from Git, referring to a commit #} + {% trans %}Revision{% endtrans %} {{ commit }}. + + {%- endif %} + {%- if last_updated %} + + {% trans last_updated=last_updated|e %}Last updated on {{ last_updated }}.{% endtrans %} + + {%- endif %} + +

+ {%- endblock %} +
+ + {%- if show_sphinx %} + {% set sphinx_web = 'Sphinx' %} + {% set readthedocs_web = 'Read the Docs' %} + {# Translators: the variable "sphinx_web" is a link to the Sphinx project documentation with the text "Sphinx" #} + {% trans sphinx_web=sphinx_web, readthedocs_web=readthedocs_web %}Built with {{ sphinx_web }} using a{% endtrans %} + {# Translators: "theme" refers to a theme for Sphinx, which alters the appearance of the generated documenation #} + {% trans %}theme{% endtrans %} + {# Translators: this is always used as "provided by Read the Docs", and should not imply Read the Docs is an author of the generated documentation. #} + {% trans %}provided by {{ readthedocs_web }}{% endtrans %}. + {%- endif %} + + {%- block extrafooter %} {% endblock %} + +
+ diff --git a/docs/sphinx/_templates/layout.html b/docs/sphinx/_templates/layout.html new file mode 100644 index 0000000000000000000000000000000000000000..3623d1c9e8283828c7fdfacf2926ccff48a19e74 --- /dev/null +++ b/docs/sphinx/_templates/layout.html @@ -0,0 +1,16 @@ +{%- extends "!layout.html" %} + {%- block htmltitle %} + {% if versions %} + {{ title|striptags|e }} - {{ project }} {{ current_version.name }} Documentation + {% else %} + {{ title|striptags|e }}{{ titlesuffix }} + {% endif %} + {%- endblock %} + + {%- block rootrellink %} + {% if versions %} +
  • {{ project }} {{ current_version.name }} Documentation{{ reldelim1 }}
    • + {% else %} +
  • {{ shorttitle|e }}{{ reldelim1 }}
    • + {% endif %} +{%- endblock %} diff --git a/docs/sphinx/_templates/versions.html b/docs/sphinx/_templates/versions.html new file mode 100644 index 0000000000000000000000000000000000000000..31a1257898a9dc750d7c4a9fb281e135f04a8273 --- /dev/null +++ b/docs/sphinx/_templates/versions.html @@ -0,0 +1,27 @@ +{%- if current_version %} +
    + + Other Versions + v: {{ current_version.name }} + + +
    + {%- if versions.tags %} +
    +
    Tags
    + {%- for item in versions.tags %} +
    {{ item.name }}
    + {%- endfor %} +
    + {%- endif %} + {%- if versions.branches %} +
    +
    Branches
    + {%- for item in versions.branches %} +
    {{ item.name }}
    + {%- endfor %} +
    + {%- endif %} +
    +
    +{%- endif %} diff --git a/docs/sphinx/conf.py.in b/docs/sphinx/conf.py.in new file mode 100644 index 0000000000000000000000000000000000000000..bd1497e18c1e01dd69bcf1c5dcb06d8766c72107 --- /dev/null +++ b/docs/sphinx/conf.py.in @@ -0,0 +1,86 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'GekkoFS' +copyright = ['2018-2022, Barcelona Supercomputing Center, Spain', '2015-2022, Johannes Gutenberg Universitaet Mainz, Germany' ] +author = 'GekkoFS committers' + +# The short X.Y version +version = '0.8.0' + +# The full version, including alpha/beta/rc tags +release = '0.8.0-snapshot+96-g9cafaaa3-dirty' + +# Tell sphinx what the primary language being documented is. +primary_domain = 'cpp' + +# Tell sphinx what the pygments highlight language should be. +highlight_language = 'cpp' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx_copybutton', + 'sphinx_multiversion', + 'myst_parser', + @SPHINX_EXTENSIONS@ +] + +source_suffix = { + '.md': 'markdown', + '.rst': 'restructuredtext', + '.py': 'python', + '.cpp': 'c++' +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['@CMAKE_CURRENT_SOURCE_DIR@/_templates'] + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Extension configuration ------------------------------------------------- +copybutton_prompt_text = '$ ' diff --git a/docs/sphinx/devs/testing.md b/docs/sphinx/devs/testing.md new file mode 100644 index 0000000000000000000000000000000000000000..c288a8c1a84fe55e35ff0f2dc9b87cb10bfc8aeb --- /dev/null +++ b/docs/sphinx/devs/testing.md @@ -0,0 +1,568 @@ +# Testing + +This page describes the testing infrastructure of the GekkoFS file system. It is +intended to provide a general idea of how the different infrastructure +components are organized and their interactions, so that developers can use them +to write new tests (and/or extend the framework) with as few prototyping +overhead as possible. + +## Integration and functionality tests + +GekkoFS provides an automated testing harness to simplify writing and running +integration and functional tests for the file system. For simplicity and ease of +development, tests are written in Python (3.6+ required). The harness itself +relies on the [pytest](https://docs.pytest.org/en/latest/) framework to simplify +testing the GekkoFS infrastructure. Among others, `pytest` offers the following +features: + +- Detailed info on failing assert statements; +- [Auto-discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) + of test modules and functions; +- [Modular fixtures](https://docs.pytest.org/en/latest/fixture.html#fixture) for + managing small or parametrized long-lived test resources; +- Rich plugin architecture, in case further functionality is required. + +The code below shows an example of a simple test that creates a directory on the +file system's root by invoking `mkdir()` and checks the returned value +and `errno` error code. + +```python +import harness, errno, stat +from pathlib import Path + + +def test_mkdir(gkfs_daemon, gkfs_client): + """Create a new directory in the FS's root""" + + topdir = gkfs_daemon.mountdir / "top" + + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 0 +``` + +### Executing functional tests + +In order for tests to be built, the `GKFS_BUILD_TESTS` option needs to be +enabled when configuring the package: + +```console +$ mkdir build && cd build +$ cmake -DGKFS_BUILD_TESTS=ON .. +$ make -j8 +``` + +Once built, tests can be run in several ways: + +```console +# 1. run all tests registered with CTest +$ make test + +# 2. build and run all tests registered with CTest +# (equivalent to ctest --force-new-ctest-process --verbose --output-on-failure) +$ make check + +# 3. configure the run (see `ctest --help` for available options) +$ ctest [ options ] +``` + +It is also possible to invoke `pytest` directly by taking advantage of the +Python virtualenv that CMake creates to run the tests. This has the advantage +that the user can pass arguments directly to the `pytest` framework and that +pretty-printed output is sent directly to the console. + +```{eval-rst} +.. note:: + :code:`py.test` must be run from the :code:`build/tests/integration` + directory to work. +``` + + +```console +# option 1: run the 'mkdir' test directly from the 'build/tests' directory +# in verbose mode, without console capture and using the 'docker0' interface +$ pytest-venv/bin/py.test -s -v --interface=docker0 -k test_mkdir + + +# option2: same example but activating the virtualenv +$ source pytest-venv/bin/activate +(pytest-venv) $ py.test -s -v --interface=docker0 -k test_mkdir +(pytest-venv) $ deactivate +``` + +In both cases, the test output should be similar to this: + +```shell +===================================================================================== test session starts ====================================================================================== +platform linux -- Python 3.7.3, pytest-5.3.5, py-1.8.1, pluggy-0.13.1 -- /opt/gekkofs/build/tests/pytest-venv/bin/python3.7 +cachedir: .pytest_cache +rootdir: /opt/gekkofs/build/tests, inifile: pytest.ini, testpaths: /opt/gekkofs/tests +plugins: xdist-1.31.0, dependency-0.4.0, forked-1.1.3 +collected 20 items / 19 deselected / 1 selected + +test_directories.py::test_mkdir 2020-02-25 14:42:08.923 | INFO | harness.workspace:__init__:43 - workspace created at /tmp/pytest-of-testuser/pytest-190/test_mkdir0 +2020-02-25 14:42:08.924 | INFO | harness.gkfs:run:122 - spawning daemon +2020-02-25 14:42:08.924 | INFO | harness.gkfs:run:123 - cmdline: /opt/gekkofs/build/src/daemon/gkfs_daemon --mountdir /tmp/pytest-of-testuser/pytest-190/test_mkdir0/mnt --rootdir /tmp/pytest-of-testuser/pytest-190/test_mkdir0/root -l docker0:12769 +2020-02-25 14:42:08.924 | INFO | harness.gkfs:run:124 - env: {'LD_LIBRARY_PATH': ':/opt/gekkofs/build/tests:/opt/gekkofs/prefix/lib', 'GKFS_HOSTS_FILE': PosixPath('/tmp/pytest-of-testuser/pytest-190/test_mkdir0/gkfs_hosts.txt'), 'GKFS_DAEMON_LOG_PATH': PosixPath('/tmp/pytest-of-testuser/pytest-190/test_mkdir0/logs/gkfs_daemon.log'), 'GKFS_LOG_LEVEL': '100'} +2020-02-25 14:42:08.927 | INFO | harness.gkfs:run:134 - daemon process spawned (PID=14946) +2020-02-25 14:42:09.040 | INFO | harness.gkfs:run:251 - running client +2020-02-25 14:42:09.041 | INFO | harness.gkfs:run:252 - cmdline: /opt/gekkofs/build/tests/harness/gkfs.io /tmp/pytest-of-testuser/pytest-190/test_mkdir0/mnt/top 511 +2020-02-25 14:42:09.042 | INFO | harness.gkfs:run:253 - env: {'LD_LIBRARY_PATH': ':/opt/gekkofs/build/tests:/opt/gekkofs/prefix/lib', 'LD_PRELOAD': PosixPath('libgkfs_intercept.so'), 'LIBGKFS_HOSTS_FILE': PosixPath('/tmp/pytest-of-testuser/pytest-190/test_mkdir0/gkfs_hosts.txt'), 'LIBGKFS_LOG': 'all', 'LIBGKFS_LOG_OUTPUT': PosixPath('/tmp/pytest-of-testuser/pytest-190/test_mkdir0/logs/gkfs_client.log')} +2020-02-25 14:42:09.390 | DEBUG | harness.gkfs:run:262 - command output: b'{\n "errnum": 0,\n "retval": 0\n}\n' +PASSED +2020-02-25 14:42:15.219 | INFO | harness.gkfs:shutdown:197 - terminating daemon + + +=============================================================================== 1 passed, 19 deselected in 1.54s ================================================================================ + +``` + +```{eval-rst} +.. warning:: + + Be careful not to run :code:`make` with the virtualenv activated. When that + happens, CMake considers the virtualenv's Python interpreter as the + system-wide one and caches this information, thus failing to run tests when + the virtualenv is deactivated. +``` +*** + +### Integration with CMake + +As shown in the examples above, the functional testing harness is integrated +with CMake's built-in testing tool. The CMake software suite includes +the `CTest` tool which can be used to automate the testing phase, or even the +entire process of configuring, building, testing and even submitting results to +a dashboard. Thus, integrating the `pytest` testing harness with it allows +end-users to easily execute tests, and also integrate with expected CMake +workflows. Nevertheless, `CTest` requires all tests to be defined by hand in any +involved `CMakeLists.txt`, whereas `pytest` is capable of automatically finding +all Pyhton test functions whose names follow certain patterns, which is why +tests are *semantically organized* into directories, and test groups are added +to CTest as shown below. + +#### Adding new tests to CMake + +The GekkoFS testing framework provides the `gkfs_add_python_test` CMake function +to simplify creating test groups. Thus, if a `${GKFS_ROOTDIR}/tests/io` +subdirectory exists that should contain all the tests that exercise and verify +that I/O works as expected, the test group could be added to CMake by adding the +following code to `${GKFS_ROOTDIR}/tests/CMakeLists.txt`: + +```cmake +gkfs_add_python_test( + NAME test_io + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + SOURCE tests/io/ +) +``` + +The function creates a new `CTest` test called `test_io` that will internally +call `pytest` and instruct it to auto-discover tests starting on the file or +directory defined by `SOURCE`. It is also possible to define +the `WORKING_DIRECTORY` required for the test, though for now only a value +of `${PROJECT_SOURCE_DIR}` is supported. + +### Testing harness organization + +The testing harness resides on the `${GKFS_ROOTDIR}/tests` subdirectory and +consists of the following files, directories, and helper programs: + +- `pytest.ini.in`: `pytest` determines a `rootdir` for each test run which + depends on the command line arguments (specified test files, paths) and on the + existence of *ini-files*, which allow to set pre-configured settings for a + test run without having to rely on command line arguments ( + see [here](https://docs.pytest.org/en/latest/customize.html#initialization-determining-rootdir-and-inifile)) + . Unfortunately, `pytest`'s `rootdir` finding algorithm will always determine + that the first directory to contain a `pytest.ini` file will become + the `rootdir`. Since we want to be able to run the tests by + invoking `make test` from `${CMAKE_BINARY_DIR}`, **but** we don't want to copy + the tests source files to the binary directory, we use this template to + generate a `pytest.ini` file that instructs `pytest` to run the tests + in `${CMAKE_CURRENT_SOURCE_DIR}`, i.e. our main `tests` subdirectory. + +- `conftest.py`: This is the main `pytest` configuration file that should + contain the definition of all fixtures that should be shared by multiple + tests (refer to `pytest`' + s [documentation](https://docs.pytest.org/en/latest/fixture.html#fixtures-a-prime-example-of-dependency-injection) + for more information). Its purpose is to add extra options to `pytest`'s CLI, + setup logging, and define the fixtures for the `Workspace`, `Daemon`, + and `Client` classes (see below). + +- `conftest.py.in`: This template file allows adding CLI arguments to `pytest` + regardless of whether it is run from `${CMAKE_CURRENT_SOURCE_DIR}` + or `${CMAKE_BINARY_DIR}`. As mentioned, the `pytest` framework auto-discovers + tests, *ini-files*, and `conftest.py` files and uses this information to + determine the `rootdir` of the tests. Unfortunately, when the first ini-file + is found, `pytest` considers its parent directory as the `rootdir` and expects + to find the *main* `conftest.py` in the same directory. Thus, in order to + successfully add extra arguments to `pytest`'s CLI *while keeping the tests + sources in `${CMAKE_CURRENT_SOURCE_DIR}`*, we must have a valid `conftest.py` + file in `${CMAKE_BINARY_DIR}`. + +- `cli.py`: This module exports the function `add_cli_options()` that adds + the `--interface`, `--bin-dir`, and `--lib-dir` CLI arguments to `pytest`. + This file is the only python source file that is copied + to `${CMAKE_BINARY_DIR}` so that it can be called + from `${CMAKE_BINARY_DIR}/conftest.py`. + +- `gkfs.io/`: This directory contains the sources for the `gkfs.io` helper + program. This program acts as a proxy for the `Client` class described below + to execute I/O-related system calls and library functions from a `LD_PRELOAD` + context. Results from the function execution are returned in JSON format so + that they can be easily parsed by the `Client` class (see `io` module below). + +- `gkfs.py`: This module exports the `Daemon`, `Client`, and `ShellClient` + classes which allow tests to interface easily with the GekkoFS daemon, client + library, and shell, respectively (see below for more details). + +- `io.py`: This module exports an `IOParser` class that is used internally by + the `Client` class to deserialize any JSON output produced by the `gkfs.io` + helper program. JSON deserialization relies + on [marshmallow](https://marshmallow.readthedocs.io/en/stable/) fields and + schemas to convert JSON strings to native Python datatypes. + +- `cmd.py`: This module exports a `CommandParser` class that is used internally + by `ShellClient` class to deserialize output strings generated by shell + commands into native Python datatypes. + +- `logger.py`: This module exports the `harness.logger` alias which hides the + implementation details of the actual logging framework used. Any tests willing + to produce logging messages only need to `import harness.logger` and call the + appropriate `logger.LEVEL(msg)` function. All standard Python logging levels + are supported. + +- `workspace.py`: This module exports the `Workspace` class which allows tests + to setup and interact with their workspace. + +### Useful fixtures available to tests + +The project's testing harness relies on `pytest` internal fixture mechanism to +setup and teardown resources for the test (e.g. daemon and client processes). By +leveraging `pytest`'s mechanism for automatic fixture injection, any test can +activate the harness' automatic resource management by simply declaring the use +of a harness fixture from test functions, modules, classes or whole projects. + +In order to simplify writing tests and to ensure that they can run in parallel, +the harness currently provides the following fixtures, which are defined +in [`${GKFS_ROOTDIR}/tests/conftest.py`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/conftest.py): + +#### workspace + +The `workspace` fixture returns an instance of +the [`Worskpace`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/harness/workspace.py#L3) +class that implements a self-contained subdirectory where a test can run and +generate artifacts in a self-contained manner. Given a *test working +directory* `twd`, a workspace for a test is initialized by creating the +following directories under `twd`: + +```shell +twd # base directory for test (typically under the system's temporary directory) +├── logs # directory for logs +│   ├── gkfs_client.log +│   └── gkfs_daemon.log +├── mnt # directory for GekkoFS' virtual mount point +├── root # directory for GekkoFS' internal data +│ └── 14935 +│ ├── data +│ │   └── chunks +│ └── rocksdb +│ ├── 000003.log +│ ├── 000006.sst +│ ├── CURRENT +│ ├── IDENTITY +│ ├── LOCK +│ ├── LOG +│ ├── MANIFEST-000007 +│ └── OPTIONS-000005 +└── tmp # temporary directory for test +``` + +For convenience, the `Workspace` class can be passed a `bindirs` and `libdirs` +arguments that allow to respectively influence the `PATH` and `LD_LIBRARY_PATH` +environment variables that will be used when executing internal shell commands +under it. + +*** +**IMPORTANT** +*The workspace fixture is a direct dependency of the `gkfs_daemon` +and `gkfs_client` test fixtures described below, which means that it's +transitively added to any tests depending on these fixtures. There is seldom any +need to directly declare it or use it when writing a test.* +*** + +#### file_factory + +The `file_factory` fixture returns an instance of +the [`harness.workspace.FileCreator`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/harness/workspace.py#L110) +class that allows test developers to create custom files with random binary +contents in the test workspace. To do so, the `FileCreator` class provides +a `create(pathname, size, unit)` function to generate the desired file, which in +turn returns a `workspace.File` object that represents the newly created file. +This `File` object offers convenience methods and properties to interact with +the created file in the context of a test (e.g. `md5sum()`): + +```python +def test_foobar(gkfs_daemon, gkfs_shell, file_factory): + # create a 4MB file and compute its md5sum + lf01 = file_factory.create(file01, size=4.0, unit='MB') + digest = lf01.md5sum() # digest => '7f45c62700402ce5f9abe5b8d70d2844' +``` + +#### gkfs_daemon + +The `gkfs_daemon` fixture returns an instance of +the [`harness.gkfs.Daemon`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/daemon/tests/harness/gkfs.py#L87) +class that represents a local running daemon running in an isolated test +workspace. The `Daemon` class provides methods to control the daemon process +which are automatically invoked by `pytest` when a test starts/finishes to start +up/shut down the daemon required by the test. + +The class relies on Python's [sh](https://amoffat.github.io/sh/) module to +configure, spawn and monitor a GekkoFS daemon process. The process is executed +independently of the test process and the class sends the appropriate `SIGTERM` +signal when the test finishes to properly shut down the daemon. + +Currently, the daemon is executed with the following configuration: + +- `LD_LIBRARY_PATH`: The `LD_LIBRARY_PATH` variable is set to the contents + of `$LD_LIBRARY_PATH` when the test is invoked plus any additional paths + defined in `libdirs`. +- `GKFS_HOSTS_FILE`: The `GKFS_HOSTS_FILE` variable is set + to `/gkfs_hosts.txt`. +- `GKFS_DAEMON_LOG_PATH`: The `GKFS_DAEMON_LOG_PATH` variable is set + to `/logs/gkfs_daemon.log`. +- `GKFS_LOG_LEVEL`: The `GKFS_LOG_LEVEL` variable is set to `100`. +- `--mountdir,-m`: The `--mountdir` CLI argument is set to `/mnt`. +- `--rootdir,-r`: The `--rootdir` CLI argument is set to `/root`. +- `--listen,-l`: The `--listen` CLI argument is set to an ephemeral network + address (IPv4:port) generated from the interface defined provided by the + test's `Workspace` (`lo` by default) and a randomly selected unused port in + the range `[1024, 32768)`. + +The `Daemon` class exposes the following properties from a running daemon to +tests: + +- `cwd`: the daemon's current working directory. +- `rootdir`: the daemon's root directory in the host file system. +- `mountdir`: the daemon's mount directory in the host file system. +- `logdir`: the directory used by the daemon to store logs. +- `interface`: the daemon's address used for communication. + +#### gkfs_client + +The `gkfs_client` fixture returns an instance of +the [`harness.gkfs.Client`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/harness/gkfs.py#L220) +class. The `Client` class allows users to execute I/O-related functions from the +glibc (i.e. system calls and library functions) in their own separate processes +directly from Python code. The harness again relies on the `sh` module to spawn +the processes with an appropriately patched environment so that they can +communicate successfully with a running daemon. + +To simplify usage and reduce coding overhead, the class provides a +special `__getattr__()` method that tries to transform any method called on +a `Client` instance to a `gkfs.io` command. Thus, the following code: + +```python +gkfs_client.mkdir("/foobar/", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) +``` + +transforms into: + +```console +$ gkfs.io mkdir /foobar/ 0777 +{ + "errnum": 0, + "retval": 0 +} +``` + +As shown in the excerpt above, `gkfs.io` returns function call results as JSON +records through `stdout`, which are deserialized by the `IOParser` +in [`io.py`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/harness/io.py#L165) +and transformed into a Python's `namedtuple`. Thus, the previous function call +would return the following: + +```ipython +In[1]: gkfs_client.mkdir("/foobar/", stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) +MkdirReturn(retval=0, errno=115) +``` + +```{eval-rst} +.. warning:: + Though providing a :code:`__getattr__()` method allows for more expressive + tests, it also means that **any function name** called from an instance of + the :code:`Client class` automatically gets transformed into a + :code:`gkfs.io` argument. This may cause unexpected errors if the + corresponding subcommand has not yet been implemented in :code:`gkfs.io` +``` + +#### gkfs_shell + +The `gkfs_shell` fixture returns an instance of +the [`harness.gkfs.ShellClient`](https://storage.bsc.es/gitlab/hpc/gekkofs/blob/master/tests/harness/gkfs.py#L346) +class. The `ShellClient` class allows users to execute shell commands and +scripts in their own separate processes directly from Python test code. The +harness again relies on the `sh` module to spawn the processes with an +appropriately patched environment so that they can communicate successfully with +a running daemon. + +##### Single commands + +To simplify usage and reduce coding overhead, the class provides a +special `__getattr__()` method that tries to transform any method called on +a `ShellClient` instance to a `bash -c` command. Thus, in the following code: + +```python +def test_cp(gkfs_daemon, gkfs_shell, file_factory): + """Copy a file into gkfs using the shell""" + + lf01 = file_factory.create(file01, size=4.0, unit='MB') + + # lf01.pathname: '${TWD}/tmp/file01' + # gkfs_daemon.mountdir: '${TWD}/mnt/' + cmd = gkfs_shell.cp(lf01.pathname, gkfs_daemon.mountdir) + assert cmd.exit_code == 0 + assert cmd.stdout.decode() == '' + assert cmd.stderr.decode() == '' +``` + +`gkfs_shell.cp(lf01.pathname, gkfs_daemon.mountdir)` transforms into: + +```console +$ LD_LIBRARY_PATH= LD_PRELOAD= bash -c 'cp ${TWD}/tmp/file01 ${TWD}/mnt/' +``` + +where `XXX` and `YYY` are respectively substituted by the appropriate library +paths and `libgfs_intercept.so` library required for the test. + +Similarly to the `sh` module, the raw output generated by a shell command +executed in isolation (e.g. `cp`, `stat`, `md5sum`, etc.) can be accessed using +the `stdout` and `stderr` properties as exemplified in the code above. Note, +however, that the output for such commands can be accessed more conveniently as +a Python object using the `parsed_stdout` property, provided that an +appropriate `CommandParser` has been implemented for the command: + +```ipython +In[1]: gkfs_shell.stat("--terse /tmp/foobar") +statOutput(filename='/tmp/foobar', size=4000000, blocks=0, raw_mode='81b4', uid=1000, gid=1000, device='0', inode=10075480095715127217, hard_links=1, major='0', minor='0', last_access=0, last_modification=0, last_status_change=0, creation=0, transfer_size=524288) +``` + +If no command parser is available for the command, a `NotImplementedError` will +be raised by the `harness.cmd.CommandParser` class. + +##### Complex scripts + +For convenience, the `ShellClient` class also provides a `script()` method that +allows to execute complex chains of commands: + +```python +def test_shell_if_e(gkfs_daemon, gkfs_shell, file_factory): + """ + Copy a file into gkfs using the shell and check that it exists using `if [[ -e ]]`. + """ + + logger.debug("creating input file") + lf01 = file_factory.create(file01, size=4.0, unit='MB') + + logger.debug("copying into gkfs") + cmd = gkfs_shell.cp(lf01.pathname, gkfs_daemon.mountdir) + assert cmd.exit_code == 0 + + logger.debug("checking if file exists") + cmd = gkfs_shell.script( + f""" + expected_pathname={gkfs_daemon.mountdir / file01} + if [[ -e ${{expected_pathname}} ]]; + then + exit 0 + fi + exit 1 + """) + + assert cmd.exit_code == 0 +``` + +Thanks to Python 3's [f-strings](https://realpython.com/python-f-strings/), it +is really simple to expand Python expressions in the shell script, though care +must be taken to escape braces when referencing shell variables in the script +code (e.g. `${{expected_pathname}}` in the excerpts above). + +Note that by default, `gkfs_shell.script()` will patch the `LD_LIBRARY_PATH` +and `LD_PRELOAD` of the shell executing the script so that it gets intercepted +by the GekkoFS client library. If such intrusive interception is not desired, ( +because a test may require a single command of a complex script to be +intercepted), it is possible to disable it by setting the `intercept_shell` +argument to `False` as shown in the code below: + +```python +def test_shell_stat_script(gkfs_daemon, gkfs_shell, file_factory): + """ + Copy a file into gkfs using the shell and check that `stat succeeds` + """ + + logger.debug("creating input file") + lf01 = file_factory.create(file01, size=4.0, unit='MB') + + logger.debug("copying into gkfs") + cmd = gkfs_shell.cp(lf01.pathname, gkfs_daemon.mountdir) + assert cmd.exit_code == 0 + + logger.debug("checking metadata") + cmd = gkfs_shell.script( + f""" + expected_pathname={gkfs_daemon.mountdir / file01} + {gkfs_shell.patched_environ} stat ${{expected_pathname}} + exit $? + """, + intercept_shell=False) + + assert cmd.exit_code == 0 +``` + +As demonstrated by the code above, the `gkfs_shell` class provides +the `patched_environ` property to make it simple to provide the appropriate +environment variables to any commands that should be intercepted. Thus, the +above line: + +```python +f""" + expected_pathname={gkfs_daemon.mountdir / file01} + {gkfs_shell.patched_environ} stat ${{expected_pathname}} + exit $? +""", +``` + +will be transformed by the harness into: + +```shell +LD_LIBRARY_PATH="/opt/gekkofs/build/tests:/opt/gekkofs/prefix/lib" \ +LD_PRELOAD="/opt/gekkofs/build/src/client/libgkfs_intercept.so" \ +LIBGKFS_HOSTS_FILE="/tmp/pytest-of-user/pytest-45/test_stat_script0/gkfs_hosts.txt" \ +LIBGKFS_LOG="all" LIBGKFS_LOG_OUTPUT="/tmp/pytest-of-user/pytest-45/test_stat_script0/logs/gkfs_client.log" \ + stat ${expected_pathname} +``` + +```{eval-rst} +.. warning:: + Please note that, unlike for single shell commands, the :code:`parsed_stdout` + property is not available for complex shell scripts run + using :code:`gkfs_shell.script()`. The reason is that it is not possible to + provide a generic parser for all possible scripts. The raw output generated + by a script, if any, can however be accessed using the :code:`stdout` and + :code:`stderr` properties. +``` + +Please note that, unlike for single shell commands, the `parsed_stdout` +property is not available for complex shell scripts run +using `gkfs_shell.script()`. The reason is that it is not possible to provide a +generic parser for all possible scripts. The raw output generated by a script, +if any, can however be accessed using the `stdout` and `stderr` properties. + +## Unit tests + +TBD \ No newline at end of file diff --git a/docs/sphinx/images/gekkofs-small.png b/docs/sphinx/images/gekkofs-small.png new file mode 100644 index 0000000000000000000000000000000000000000..69047fcf09b5ed9eb5363a4f5dd052c4beb964f5 Binary files /dev/null and b/docs/sphinx/images/gekkofs-small.png differ diff --git a/docs/sphinx/images/gekkofwd-small.png b/docs/sphinx/images/gekkofwd-small.png new file mode 100644 index 0000000000000000000000000000000000000000..e85bbda7d31ea8468ebf44dcea4397e35e05f7eb Binary files /dev/null and b/docs/sphinx/images/gekkofwd-small.png differ diff --git a/docs/sphinx/images/gkfs_architecture.png b/docs/sphinx/images/gkfs_architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..824179c703ebfb17e17e800c94fb68db798ce80f Binary files /dev/null and b/docs/sphinx/images/gkfs_architecture.png differ diff --git a/docs/sphinx/index.rst b/docs/sphinx/index.rst new file mode 100644 index 0000000000000000000000000000000000000000..283ca401af0a9c4dee84df565582ae6b4e599e6b --- /dev/null +++ b/docs/sphinx/index.rst @@ -0,0 +1,66 @@ +.. GekkoFS documentation master file, created by + sphinx-quickstart on Sun Mar 3 23:49:53 2019. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +====================================================== +GekkoFS: A burst-buffer file system for HPC +====================================================== + +GekkoFS is a highly scalable user-level distributed file system for HPC +clusters. GekkoFS is capable of aggregating the local +I/O capacity and performance of compute nodes to produce a high-performance +storage space for applications. With GekkoFS, HPC applications and simulations +can run in isolation from each other with regards to I/O, which reduces +interferences and improves performance. Furthermore, GekkoFS has been designed +with configurability in mind, and allows users to fine tune several of the +default POSIX file system semantics (e.g. support for symbolic links or strict +bookkeeping of file access timestamps) that, even if useful, might not be +required by their applications and hence negatively impact their I/O +performance. + +Contrary to general purpose file systems, a GekkoFS file system is ephemeral in +nature. That is, the lifetime of a GekkoFS file system instance is linked to +the duration of the execution of its GekkoFS server processes, which are +typically spawned when an HPC job starts and shut down when it ends. This +means that users must copy any files that need to be persisted beyond the +lifetime of the job from GekkoFS to a more permanent file system such as Lustre or +GPFS. Also, because GekkoFS is implemented at user-level, the file system is +only visible to applications using one of the GekkoFS client libraries. +A consequence of this is that traditional file system tools (ls, cd, etc.) +installed by system administrators will not be aware of files in a GekkoFS file +system. To solve this, GekkoFS provides a client interception library that can +be preloaded before calling these tools. + +--------------------------- +Architecture +--------------------------- + +.. image:: images/gkfs_architecture.png + +.. toctree:: + :maxdepth: 2 + :caption: User guide + + users/building + users/running + users/forwarding + users/scripts + +.. toctree:: + :maxdepth: 2 + :caption: Contributing + +.. toctree:: + :maxdepth: 2 + :caption: Developer documentation + + devs/testing + api/reference + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/docs/sphinx/users/building.rst b/docs/sphinx/users/building.rst new file mode 100644 index 0000000000000000000000000000000000000000..3fa5f1684916559776c4edae64869b5fe172ac04 --- /dev/null +++ b/docs/sphinx/users/building.rst @@ -0,0 +1,216 @@ +.. _building_gekkofs: + +Installing GekkoFS +****************** + +This section describes how to build and install GekkoFS. + +.. _gkfs_base_dependencies: + +Base dependencies +================= + +GekkoFS relies on CMake for building and testing the daemon and client +binaries, which are written in C++17. As such, GekkoFS has the following +base dependencies for its build system: + +- A compiler with C++17 support: + + - `GCC `_ version 8.0 and newer + - `Clang `_ 5.0 and newer. + +- `CMake `_ version 3.6 or newer to build the main code. + + - Version 3.11 or newer to build the tests. + +GekkoFS also needs :code:`curl` and :code:`git` to be installed in +your system if you plan on using the dependency management scripts provided +with the code, as well as `Python 3.6 `_ +or later to run some of the tests. + +Additionally, since some GekkoFS dependencies rely on `GNU Automake +`_ as +their build system, it is also recommended to install the latest version of +:code:`autoconf`, :code:`automake`, and :code:`libtool`. + +This set of dependencies are typically available in the system repositories. +We show below how to install them for some of the most popular Linux +distributions: + +Debian/Ubuntu +------------- + +.. code-block:: console + + # Base dependencies + $ apt install git curl cmake autoconf automake libtool libconfig-dev + + # Testing support: + $ apt install python3-dev python3 python3-venv + +CentOS/Red Hat +-------------- + +.. code-block:: console + + # Base dependencies: + $ yum install gcc-c++ git curl cmake autoconf automake libtool libconfig + + # Testing support: + $ yum install python38-devel + +.. _gkfs_dependencies: + +Software dependencies +===================== + +GekkoFS requires several software packages to be available in your system in +order to function properly. We list them here for informational purposes. + +.. warning:: + + Though it is possible to install all dependencies manually, GekkoFS + provides some dependency management scripts that automate and + greatly simplify this task. We strongly suggest that you take read through + the :ref:`step-by-step installation` section + before attempting a manual install. + +- `RocksDB `_ version 6.2.2 or newer and its dependencies: + + - `bzip2 `_ version 1.0.6 or newer. + + - `zstd `_ version 1.3.2 or newer. + + - `lz4 `_ version 1.8.0 or newer. + + - `snappy `_ version 1.1.7 or newer. + + +- `Margo `_ version 0.6.3 and its dependencies: + + - `Argobots `_ version 1.0rc1. + - `Mercury `_ version 2.0.0. + + - `libfabric `_ and/or `bmi `_. + + +- `syscall_intercept `_ (commit f7cebb7 or newer) and its dependencies: + + - `capstone `_ version 4.0.1 or newer. + +- `Date `_ (commit e7e1482 or newer). + +.. important:: + + Mercury may need additional plugins depending on the fabric technology used in the cluster. For instance, + OmniPath requires the `opa-psm2 `_ library to work. + +Optional dependencies +--------------------- + +Additionally, some GekkoFS optional execution modes have additional +dependencies: + +- `AGIOS `_ (commit c26a654 or + newer) to enable the :code:`GekkoFWD` I/O forwarding mode. + +.. _step_by_step_installation: + +Step-by-step installation +========================= + +1. Make sure that the :ref:`GekkoFS base dependencies ` + are available on your machine. + +2. Clone GekkoFS: + + .. code-block:: console + + $ git clone --recurse-submodules https://storage.bsc.es/gitlab/hpc/gekkofs.git + + + (Optional) If you checked out the sources using :code:`git` without the + :code`--recursive` option, you need to execute the following command from + the root of the source directory: + + .. code-block:: console + + $ git submodule update --init + +3. Set up the necessary environment variables where the compiled GekkoFS + :ref:`software dependencies ` should be installed at. + Throughout this example we assume dependencies will live in the + + :code:`/home/foo/gekkofs_deps/install` directory): + + .. code-block:: console + + $ export GKFS_INSTALL_PATH=/home/foo/gekkofs_deps/install + $ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${GKFS_INSTALL_PATH}/lib:${GKFS_INSTALL_PATH}/lib64 + +4. Download all the required dependencies using :code:`dl_dep.sh`, one of + GekkoFS' dependency management scripts. We will use the + :code:`/home/foo/gekkofs_deps/git` directory to keep all the source files: + + .. code-block:: console + + $ gekkofs/scripts/dl_dep.sh /home/foo/gekkofs_deps/git + +.. important:: + + The :code:`/home/foo/gekkofs_deps/git` directory containing the source code + for all our downloaded dependencies can be safely removed once installation + is complete. + +5. Build and install the dependencies into :code:`GKFS_INSTALL_PATH` using + :code:`compile_dep.sh`, the second GekkoFS' dependency management script: + + .. code-block:: console + + $ gekkofs/scripts/compile_dep.sh /home/foo/gekkofs_deps/git /home/foo/gekkofs_deps/install + +6. Now let's configure the GekkoFS build by setting the appropriate options. + GekkoFS makes use of the CMake build system and requires that you do an + out-of-source build. In order to do that, you must create a new build + directory and run the :code:`cmake` command from it: + + .. code-block:: console + + # Create the build directory: + $ cd gekkofs + $ mkdir build && cd build + $ cmake \ + -DCMAKE_BUILD_TYPE:STRING=Release \ + -DCMAKE_PREFIX_PATH:STRING=/home/foo/gekkofs_deps/install \ + -DCMAKE_INSTALL_PREFIX:STRING=/home/foo/gekkofs_deps/install \ + -DGKFS_BUILD_TESTS:BOOL=ON \ + .. + + + For this example, we set the :code:`CMAKE_BUILD_TYPE` variable to + :code:`Release` to let CMake know that we need an optimized build. + It is very important to make CMake aware of where GekkoFS dependencies are + installed, which is why we set :code:`CMAKE_PREFIX_PATH` to + :code:`${GKFS_INSTALL_PATH}`. We also set :code:`CMAKE_INSTALL_PREFIX` to + the same directory, because we want the GekkoFS binaries to be + installed in the same location. Finally, we also enable the compilation of + GekkoFS tests (which are not enabled by default) by setting + :code:`GKFS_BUILD_TESTS`. + +.. attention:: + If you prefer a more interactive approach, it is also possible to use + :code:`ccmake` or :code:`cmake-gui` to configure the package. + +7. We are finally ready to build, test and install GekkoFS: + + .. code-block:: console + + $ make -j8 + $ make test + $ make install + +After following this guide, GekkoFS binaries should now be available in the +appropriate subdirectories of :code:`GKFS_INSTALL_PATH`: + +- GekkoFS daemon (server): :code:`${GKFS_INSTALL_PATH}/bin/gkfs_daemon` +- GekkoFS client interception library: :code:`${GKFS_INSTALL_PATH}/lib/libgkfs_intercept.so` diff --git a/docs/sphinx/users/dependencies.rst b/docs/sphinx/users/dependencies.rst new file mode 100644 index 0000000000000000000000000000000000000000..0d3732410f1b133cc755047a1df1ad2601fa2f0b --- /dev/null +++ b/docs/sphinx/users/dependencies.rst @@ -0,0 +1,46 @@ +==================== +GekkoFS dependencies +==================== + +-------- +Required +-------- + +- A compiler with C++17 support, i.e. `GCC `_ version 8.0 and newer or `Clang `_ 5.0 and newer. + +- `CMake `_ version 3.6 or newer to build the main code. Version 3.11 or newer to build the tests. + +- `RocksDB `_ version 6.2.2 or newer and its dependencies: + + - `bzip2 `_ version 1.0.6 or newer. + + - `zstd `_ version 1.3.2 or newer. + + - `lz4 `_ version 1.8.0 or newer. + + - `snappy `_ version 1.1.7 or newer. + + +- `Margo `_ version 0.6.3 and its dependencies: + + - `Argobots `_ version 1.0rc1. + - `Mercury `_ version 2.0.0. + + - `libfabric `_ and/or `bmi `_. + + +- `syscall_intercept `_ (commit f7cebb7 or newer) and its dependencies: + - `capstone `_ version 4.0.1 or newer. + +- `date `_ (commit e7e1482 or newer). + +.. important:: + + Mercury may need additional plugins depending on the fabric technology used in the cluster. For instance, + OmniPath requires the `opa-psm2 `_ library to work. + +-------- +Optional +-------- + +- `agios `_ (commit c26a654 or newer) to enable GekkoFWD mode. diff --git a/docs/sphinx/users/forwarding.rst b/docs/sphinx/users/forwarding.rst new file mode 100644 index 0000000000000000000000000000000000000000..db0afc7b22e587f629bcacadfde6de82c62f7e80 --- /dev/null +++ b/docs/sphinx/users/forwarding.rst @@ -0,0 +1,84 @@ +Forwarding mode +=============== + +GekkoFS is capable of working as an I/O forwarding layer by means of +the :code:`GekkoFWD` plugin. This plugin enables an I/O forwarding mode that +allows GekkoFS data servers to function as intermediate I/O nodes between +compute nodes and parallel file system servers. + +In GekkoFS, data operations are typically distributed across all nodes. Once an +operation is intercepted, the client forwards it to the responsible server, +determined by hashing the file's path. Furthermore, GekkoFS uses the node-local +FS to store data and metadata and no request scheduling in the daemon. + +.. image:: /images/gekkofs-small.png + +In GekkoFWD, conversely, the data is forwarded to a single server +determined by an allocation policy, without breaking the file into chunks, +as the PFS already has its striping mechanism. In the daemon, requests can be +scheduled before being issued to the file system. GekkoFWD relies on the shared +PFS for storage, instead of a local store available at the compute nodes. + +.. image:: /images/gekkofwd-small.png + +Enabling GekkoFWD +------------------ + +To enable the I/O forwarding mode of GekkoFS, the +:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when +configuring ` the build: + +.. code-block:: console + + $ cmake -DENABLE_FORWARDING:BOOL=ON + +I/O Scheduling +-------------- + +Because a forwarding layer is transparent to applications, it usually is the +target of I/O optimizations such as file-level request scheduling. For that +reason, we have integrated the `AGIOS `_ scheduling library into GekkoFWD. AGIOS +has several schedulers available, and it also allows us to prototype new +scheduling solutions. On the GekkoFWD daemon running at the I/O nodes, once a +request is received, it is sent to AGIOS to determine when it should be +processed. Once scheduled, it is then dispatched and executed. + +.. note:: + I/O scheduling in :code:`GekkoFWD` is an optional feature. You can enable + it during compilation time with the :code:`GKFWD_ENABLE_AGIOS` CMake option. + +You need to make sure to have the :code:`agios.conf` file in the :code:`/tmp` +folder of each node (or in another pre-defined path)· This file contains +the AGIOS configurations, such as the I/O scheduler it should use. An example +of this file can be found in the +`GekkoFS repository `_. + +Running +------- + +Before running any applications with GekkoFWD, you need to have a forwarding +map file that can be dynamically updated according to an allocation policy. It +should contain a hostname of a compute node and the ID of an I/O node, to which +all intercepted requests should be forwarded. Here is an example with four +compute nodes and two I/O nodes: + +.. code-block:: console + + $ cat gkfwd.map + node-1 0 + node-2 0 + node-3 1 + node-4 1 + +In this scenario, :code:`node-1` and :code:`node-2` will forward all its I/O +requests to the first I/O node (i.e. node 0), while :code:`node-3` and +:code:`node-4` will forward all its I/O requests to the second I/O node (i.e. +node 1). + +An environment variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided +to allow users to identify the map file on each client. Since +:code:`GekkoFWD` doesn't yet have a mechanism to distribute this +information across all participating nodes, this file should be stored in a +shared infrastructure such as the default PFS. This guarantees that all +:code:`GekkoFWD` clients can see the same mapping. diff --git a/docs/sphinx/users/running.rst b/docs/sphinx/users/running.rst new file mode 100644 index 0000000000000000000000000000000000000000..8bf8ef498bf497e38242fa08ee621321c9d1934a --- /dev/null +++ b/docs/sphinx/users/running.rst @@ -0,0 +1,2 @@ +Running GekkoFS +================ diff --git a/docs/sphinx/users/scripts.rst b/docs/sphinx/users/scripts.rst new file mode 100644 index 0000000000000000000000000000000000000000..5b196d3d5a5300a81d2106a9f575f57b70b9af95 --- /dev/null +++ b/docs/sphinx/users/scripts.rst @@ -0,0 +1,187 @@ +Scripts +======= + +The GekkoFS package includes several scripts that automate/simplify common +tasks. This section describes such scripts, which can be classified as follows: + +.. contents:: + :local: + :depth: 1 + :backlinks: none + +Dependency Management Scripts +----------------------------- + +GekkoFS provides two scripts for managing dependencies which can be found in +the :code:`PROJECT_ROOT/scripts` directory: :code:`dl_dep.sh` and +:code:`compile_dep.sh`. As their names suggest, the :code:`dl_dep.sh` script +is in charge of downloading any dependencies required by GekkoFS to build or +run correctly, while :code:`compile_dep.sh` is responsible for installing them +once downloaded. + +Since dependencies may change or need to be configured differently depending +on the specifics of the particular GekkoFS build, both scripts rely on +:code:`configuration profiles` which define a set of related software +packages which should be downloaded and installed for a specific GekkoFS +version and/or configuration. To illustrate this, let's take a look at the +contents of the :code:`default` profile for GekkoFS version :code:`0.8.0`: + +.. code-block:: console + + $ dl_dep.sh -l default:0.8.0 + Configuration profiles for 'default:0.8.0': + + * default:0.8.0 (/home/user/gekkofs/source/scripts/profiles/0.8.0/default.specs) + + All dependencies + + bzip2: 1.0.6 + zstd: 1.3.2 + lz4: 1.8.0 + snappy: 1.1.7 + capstone: 4.0.1 + bmi: 6ea0b78fce1b964e45102828cdd05df7040a94c8 + libfabric: HEAD@v1.8.1 + libfabric%experimental: HEAD@v1.9.1 + libfabric%verbs: HEAD@v1.7.2 + mercury: 41caa143a07ed179a3149cac4af0dc7aa3f946fd + argobots: 1.0rc1 + margo: v0.6.3 + rocksdb: 6.2.2 + rocksdb%experimental: 6.11.4 + syscall_intercept: f7cebb7b7e7512a19b78a31ce236ad6ca22636dd + date: e7e1482087f58913b80a20b04d5c58d9d6d90155 + psm2: 11.2.86 + agios: c26a6544200f823ebb8f890dd94e653d148bf226@development + + + +Downloading dependencies +######################## + +Dependencies defined in a profile can be downloaded using the :code:`dl_dep.sh` +script and the :code:`-p` option. As shown below, profile names follow the +:code:`PROFILE_NAME[:VERSION_TAG]` naming convention, where :code:`PROFILE_NAME` +serves to uniquely identify a particular configuration (e.g. for a specific +supercomputer) followed by an optional :code:`VERSION_TAG`. + +.. code-block:: console + + $ ./dl_dep.sh -p default:0.8.0 /home/user/gfks/deps + Destination path is set to "/tmp/foo" + Profile name: default + Profile version: 0.8.0 + ------------------------------------ + Cloned 'https://github.com/francielizanon/agios.git' to 'agios' with commit '[c26a6544200f823ebb8f890dd94e653d148bf226]' and flags '--branch=development' + Downloaded 'https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz' to 'argobots' + Downloaded 'https://github.com/google/snappy/archive/1.1.7.tar.gz' to 'snappy' + Cloned 'https://github.com/radix-io/bmi/' to 'bmi' with commit '[6ea0b78fce1b964e45102828cdd05df7040a94c8]' and flags '' + Downloaded 'https://github.com/lz4/lz4/archive/v1.8.0.tar.gz' to 'lz4' + Cloned 'https://github.com/pmem/syscall_intercept.git' to 'syscall_intercept' with commit '[f7cebb7b7e7512a19b78a31ce236ad6ca22636dd]' and flags '' + Checking patch include/libsyscall_intercept_hook_point.h... + Checking patch src/intercept.c... + Hunk #2 succeeded at 700 (offset 31 lines). + Hunk #3 succeeded at 730 (offset 31 lines). + Checking patch test/test_clone_thread_preload.c... + Applied patch include/libsyscall_intercept_hook_point.h cleanly. + Applied patch src/intercept.c cleanly. + Applied patch test/test_clone_thread_preload.c cleanly. + Downloaded 'https://github.com/facebook/zstd/archive/v1.3.2.tar.gz' to 'zstd' + Downloaded 'https://github.com/intel/opa-psm2/archive/PSM2_11.2.86.tar.gz' to 'psm2' + Downloaded 'https://github.com/aquynh/capstone/archive/4.0.1.tar.gz' to 'capstone' + Cloned 'https://github.com/HowardHinnant/date.git' to 'date' with commit '[e7e1482087f58913b80a20b04d5c58d9d6d90155]' and flags '' + Cloned 'https://xgitlab.cels.anl.gov/sds/margo.git' to 'margo' with commit '[v0.6.3]' and flags '' + Downloaded 'https://github.com/facebook/rocksdb/archive/v6.2.2.tar.gz' to 'rocksdb' + Downloaded 'https://github.com/facebook/rocksdb/archive/v6.11.4.tar.gz' to 'rocksdb%experimental' + Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' + Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%verbs' with commit '[HEAD]' and flags '--branch=v1.7.2' + Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric' with commit '[HEAD]' and flags '--branch=v1.8.1' + Cloned 'https://github.com/ofiwg/libfabric.git' to 'libfabric%experimental' with commit '[HEAD]' and flags '--branch=v1.9.1' + Downloaded 'https://sourceforge.net/projects/bzip2/files/bzip2-1.0.6.tar.gz' to 'bzip2' + Done + +It is also possible to download a specific dependency with the :code:`-d` +option. In this case, dependency names follow the +:code:`DEPENDENCY_NAME[@PROFILE_NAME[:VERSION_TAG]]` naming convention. + +.. code-block:: console + + $ ./dl_dep.sh -d mercury@default:0.8.0 /home/user/gfks/deps + Destination path is set to "/tmp/foo" + Profile name: default + Profile version: 0.8.0 + ------------------------------------ + Cloned 'https://github.com/mercury-hpc/mercury' to 'mercury' with commit '[41caa143a07ed179a3149cac4af0dc7aa3f946fd]' and flags '--recurse-submodules' + Done + +.. warning:: + + Note that :code:`PROFILE_NAME` and :code:`VERSION_TAG` can be optional + in most script invocations. If :code:`PROFILE_NAME` is left unspecified, + the scripts will assume that the :code:`default` profile was selected. + Similarly, if a :code:`VERSION_NAME` is not provided, the scripts will + assume that the :code:`latest` version should be used. + +Installing dependencies +######################## + +Once dependencies in a configuration profile have been downloaded to a +certain directory (e.g. :code:`/home/user/gkfs/deps`), the +:code:`compile_dep.sh` script can be used to install them. + +.. code-block:: console + + $ ./compile_dep.sh -p default:0.8.0 /home/user/gkfs/deps /home/user/gkfs/install -j8 + CORES = 8 (default) + Sources download path = /tmp/foo + Installation path = /tmp/bar + Profile name: default + Profile version: 0.8.0 + ------------------------------------ + + + ######## Installing: bzip2 ############################### + ... + + ######## Installing: zstd ############################### + ... + + ######## Installing: lz4 ############################### + ... + + ######## Installing: snappy ############################### + ... + + ######## Installing: capstone ############################### + ... + + ######## Installing: bmi ############################### + ... + + ######## Installing: libfabric ############################### + ... + + ######## Installing: mercury ############################### + ... + + ######## Installing: argobots ############################### + ... + + ######## Installing: margo ############################### + ... + + ######## Installing: rocksdb ############################### + ... + + ######## Installing: syscall_intercept ############################### + ... + + ######## Installing: date ############################### + ... + + ######## Installing: psm2 ############################### + ... + + ######## Installing: agios ############################### + ... + Done