diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3ca0f9d1a6b81bf4eb3609ebcca529b4046c64d0..0f935d3af69c2bb70d2f987cbf2026b50020c9e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,15 +6,17 @@ stages: variables: DEPS_SRC_PATH: "${CI_PROJECT_DIR}/deps/src" - DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/build" - DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/build/gkfs_deps_commit" - BUILD_PATH: "${CI_PROJECT_DIR}/build" + DEPS_INSTALL_PATH: "${CI_PROJECT_DIR}/deps/install" + DEPS_COMMIT: "${CI_PROJECT_DIR}/deps/install/gkfs_deps_commit" + BUILD_PATH: "${CI_PROJECT_DIR}/gkfs/build" + INSTALL_PATH: "${CI_PROJECT_DIR}/gkfs/install" TESTS_BUILD_PATH: "${CI_PROJECT_DIR}/test/build" LOG_PATH: "${CI_PROJECT_DIR}/logs" + LD_LIBRARY_PATH: "${CI_PROJECT_DIR}/deps/install/lib;${CI_PROJECT_DIR}/deps/install/lib64" # Configuration variables - ADAFS_LOG_LEVEL: "100" - ADAFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" - ADAFS_PRELOAD_LOG_PATH: "${CI_PROJECT_DIR}/logs/preload.log" + GKFS_LOG_LEVEL: "100" + GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log" + GKFS_PRELOAD_LOG_PATH: "${CI_PROJECT_DIR}/logs/preload.log" image: gekkofs/gekkofs:build_env @@ -48,11 +50,12 @@ compile GekkoFS: -Wdeprecate -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=${DEPS_INSTALL_PATH} - .. - - make -j$(nproc) + -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} + ${CI_PROJECT_DIR} + - make -j$(nproc) install artifacts: paths: - - ${BUILD_PATH} + - ${INSTALL_PATH} compile tests: stage: build @@ -70,9 +73,9 @@ test wr: stage: test script: - mkdir -p "${LOG_PATH}" - - ${BUILD_PATH}/bin/adafs_daemon --mount /tmp/mountdir --root /tmp/adafs_root & + - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 - - LD_PRELOAD=${BUILD_PATH}/lib/libadafs_preload_client.so ${TESTS_BUILD_PATH}/gkfs_test_wr + - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_wr artifacts: paths: - "${LOG_PATH}" @@ -81,9 +84,9 @@ test directories: stage: test script: - mkdir -p "${LOG_PATH}" - - ${BUILD_PATH}/bin/adafs_daemon --mount /tmp/mountdir --root /tmp/adafs_root & + - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 - - LD_PRELOAD=${BUILD_PATH}/lib/libadafs_preload_client.so ${TESTS_BUILD_PATH}/gkfs_test_dir + - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_dir artifacts: paths: - "${LOG_PATH}" @@ -92,9 +95,9 @@ test truncate: stage: test script: - mkdir -p "${LOG_PATH}" - - ${BUILD_PATH}/bin/adafs_daemon --mount /tmp/mountdir --root /tmp/adafs_root & + - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 - - LD_PRELOAD=${BUILD_PATH}/lib/libadafs_preload_client.so ${TESTS_BUILD_PATH}/gkfs_test_truncate + - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_truncate artifacts: paths: - "${LOG_PATH}" @@ -103,9 +106,9 @@ test path resolution: stage: test script: - mkdir -p "${LOG_PATH}" - - ${BUILD_PATH}/bin/adafs_daemon --mount /tmp/mountdir --root /tmp/adafs_root & + - ${INSTALL_PATH}/bin/gkfs_daemon --mount /tmp/mountdir --root /tmp/root & - sleep 4 - - LD_PRELOAD=${BUILD_PATH}/lib/libadafs_preload_client.so ${TESTS_BUILD_PATH}/gkfs_test_path_resolution + - LD_PRELOAD=${INSTALL_PATH}/lib/libgkfs_intercept.so ${TESTS_BUILD_PATH}/gkfs_test_path_resolution artifacts: paths: - "${LOG_PATH}" diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index c886e93d61771f4bc23c5f9541d13c9f1657b765..0000000000000000000000000000000000000000 --- a/CHANGELOG +++ /dev/null @@ -1,5 +0,0 @@ -============ -v 0.3.1 -============ -Marc Vef: - 2018-03-04 Read-write process improved, adafs version to 0.3.1 with improved Filemap diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..7cfa785d04f67c3797be4476cd8112917f6aa38c --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,17 @@ +# Changelog +All notable changes to GekkoFS project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [0.4.0] - 2019-04-18 +First GekkoFS pubblic release + +This version provides a client library that uses GLibC I/O function interception. + +## [0.3.1] - 2018-03-04 +### Changed +- Read-write process improved. @Marc vef +- Improved Filemap. @Marc Vef diff --git a/CMake/FindSnappy.cmake b/CMake/FindSnappy.cmake index 025f0a5e46b96c01d5ab9b4870e45f1e54d9954d..6ff0ecd434f972b2e3bdce9afd19a91c3c000810 100644 --- a/CMake/FindSnappy.cmake +++ b/CMake/FindSnappy.cmake @@ -1,11 +1,9 @@ find_library(Snappy_LIBRARY NAMES snappy - HINTS ${ADAFS_DEPS_INSTALL} ) find_path(Snappy_INCLUDE_DIR NAMES snappy.h - HINTS ${ADAFS_DEPS_INSTALL} ) set(Snappy_LIBRARIES ${Snappy_LIBRARY}) diff --git a/CMakeLists.txt b/CMakeLists.txt index 86bd47ac62ceb915d51464f69e20b1d6c3b81a3e..52420d4b570c48883fd372161ad933ed6fbbedf9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,10 @@ cmake_minimum_required(VERSION 3.6) -project(GekkoFS) + +project( + GekkoFS + VERSION 0.4.0 +) + if(NOT CMAKE_COMPILER_IS_GNUCC) message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported") @@ -11,6 +16,8 @@ endif() set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED ON) +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build: Debug Release Memcheck" FORCE) @@ -25,11 +32,44 @@ set(CMAKE_CXX_FLAGS_MEMCHECK "${WARNINGS_FLAGS} -g -O0 -fsanitize=address -fno-o set(CMAKE_CXX_FLAGS_MAINTAINER "${WARNINGS_FLAGS} -g -O0 -pg -no-pie") mark_as_advanced(CMAKE_CXX_FLAGS_MAINTAINER) +# Project version +set(GIT_VERSION_FOUND FALSE) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/.git) + find_package(Git) + if(Git_FOUND) + execute_process(COMMAND /bin/bash -c "${GIT_EXECUTABLE} describe --tags --match='v*.*.*' --long --dirty" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + RESULT_VARIABLE GIT_RET + OUTPUT_VARIABLE GIT_VERSION + ERROR_VARIABLE GIT_ERR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + if("${GIT_RET}" STREQUAL "0") + string(REGEX MATCH + "^v([0-9]+)\.([0-9]+)\.([0-9]+)-([0-9]+)-(.*)$" + DROP ${GIT_VERSION}) + set(PROJECT_VERSION_MAJOR ${CMAKE_MATCH_1}) + set(PROJECT_VERSION_MINOR ${CMAKE_MATCH_2}) + set(PROJECT_VERSION_PATCH ${CMAKE_MATCH_3}) + set(GKFS_COMMIT_GAP ${CMAKE_MATCH_4}) + set(GKFS_COMMIT_HASH ${CMAKE_MATCH_5}) + set(PROJECT_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}-snapshot+${GKFS_COMMIT_GAP}-${GKFS_COMMIT_HASH}") + set(GIT_VERSION_FOUND TRUE) + else() + message(WARNING "Failed to run `git describe`: [${GIT_RET}] '${GIT_ERR}'") + endif() + endif() +endif() +message(STATUS "Project version: ${PROJECT_VERSION}") +set(GKFS_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) +set(GKFS_VERSION_MINOR ${PROJECT_VERSION_MINOR}) +set(GKFS_VERSION_PATCH ${PROJECT_VERSION_PATCH}) +set(GKFS_VERSION_STRING ${PROJECT_VERSION}) +configure_file(include/version.hpp.in include/version.hpp) + set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) set(CMAKE_EXPORT_COMPILE_COMMANDS 0) -set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) -set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) # Rocksdb dependencies find_package(LZ4 REQUIRED) @@ -99,14 +139,14 @@ add_library(spdlog INTERFACE) # we cannot use target_include_directories with CMake < 3.11 set_target_properties(spdlog PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/extern" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/external" ) add_library(fmt INTERFACE) # we cannot use target_include_directories with CMake < 3.11 set_target_properties(fmt PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/extern/spdlog" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/external/spdlog" ) set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") @@ -114,8 +154,11 @@ set(INCLUDE_DIR "${CMAKE_SOURCE_DIR}/include") # define include directories that are relevant for all targets include_directories( ${INCLUDE_DIR} + ${CMAKE_BINARY_DIR}/include ) +include(GNUInstallDirs) + # Global components add_subdirectory(src/global) # Daemon diff --git a/LICENSE b/LICENSE index 501da8c5c7bf25f7f6f7387239edfb9e69ad4815..3462287774c3af1ecbbee9c16de447142f6a03ca 100644 --- a/LICENSE +++ b/LICENSE @@ -1,22 +1,25 @@ +Valid-License-Identifier: MIT +License-Text: + MIT License -Copyright (c) 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany Copyright (c) 2018-2019, Barcelona Supercomputing Center (BSC), Spain +Copyright (c) 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. +The above copyright notice and this permission notice (including the next +paragraph) shall be included in all copies or substantial portions of the +Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF +OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index db34cfbc7c94cbe6fa720c52a392bf2c18067cc5..f2fd9adc1df9d8ca580d18a9bc1589e90b9ba23f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# ADA-FS +# GekkoFS This is a file system. # Dependencies @@ -36,7 +36,7 @@ This is a file system. # Usage -## Clone and compile direct ADA-FS dependencies +## Clone and compile direct GekkoFS dependencies - Go to the `scripts` folder and first clone all dependencies projects. You can choose the according na_plugin (execute the script for help): @@ -46,7 +46,7 @@ usage: dl_dep.sh [-h] [-n ] [-c ] source_path -This script gets all ADA-FS dependency sources (excluding the fs itself) +This script gets all GekkoFS dependency sources (excluding the fs itself) positional arguments: source_path path where the dependency downloads are put @@ -69,7 +69,7 @@ usage: compile_dep.sh [-h] [-n ] [-c ] [-j ] source_path install_path -This script compiles all ADA-FS dependencies (excluding the fs itself) +This script compiles all GekkoFS dependencies (excluding the fs itself) positional arguments: source_path path to the cloned dependencies path from clone_dep.sh @@ -89,7 +89,7 @@ optional arguments: defaults to number of available cores ``` -## Compile ADA-FS +## Compile GekkoFS You need to decide what Mercury NA plugin you want to use. The following NA plugins are available, although only BMI is considered stable at the moment. - `ofi+tcp` for using the libfabric plugin with TCP - `ofi+verbs` for using the libfabric plugin with Infiniband verbs (not threadsafe. Do not use.) @@ -97,29 +97,27 @@ You need to decide what Mercury NA plugin you want to use. The following NA plug - `cci+verbs` for using the cci plugin with Infiniband verbs - `bmi+tcp` for using the bmi plugin with the tcp protocol -In addition you can add a specific directory where all dependencies are located, i.e., headers and libraries. This can -be done by using `-DADAFS_DEPS_INSTALL=`. If the variable is not set this path points to `/usr/local`. ```bash mkdir build && cd build -cmake -DADAFS_DEPS_INSTALL= -DCMAKE_BUILD_TYPE={Release, Debug}} .. +cmake -DCMAKE_BUILD_TYPE=Release .. make ``` -## Run ADA-FS +## Run GekkoFS -First on each node a daemon has to be started. This can be done in two ways using the `adafs_daemon` binary directly or +First on each node a daemon has to be started. This can be done in two ways using the `gkfs_daemon` binary directly or the corresponding startup and shutdown scripts. The latter is recommended for cluster usage. It requires pssh (or parallel-ssh) with python2. ### Start and shut down daemon directly -`./build/bin/adafs_daemon -r -m --hosts ` +`./build/bin/gkfs_daemon -r -m --hosts ` Shut it down by gracefully killing the process. ### Startup and shutdown scripts -The scripts are located in `scripts/{startup_adafs.py, shutdown_adafs.py}`. Use the -h argument for their usage. +The scripts are located in `scripts/{startup_gkfs.py, shutdown_gkfs.py}`. Use the -h argument for their usage. ## Miscellaneous @@ -131,7 +129,14 @@ an MPI application use the `{mpirun, mpiexec} -x` argument. ### Logging To enable logging the following environment variables are used: -ADAFS_PRELOAD_LOG_PATH="" to set the path to the logging file of the client library. -ADAFS_DAEMON_LOG_PATH="" to set the path to the logging file of the daemon. -ADAFS_LOG_LEVEL={off,critical,err,warn,info,debug,trace} to set the trace level verbosity. -Numbers from 0-6 may also be used where as 0 is off and 6 represents trace. \ No newline at end of file +GKFS_PRELOAD_LOG_PATH="" to set the path to the logging file of the client library. +GKFS_DAEMON_LOG_PATH="" to set the path to the logging file of the daemon. +GKFS_LOG_LEVEL={off,critical,err,warn,info,debug,trace} to set the trace level verbosity. +Numbers from 0-6 may also be used where as 0 is off and 6 represents trace. + + +### Acknoledgment + +This software was partially supported by the EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + +This software was partially supported by the ADA-FS project under the SPPEXA project funded by the DFG. \ No newline at end of file diff --git a/docker/build_env.docker b/docker/build_env.docker index 8b01c013529389fef97e4bf493b0592813991cd2..3ba53b726b93227c2cf5e6dc61ce264fbeb82afe 100644 --- a/docker/build_env.docker +++ b/docker/build_env.docker @@ -1,11 +1,11 @@ FROM centos:7 as builder -ENV ADAFS_PATH /opt/adafs +ENV GKFS_PATH /opt/gkfs -ENV SCRIPTS_PATH ${ADAFS_PATH}/scripts -ENV DEPS_SRC_PATH ${ADAFS_PATH}/deps_src -ENV INSTALL_PATH ${ADAFS_PATH}/build_deps +ENV SCRIPTS_PATH ${GKFS_PATH}/scripts +ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src +ENV INSTALL_PATH ${GKFS_PATH}/build_deps # Enable epel and scl repository @@ -57,8 +57,8 @@ RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH FROM centos:7 -ENV ADAFS_PATH /opt/adafs -ENV INSTALL_PATH ${ADAFS_PATH}/build_deps +ENV GKFS_PATH /opt/gkfs +ENV INSTALL_PATH ${GKFS_PATH}/build_deps # Enable epel and scl repository diff --git a/include/extern/spdlog/async.h b/external/spdlog/async.h similarity index 100% rename from include/extern/spdlog/async.h rename to external/spdlog/async.h diff --git a/include/extern/spdlog/async_logger.h b/external/spdlog/async_logger.h similarity index 100% rename from include/extern/spdlog/async_logger.h rename to external/spdlog/async_logger.h diff --git a/include/extern/spdlog/common.h b/external/spdlog/common.h similarity index 100% rename from include/extern/spdlog/common.h rename to external/spdlog/common.h diff --git a/include/extern/spdlog/details/async_logger_impl.h b/external/spdlog/details/async_logger_impl.h similarity index 100% rename from include/extern/spdlog/details/async_logger_impl.h rename to external/spdlog/details/async_logger_impl.h diff --git a/include/extern/spdlog/details/circular_q.h b/external/spdlog/details/circular_q.h similarity index 100% rename from include/extern/spdlog/details/circular_q.h rename to external/spdlog/details/circular_q.h diff --git a/include/extern/spdlog/details/console_globals.h b/external/spdlog/details/console_globals.h similarity index 100% rename from include/extern/spdlog/details/console_globals.h rename to external/spdlog/details/console_globals.h diff --git a/include/extern/spdlog/details/file_helper.h b/external/spdlog/details/file_helper.h similarity index 100% rename from include/extern/spdlog/details/file_helper.h rename to external/spdlog/details/file_helper.h diff --git a/include/extern/spdlog/details/fmt_helper.h b/external/spdlog/details/fmt_helper.h similarity index 100% rename from include/extern/spdlog/details/fmt_helper.h rename to external/spdlog/details/fmt_helper.h diff --git a/include/extern/spdlog/details/log_msg.h b/external/spdlog/details/log_msg.h similarity index 100% rename from include/extern/spdlog/details/log_msg.h rename to external/spdlog/details/log_msg.h diff --git a/include/extern/spdlog/details/logger_impl.h b/external/spdlog/details/logger_impl.h similarity index 100% rename from include/extern/spdlog/details/logger_impl.h rename to external/spdlog/details/logger_impl.h diff --git a/include/extern/spdlog/details/mpmc_blocking_q.h b/external/spdlog/details/mpmc_blocking_q.h similarity index 100% rename from include/extern/spdlog/details/mpmc_blocking_q.h rename to external/spdlog/details/mpmc_blocking_q.h diff --git a/include/extern/spdlog/details/null_mutex.h b/external/spdlog/details/null_mutex.h similarity index 100% rename from include/extern/spdlog/details/null_mutex.h rename to external/spdlog/details/null_mutex.h diff --git a/include/extern/spdlog/details/os.h b/external/spdlog/details/os.h similarity index 100% rename from include/extern/spdlog/details/os.h rename to external/spdlog/details/os.h diff --git a/include/extern/spdlog/details/pattern_formatter.h b/external/spdlog/details/pattern_formatter.h similarity index 100% rename from include/extern/spdlog/details/pattern_formatter.h rename to external/spdlog/details/pattern_formatter.h diff --git a/include/extern/spdlog/details/periodic_worker.h b/external/spdlog/details/periodic_worker.h similarity index 100% rename from include/extern/spdlog/details/periodic_worker.h rename to external/spdlog/details/periodic_worker.h diff --git a/include/extern/spdlog/details/registry.h b/external/spdlog/details/registry.h similarity index 100% rename from include/extern/spdlog/details/registry.h rename to external/spdlog/details/registry.h diff --git a/include/extern/spdlog/details/thread_pool.h b/external/spdlog/details/thread_pool.h similarity index 100% rename from include/extern/spdlog/details/thread_pool.h rename to external/spdlog/details/thread_pool.h diff --git a/include/extern/spdlog/fmt/bin_to_hex.h b/external/spdlog/fmt/bin_to_hex.h similarity index 100% rename from include/extern/spdlog/fmt/bin_to_hex.h rename to external/spdlog/fmt/bin_to_hex.h diff --git a/include/extern/spdlog/fmt/bundled/LICENSE.rst b/external/spdlog/fmt/bundled/LICENSE.rst similarity index 100% rename from include/extern/spdlog/fmt/bundled/LICENSE.rst rename to external/spdlog/fmt/bundled/LICENSE.rst diff --git a/include/extern/spdlog/fmt/bundled/colors.h b/external/spdlog/fmt/bundled/colors.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/colors.h rename to external/spdlog/fmt/bundled/colors.h diff --git a/include/extern/spdlog/fmt/bundled/core.h b/external/spdlog/fmt/bundled/core.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/core.h rename to external/spdlog/fmt/bundled/core.h diff --git a/include/extern/spdlog/fmt/bundled/format-inl.h b/external/spdlog/fmt/bundled/format-inl.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/format-inl.h rename to external/spdlog/fmt/bundled/format-inl.h diff --git a/include/extern/spdlog/fmt/bundled/format.h b/external/spdlog/fmt/bundled/format.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/format.h rename to external/spdlog/fmt/bundled/format.h diff --git a/include/extern/spdlog/fmt/bundled/ostream.h b/external/spdlog/fmt/bundled/ostream.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/ostream.h rename to external/spdlog/fmt/bundled/ostream.h diff --git a/include/extern/spdlog/fmt/bundled/posix.h b/external/spdlog/fmt/bundled/posix.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/posix.h rename to external/spdlog/fmt/bundled/posix.h diff --git a/include/extern/spdlog/fmt/bundled/printf.h b/external/spdlog/fmt/bundled/printf.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/printf.h rename to external/spdlog/fmt/bundled/printf.h diff --git a/include/extern/spdlog/fmt/bundled/ranges.h b/external/spdlog/fmt/bundled/ranges.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/ranges.h rename to external/spdlog/fmt/bundled/ranges.h diff --git a/include/extern/spdlog/fmt/bundled/time.h b/external/spdlog/fmt/bundled/time.h similarity index 100% rename from include/extern/spdlog/fmt/bundled/time.h rename to external/spdlog/fmt/bundled/time.h diff --git a/include/extern/spdlog/fmt/fmt.h b/external/spdlog/fmt/fmt.h similarity index 100% rename from include/extern/spdlog/fmt/fmt.h rename to external/spdlog/fmt/fmt.h diff --git a/include/extern/spdlog/fmt/ostr.h b/external/spdlog/fmt/ostr.h similarity index 100% rename from include/extern/spdlog/fmt/ostr.h rename to external/spdlog/fmt/ostr.h diff --git a/include/extern/spdlog/formatter.h b/external/spdlog/formatter.h similarity index 100% rename from include/extern/spdlog/formatter.h rename to external/spdlog/formatter.h diff --git a/include/extern/spdlog/logger.h b/external/spdlog/logger.h similarity index 100% rename from include/extern/spdlog/logger.h rename to external/spdlog/logger.h diff --git a/include/extern/spdlog/sinks/android_sink.h b/external/spdlog/sinks/android_sink.h similarity index 100% rename from include/extern/spdlog/sinks/android_sink.h rename to external/spdlog/sinks/android_sink.h diff --git a/include/extern/spdlog/sinks/ansicolor_sink.h b/external/spdlog/sinks/ansicolor_sink.h similarity index 100% rename from include/extern/spdlog/sinks/ansicolor_sink.h rename to external/spdlog/sinks/ansicolor_sink.h diff --git a/include/extern/spdlog/sinks/base_sink.h b/external/spdlog/sinks/base_sink.h similarity index 100% rename from include/extern/spdlog/sinks/base_sink.h rename to external/spdlog/sinks/base_sink.h diff --git a/include/extern/spdlog/sinks/basic_file_sink.h b/external/spdlog/sinks/basic_file_sink.h similarity index 100% rename from include/extern/spdlog/sinks/basic_file_sink.h rename to external/spdlog/sinks/basic_file_sink.h diff --git a/include/extern/spdlog/sinks/daily_file_sink.h b/external/spdlog/sinks/daily_file_sink.h similarity index 100% rename from include/extern/spdlog/sinks/daily_file_sink.h rename to external/spdlog/sinks/daily_file_sink.h diff --git a/include/extern/spdlog/sinks/dist_sink.h b/external/spdlog/sinks/dist_sink.h similarity index 100% rename from include/extern/spdlog/sinks/dist_sink.h rename to external/spdlog/sinks/dist_sink.h diff --git a/include/extern/spdlog/sinks/msvc_sink.h b/external/spdlog/sinks/msvc_sink.h similarity index 100% rename from include/extern/spdlog/sinks/msvc_sink.h rename to external/spdlog/sinks/msvc_sink.h diff --git a/include/extern/spdlog/sinks/null_sink.h b/external/spdlog/sinks/null_sink.h similarity index 100% rename from include/extern/spdlog/sinks/null_sink.h rename to external/spdlog/sinks/null_sink.h diff --git a/include/extern/spdlog/sinks/ostream_sink.h b/external/spdlog/sinks/ostream_sink.h similarity index 100% rename from include/extern/spdlog/sinks/ostream_sink.h rename to external/spdlog/sinks/ostream_sink.h diff --git a/include/extern/spdlog/sinks/rotating_file_sink.h b/external/spdlog/sinks/rotating_file_sink.h similarity index 100% rename from include/extern/spdlog/sinks/rotating_file_sink.h rename to external/spdlog/sinks/rotating_file_sink.h diff --git a/include/extern/spdlog/sinks/sink.h b/external/spdlog/sinks/sink.h similarity index 100% rename from include/extern/spdlog/sinks/sink.h rename to external/spdlog/sinks/sink.h diff --git a/include/extern/spdlog/sinks/stdout_color_sinks.h b/external/spdlog/sinks/stdout_color_sinks.h similarity index 100% rename from include/extern/spdlog/sinks/stdout_color_sinks.h rename to external/spdlog/sinks/stdout_color_sinks.h diff --git a/include/extern/spdlog/sinks/stdout_sinks.h b/external/spdlog/sinks/stdout_sinks.h similarity index 100% rename from include/extern/spdlog/sinks/stdout_sinks.h rename to external/spdlog/sinks/stdout_sinks.h diff --git a/include/extern/spdlog/sinks/syslog_sink.h b/external/spdlog/sinks/syslog_sink.h similarity index 100% rename from include/extern/spdlog/sinks/syslog_sink.h rename to external/spdlog/sinks/syslog_sink.h diff --git a/include/extern/spdlog/sinks/wincolor_sink.h b/external/spdlog/sinks/wincolor_sink.h similarity index 100% rename from include/extern/spdlog/sinks/wincolor_sink.h rename to external/spdlog/sinks/wincolor_sink.h diff --git a/include/extern/spdlog/spdlog.h b/external/spdlog/spdlog.h similarity index 100% rename from include/extern/spdlog/spdlog.h rename to external/spdlog/spdlog.h diff --git a/include/extern/spdlog/tweakme.h b/external/spdlog/tweakme.h similarity index 100% rename from include/extern/spdlog/tweakme.h rename to external/spdlog/tweakme.h diff --git a/include/extern/spdlog/version.h b/external/spdlog/version.h similarity index 100% rename from include/extern/spdlog/version.h rename to external/spdlog/version.h diff --git a/include/client/adafs_functions.hpp b/include/client/adafs_functions.hpp index 58f641d13bd36816ceb859e65d32252c11329ab2..0e628511f613728d54d761c04140ad3469abff68 100644 --- a/include/client/adafs_functions.hpp +++ b/include/client/adafs_functions.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_ADAFS_FUNCTIONS_HPP #define IFS_ADAFS_FUNCTIONS_HPP diff --git a/include/client/intcp_functions.hpp b/include/client/intcp_functions.hpp index 942566bbec4b291b4ebdb32781d1107d5eef926d..83d74ef92c43b9423ad3c565446d505c70312c50 100644 --- a/include/client/intcp_functions.hpp +++ b/include/client/intcp_functions.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_INTCP_FUNCTIONS_HPP #define IFS_INTCP_FUNCTIONS_HPP diff --git a/include/client/open_dir.hpp b/include/client/open_dir.hpp index 83ddf0c1ff5182f6bb6f792525eefc49512ff4d8..5dfda56c6378b72fbd73fabd26d6faac2f66b6ca 100644 --- a/include/client/open_dir.hpp +++ b/include/client/open_dir.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_OPEN_DIR_HPP #define IFS_OPEN_DIR_HPP diff --git a/include/client/open_file_map.hpp b/include/client/open_file_map.hpp index a954694ce1581d25a5daeaeac1818d746acc4f4b..738ba49ece6959a36eb362a497c52204c9551a0d 100644 --- a/include/client/open_file_map.hpp +++ b/include/client/open_file_map.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_OPEN_FILE_MAP_HPP #define IFS_OPEN_FILE_MAP_HPP diff --git a/include/client/passthrough.hpp b/include/client/passthrough.hpp index 9014318591f4a7087147e86a81b9003068cb6d94..2153dbddbf8d162cd8a4385ece65fc78caa67b6e 100644 --- a/include/client/passthrough.hpp +++ b/include/client/passthrough.hpp @@ -1,124 +1,131 @@ -// -// Created by evie on 11/15/17. -// +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ #ifndef IFS_PASSTHROUGH_HPP #define IFS_PASSTHROUGH_HPP +#define LIBC_FUNC_NAME(FNAME) libc_##FNAME + #define LIBC_FUNC(FNAME, ...) \ ((reinterpret_cast(libc_##FNAME))(__VA_ARGS__)) -// function pointer for preloading -extern void* libc; - -extern void* libc_open; -extern void* libc_openat; - -extern void* libc_fopen; -extern void* libc_fopen64; -extern void* libc_fread; -extern void* libc_fwrite; -extern void* libc_fclose; -extern void* libc_clearerr; -extern void* libc_feof; -extern void* libc_ferror; -extern void* libc_fileno; -extern void* libc_fflush; -extern void* libc_fpurge; -extern void* libc___fpurge; - -extern void* libc_setbuf; -extern void* libc_setbuffer; -extern void* libc_setlinebuf; -extern void* libc_setvbuf; - -extern void* libc_putc; -extern void* libc_fputc; -extern void* libc_fputs; -extern void* libc_getc; -extern void* libc_fgetc; -extern void* libc_fgets; -extern void* libc_ungetc; - -extern void* libc_fseek; - -extern void* libc_mkdir; -extern void* libc_mkdirat; -extern void* libc_unlink; -extern void* libc_unlinkat; -extern void* libc_rmdir; - -extern void* libc_close; - -extern void* libc_access; -extern void* libc_faccessat; - -extern void* libc_stat; -extern void* libc_fstat; -extern void* libc_lstat; -extern void* libc___xstat; -extern void* libc___xstat64; -extern void* libc___fxstat; -extern void* libc___fxstat64; -extern void* libc___fxstatat; -extern void* libc___fxstatat64; -extern void* libc___lxstat; -extern void* libc___lxstat64; - -extern void* libc_statfs; -extern void* libc_fstatfs; -extern void* libc_statvfs; -extern void* libc_fstatvfs; - -extern void* libc_write; -extern void* libc_pwrite; -extern void* libc_pwrite64; -extern void* libc_writev; - -extern void* libc_read; -extern void* libc_pread; -extern void* libc_pread64; -extern void* libc_readv; - -extern void* libc_lseek; -extern void* libc_lseek64; -extern void* libc_fsync; -extern void* libc_fdatasync; - -extern void* libc_truncate; -extern void* libc_ftruncate; - -extern void* libc_fcntl; - -extern void* libc_dup; -extern void* libc_dup2; -extern void* libc_dup3; - -extern void* libc_dirfd; -extern void* libc_opendir; -extern void* libc_fdopendir; -extern void* libc_readdir; -extern void* libc_closedir; - -extern void* libc_chmod; -extern void* libc_fchmod; -extern void* libc_fchmodat; - -extern void* libc_chdir; -extern void* libc_fchdir; - -extern void* libc_getcwd; -extern void* libc_get_current_dir_name; - -extern void* libc_link; -extern void* libc_linkat; -extern void* libc_symlinkat; - -extern void* libc_readlinkat; - -extern void* libc_realpath; + +extern void* LIBC_FUNC_NAME(open); +extern void* LIBC_FUNC_NAME(openat); + +extern void* LIBC_FUNC_NAME(fopen); +extern void* LIBC_FUNC_NAME(fopen64); +extern void* LIBC_FUNC_NAME(fread); +extern void* LIBC_FUNC_NAME(fwrite); +extern void* LIBC_FUNC_NAME(fclose); +extern void* LIBC_FUNC_NAME(clearerr); +extern void* LIBC_FUNC_NAME(feof); +extern void* LIBC_FUNC_NAME(ferror); +extern void* LIBC_FUNC_NAME(fileno); +extern void* LIBC_FUNC_NAME(fflush); +extern void* LIBC_FUNC_NAME(__fpurge); + +extern void* LIBC_FUNC_NAME(setbuf); +extern void* LIBC_FUNC_NAME(setbuffer); +extern void* LIBC_FUNC_NAME(setlinebuf); +extern void* LIBC_FUNC_NAME(setvbuf); + +extern void* LIBC_FUNC_NAME(putc); +extern void* LIBC_FUNC_NAME(fputc); +extern void* LIBC_FUNC_NAME(fputs); +extern void* LIBC_FUNC_NAME(getc); +extern void* LIBC_FUNC_NAME(fgetc); +extern void* LIBC_FUNC_NAME(fgets); +extern void* LIBC_FUNC_NAME(ungetc); + +extern void* LIBC_FUNC_NAME(fseek); + +extern void* LIBC_FUNC_NAME(mkdir); +extern void* LIBC_FUNC_NAME(mkdirat); +extern void* LIBC_FUNC_NAME(unlink); +extern void* LIBC_FUNC_NAME(unlinkat); +extern void* LIBC_FUNC_NAME(rmdir); + +extern void* LIBC_FUNC_NAME(close); + +extern void* LIBC_FUNC_NAME(access); +extern void* LIBC_FUNC_NAME(faccessat); + +extern void* LIBC_FUNC_NAME(__xstat); +extern void* LIBC_FUNC_NAME(__xstat64); +extern void* LIBC_FUNC_NAME(__fxstat); +extern void* LIBC_FUNC_NAME(__fxstat64); +extern void* LIBC_FUNC_NAME(__fxstatat); +extern void* LIBC_FUNC_NAME(__fxstatat64); +extern void* LIBC_FUNC_NAME(__lxstat); +extern void* LIBC_FUNC_NAME(__lxstat64); + +extern void* LIBC_FUNC_NAME(statfs); +extern void* LIBC_FUNC_NAME(fstatfs); +extern void* LIBC_FUNC_NAME(statvfs); +extern void* LIBC_FUNC_NAME(fstatvfs); + +extern void* LIBC_FUNC_NAME(write); +extern void* LIBC_FUNC_NAME(pwrite); +extern void* LIBC_FUNC_NAME(pwrite64); +extern void* LIBC_FUNC_NAME(writev); + +extern void* LIBC_FUNC_NAME(read); +extern void* LIBC_FUNC_NAME(pread); +extern void* LIBC_FUNC_NAME(pread64); +extern void* LIBC_FUNC_NAME(readv); + +extern void* LIBC_FUNC_NAME(lseek); +extern void* LIBC_FUNC_NAME(lseek64); +extern void* LIBC_FUNC_NAME(fsync); +extern void* LIBC_FUNC_NAME(fdatasync); + +extern void* LIBC_FUNC_NAME(truncate); +extern void* LIBC_FUNC_NAME(ftruncate); + +extern void* LIBC_FUNC_NAME(fcntl); + +extern void* LIBC_FUNC_NAME(dup); +extern void* LIBC_FUNC_NAME(dup2); +extern void* LIBC_FUNC_NAME(dup3); + +extern void* LIBC_FUNC_NAME(dirfd); +extern void* LIBC_FUNC_NAME(opendir); +extern void* LIBC_FUNC_NAME(fdopendir); +extern void* LIBC_FUNC_NAME(readdir); +extern void* LIBC_FUNC_NAME(closedir); + +extern void* LIBC_FUNC_NAME(chmod); +extern void* LIBC_FUNC_NAME(fchmod); +extern void* LIBC_FUNC_NAME(fchmodat); + +extern void* LIBC_FUNC_NAME(chdir); +extern void* LIBC_FUNC_NAME(fchdir); + +extern void* LIBC_FUNC_NAME(getcwd); +extern void* LIBC_FUNC_NAME(get_current_dir_name); + +extern void* LIBC_FUNC_NAME(link); +extern void* LIBC_FUNC_NAME(linkat); +extern void* LIBC_FUNC_NAME(symlinkat); + +extern void* LIBC_FUNC_NAME(readlinkat); + +extern void* LIBC_FUNC_NAME(realpath); + void init_passthrough_if_needed(); + #endif //IFS_PASSTHROUGH_HPP diff --git a/include/client/preload.hpp b/include/client/preload.hpp index 936787f2d0b6c777a26a32e71a6a4e0d6e805da1..e1e12dcf5711f5e45e896c47ead65924b6bdc42e 100644 --- a/include/client/preload.hpp +++ b/include/client/preload.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IOINTERCEPT_PRELOAD_HPP #define IOINTERCEPT_PRELOAD_HPP diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 754cc4ba07c3f05110a257388b66ef6ba1aa5521..9002f1db1e027b38bf3b9e0eb849ecd87c739434 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_PRELOAD_CTX_HPP #define IFS_PRELOAD_CTX_HPP diff --git a/include/client/preload_util.hpp b/include/client/preload_util.hpp index 7da6043d9568a7f97fd1873d234ced089ec24057..02643e475ab63abfb19fcad318b0a0bb59eb400e 100644 --- a/include/client/preload_util.hpp +++ b/include/client/preload_util.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_PRELOAD_UTIL_HPP #define IFS_PRELOAD_UTIL_HPP diff --git a/include/client/resolve.hpp b/include/client/resolve.hpp index 2996052b5ce155f2698e4b7810373dc231ac0fd2..4e16d281cfd9e2e2ecae08b41782567fa7a4034b 100644 --- a/include/client/resolve.hpp +++ b/include/client/resolve.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include bool resolve_path (const std::string& path, std::string& resolved, bool resolve_last_link = true); diff --git a/include/client/rpc/ld_rpc_data_ws.hpp b/include/client/rpc/ld_rpc_data_ws.hpp index d06266ca2ce66d91051ba6f8ebdbf270476e6bdf..317124691ccba6304e6b54e84976b31a7d7935ea 100644 --- a/include/client/rpc/ld_rpc_data_ws.hpp +++ b/include/client/rpc/ld_rpc_data_ws.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_PRELOAD_C_DATA_WS_HPP #define IFS_PRELOAD_C_DATA_WS_HPP diff --git a/include/client/rpc/ld_rpc_management.hpp b/include/client/rpc/ld_rpc_management.hpp index 09cdbfb2d829a61f69cf58964a3034f266573be5..cd0e206063dc7f89fe13831b3098364949f546aa 100644 --- a/include/client/rpc/ld_rpc_management.hpp +++ b/include/client/rpc/ld_rpc_management.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_MARGO_RPC_MANAGMENT_HPP #define IFS_MARGO_RPC_MANAGMENT_HPP diff --git a/include/client/rpc/ld_rpc_metadentry.hpp b/include/client/rpc/ld_rpc_metadentry.hpp index d361ccafd02f3eef5e965cb79f59a5ba0081fe9a..1abb330869da873fd4bfe4f96a1fb396ef830305 100644 --- a/include/client/rpc/ld_rpc_metadentry.hpp +++ b/include/client/rpc/ld_rpc_metadentry.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_PRELOAD_C_METADENTRY_HPP #define IFS_PRELOAD_C_METADENTRY_HPP diff --git a/include/daemon/backend/data/chunk_storage.hpp b/include/daemon/backend/data/chunk_storage.hpp index 1b249bcc083afb5f5c8d8ec8fac832e29c1891fb..3533aaf0d6ae11bd17005706431e50b20eb17e58 100644 --- a/include/daemon/backend/data/chunk_storage.hpp +++ b/include/daemon/backend/data/chunk_storage.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_CHUNK_STORAGE_HPP #define IFS_CHUNK_STORAGE_HPP diff --git a/include/daemon/backend/exceptions.hpp b/include/daemon/backend/exceptions.hpp index ec048c56456d46d5ca4edc6266e2a90a90bae58d..01151dda5ccad8d738747124796c3d422b86a715 100644 --- a/include/daemon/backend/exceptions.hpp +++ b/include/daemon/backend/exceptions.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_DB_EXCEPTIONS_HPP #define IFS_DB_EXCEPTIONS_HPP diff --git a/include/daemon/backend/metadata/db.hpp b/include/daemon/backend/metadata/db.hpp index e7a8d01591968c7e4b246b0aa3522befa51a7595..6d66a3b5d8f737a3cdb229c231bb036ae13d737e 100644 --- a/include/daemon/backend/metadata/db.hpp +++ b/include/daemon/backend/metadata/db.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_METADATA_DB_HPP #define IFS_METADATA_DB_HPP diff --git a/include/daemon/backend/metadata/merge.hpp b/include/daemon/backend/metadata/merge.hpp index f72d818ecba8d5d6c877fcacd9a59e0a60ebea18..bd7ee8cd02e2914310212d7620d63df5bc97b289 100644 --- a/include/daemon/backend/metadata/merge.hpp +++ b/include/daemon/backend/metadata/merge.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef DB_MERGE_HPP #define DB_MERGE_HPP diff --git a/include/daemon/classes/fs_data.hpp b/include/daemon/classes/fs_data.hpp index 0539e8eb7778b05ec8bd8abcd98f83cdd03ba36a..69e9ae901f67f1a58f944ee1a5a77d5a1d869efc 100644 --- a/include/daemon/classes/fs_data.hpp +++ b/include/daemon/classes/fs_data.hpp @@ -1,8 +1,21 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef LFS_FS_DATA_H #define LFS_FS_DATA_H -#include +#include /* Forward declarations */ class MetadataDB; diff --git a/include/daemon/classes/rpc_data.hpp b/include/daemon/classes/rpc_data.hpp index eeb3e4af5c90c6f82d5aa509f42666c119db4b8b..b3d16115e56921d369fba23dd3e87c52515d2155 100644 --- a/include/daemon/classes/rpc_data.hpp +++ b/include/daemon/classes/rpc_data.hpp @@ -1,8 +1,21 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef LFS_RPC_DATA_HPP #define LFS_RPC_DATA_HPP -#include +#include class RPCData { diff --git a/include/daemon/handler/rpc_defs.hpp b/include/daemon/handler/rpc_defs.hpp index daf63b7c96bfc823372e020136e864c1e6e386eb..e7a252deed4b0399832710d7dd3f1ee8a973707e 100644 --- a/include/daemon/handler/rpc_defs.hpp +++ b/include/daemon/handler/rpc_defs.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef LFS_RPC_DEFS_HPP #define LFS_RPC_DEFS_HPP diff --git a/include/daemon/adafs_daemon.hpp b/include/daemon/main.hpp similarity index 58% rename from include/daemon/adafs_daemon.hpp rename to include/daemon/main.hpp index c8e68734658a17d39c0d4f9f3f0cddc885f6450d..98b47bde5722045232e62da7db92358293129bd1 100644 --- a/include/daemon/adafs_daemon.hpp +++ b/include/daemon/main.hpp @@ -1,12 +1,23 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany -#ifndef IFS_ADAFS_DAEMON_HPP -#define IFS_ADAFS_DAEMON_HPP + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +#ifndef GKFS_DAEMON_MAIN_HPP +#define GKFS_DAEMON_MAIN_HPP // std libs #include #include -// adafs config #include #include // margo @@ -15,7 +26,6 @@ extern "C" { #include #include } -// adafs #include #include @@ -37,4 +47,4 @@ bool deregister_daemon_proc(); void populate_lookup_file(); void destroy_lookup_file(); -#endif //IFS_ADAFS_DAEMON_HPP +#endif // GKFS_DAEMON_MAIN_HPP diff --git a/include/daemon/adafs_ops/metadentry.hpp b/include/daemon/ops/metadentry.hpp similarity index 62% rename from include/daemon/adafs_ops/metadentry.hpp rename to include/daemon/ops/metadentry.hpp index fe9ab7af5fba0e5b5f68788d5a9484eb2b4b0b75..39acc8f582ee0ba6aacfb00f232f6abf9c90e03f 100644 --- a/include/daemon/adafs_ops/metadentry.hpp +++ b/include/daemon/ops/metadentry.hpp @@ -1,8 +1,21 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_METADENTRY_HPP #define IFS_METADENTRY_HPP -#include +#include #include int create_node(const std::string& path, const uid_t uid, const gid_t gid, mode_t mode); diff --git a/include/global/chunk_calc_util.hpp b/include/global/chunk_calc_util.hpp index 13553d814cc600784dad315468384a71beab0f74..bade836fb6ad8e022410752e010bdb5d3b6b97eb 100644 --- a/include/global/chunk_calc_util.hpp +++ b/include/global/chunk_calc_util.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_CHNK_CALC_UTIL_HPP #define IFS_CHNK_CALC_UTIL_HPP diff --git a/include/global/configure.hpp b/include/global/configure.hpp index 06105e5cc60dcb3bf8ca6b2d6934c09d55e3c4fe..e667f5a84470cecef9c2e845ebfa9895ab546213 100644 --- a/include/global/configure.hpp +++ b/include/global/configure.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + /** * Attention: This whole configfile is not in a final form! This will eventually be a plaintext config file. */ @@ -47,11 +60,11 @@ #define RPC_DIRENTS_BUFF_SIZE (8 * 1024 * 1024) // 8 mega // environment prefixes -#define ENV_PREFIX "ADAFS_" +#define ENV_PREFIX "GKFS_" // Log -#define DEFAULT_PRELOAD_LOG_PATH "/tmp/adafs_preload.log" -#define DEFAULT_DAEMON_LOG_PATH "/tmp/adafs_daemon.log" +#define DEFAULT_PRELOAD_LOG_PATH "/tmp/gkfs_preload.log" +#define DEFAULT_DAEMON_LOG_PATH "/tmp/gkfs_daemon.log" #define DEFAULT_PRELOAD_LOG_LEVEL 4 // info #define DEFAULT_DAEMON_LOG_LEVEL 4 // info diff --git a/include/global/global_defs.hpp b/include/global/global_defs.hpp index 5adc1c16b87ec698647a91104b49d3147a17d66b..8b8b1c4b452cc52e6288f6e5c8c3bf70428fb831 100644 --- a/include/global/global_defs.hpp +++ b/include/global/global_defs.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_GLOBAL_DEFS_HPP #define IFS_GLOBAL_DEFS_HPP diff --git a/include/global/global_func.hpp b/include/global/global_func.hpp index e41f4c2cb2922be092c53d0d9ab2ccd2b7d6a48d..265446bb4629f3d5da3de14fea67e947d4c18246 100644 --- a/include/global/global_func.hpp +++ b/include/global/global_func.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_GLOBAL_FUNC_HPP #define IFS_GLOBAL_FUNC_HPP diff --git a/include/global/log_util.hpp b/include/global/log_util.hpp index bf9d57ed7937e2bfe642a40e0507ad89b4f3c465..898b3e430d4c486b520b5a477c1ce42e3368e602 100644 --- a/include/global/log_util.hpp +++ b/include/global/log_util.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_LOG_UITIL_HPP #define IFS_LOG_UITIL_HPP diff --git a/include/global/metadata.hpp b/include/global/metadata.hpp index 62209009a977eebcd56c6b5907d233f766387f1e..e24c60783185d835e44514cd9e9934d68254f322 100644 --- a/include/global/metadata.hpp +++ b/include/global/metadata.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef FS_METADATA_H #define FS_METADATA_H #pragma once diff --git a/include/global/path_util.hpp b/include/global/path_util.hpp index 0aa2f3b65fde87abcaf506a48479c754302b02de..d93fc1aff5d265ddd9fd9938f9f324b9549b5522 100644 --- a/include/global/path_util.hpp +++ b/include/global/path_util.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_PATH_UTIL_HPP #define IFS_PATH_UTIL_HPP diff --git a/include/global/rpc/distributor.hpp b/include/global/rpc/distributor.hpp index 0ba1b917d7eb646b887574f429a54602a67e785b..6ca59f651c7363f5a4baca4400ce8323a1e0f378 100644 --- a/include/global/rpc/distributor.hpp +++ b/include/global/rpc/distributor.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_RPC_DISTRIBUTOR_HPP #define IFS_RPC_DISTRIBUTOR_HPP diff --git a/include/global/rpc/rpc_types.hpp b/include/global/rpc/rpc_types.hpp index 066bd7fec3a939812277741e08ca2bb41cdac584..d9bafe78ffbda0cf12635ec7ba0b67e2a2e406b1 100644 --- a/include/global/rpc/rpc_types.hpp +++ b/include/global/rpc/rpc_types.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef LFS_RPC_TYPES_HPP #define LFS_RPC_TYPES_HPP diff --git a/include/global/rpc/rpc_utils.hpp b/include/global/rpc/rpc_utils.hpp index 47238dd9c12628b5bdb617724f4026e738524aeb..ee3b6b7b433dde8b2fc1d83e70fb6bb3a2941e4f 100644 --- a/include/global/rpc/rpc_utils.hpp +++ b/include/global/rpc/rpc_utils.hpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #ifndef IFS_RPC_UTILS_HPP #define IFS_RPC_UTILS_HPP diff --git a/include/version.hpp.in b/include/version.hpp.in new file mode 100644 index 0000000000000000000000000000000000000000..ebeb161c168ac3e77500ef51d97ae9a1edda37b2 --- /dev/null +++ b/include/version.hpp.in @@ -0,0 +1,9 @@ +#ifndef __GKFS_VERSION_HPP +#define __GKFS_VERSION_HPP + +#define GKFS_VERSION_MAJOR @GKFS_VERSION_MAJOR@ +#define GKFS_VERSION_MINOR @GKFS_VERSION_MINOR@ +#define GKFS_VERSION_PATCH @GKFS_VERSION_PATCH@ +#define GKFS_VERSION_STRING "@GKFS_VERSION_STRING@" + +#endif /* __GKFS_VERSION_HPP */ diff --git a/scripts/compile_dep.sh b/scripts/compile_dep.sh index e682d94373b2d0ea93619db67612eebb97da366f..21c657d05756dc237322db0701bdcd779972eaa3 100755 --- a/scripts/compile_dep.sh +++ b/scripts/compile_dep.sh @@ -11,7 +11,7 @@ help_msg() { usage_short echo " -This script compiles all ADA-FS dependencies (excluding the fs itself) +This script compiles all GekkoFS dependencies (excluding the fs itself) positional arguments: source_path path to the cloned dependencies path from clone_dep.sh @@ -141,8 +141,6 @@ else echo "No cluster configuration set." fi -#LOG=/tmp/adafs_install.log -#echo "" &> $LOG USE_BMI="-DNA_USE_BMI:BOOL=OFF" USE_CCI="-DNA_USE_CCI:BOOL=OFF" USE_OFI="-DNA_USE_OFI:BOOL=OFF" @@ -215,9 +213,9 @@ if [ "$NA_LAYER" == "cci" ] || [ "$NA_LAYER" == "all" ]; then prepare_build_dir ${CURR} cd ${CURR} # patch hanging issue - echo "########## ADA-FS injection: Applying cci hanging patch" + echo "########## Patch injection: Applying cci hanging patch" git apply ${PATCH_DIR}/cci_hang_final.patch - echo "########## ADA-FS injection: Disabling cci debug mode/devel mode entirely" + echo "########## Patch injection: Disabling cci debug mode/devel mode entirely" git apply ${PATCH_DIR}/cci_remove_devel_mode.patch ./autogen.pl cd ${CURR}/build @@ -227,7 +225,7 @@ else ../configure --prefix=${INSTALL} LIBS="-lpthread" fi - echo "########## ADA-FS injection: Replacing any remaining CFLAGS with '-g -O2' that are added by cci although debug mode is disabled with '-O3'" + echo "########## Patch injection: Replacing any remaining CFLAGS with '-g -O2' that are added by cci although debug mode is disabled with '-O3'" find . -type f -exec sed -i 's/-g -O2/-O3/g' {} \; make -j${CORES} make install diff --git a/scripts/dl_dep.sh b/scripts/dl_dep.sh index faa5f260c853d4b2f71b167e6c4dbaaf53aa83a6..a3dcdf1c8390c2d29699608a8f6519d8d54faa15 100755 --- a/scripts/dl_dep.sh +++ b/scripts/dl_dep.sh @@ -78,7 +78,7 @@ help_msg() { usage_short echo " -This script gets all ADA-FS dependency sources (excluding the fs itself) +This script gets all GekkoFS dependency sources (excluding the fs itself) positional arguments: source_path path where the dependency downloads are put @@ -201,7 +201,7 @@ wgetdeps "argobots" "https://github.com/pmodels/argobots/archive/v1.0rc1.tar.gz" # get Margo clonedeps "margo" "https://xgitlab.cels.anl.gov/sds/margo.git" "ad09e1e445a9275b24036e465bee3b4b75f0fb17" & # get rocksdb -wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v5.18.3.tar.gz" & +wgetdeps "rocksdb" "https://github.com/facebook/rocksdb/archive/v6.0.1.tar.gz" & # Wait for all download to be completed wait diff --git a/scripts/license/add_license_headers.sh b/scripts/license/add_license_headers.sh new file mode 100755 index 0000000000000000000000000000000000000000..b2d9eab0b0a7f73db80f3186109ceb38c45d6594 --- /dev/null +++ b/scripts/license/add_license_headers.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +PROJ_DIR=`readlink -f ${DIR}/../../` + +docker run --rm --volume ${PROJ_DIR}:/usr/src/ osterman/copyright-header:latest \ + --syntax scripts/license/copyright_header_syntax.yml \ + --license-file scripts/license/header \ + --guess-extension \ + --word-wrap 80 \ + --output-dir /usr/src/ \ + --add-path ./src:./include diff --git a/scripts/license/copyright_header_syntax.yml b/scripts/license/copyright_header_syntax.yml new file mode 100644 index 0000000000000000000000000000000000000000..c9b649b53c9e78ec00ffca21a057e1fcce55871d --- /dev/null +++ b/scripts/license/copyright_header_syntax.yml @@ -0,0 +1,225 @@ +ruby: + ext: ['.rb', '.rake'] + after: ['^#!', '^#.*encoding:', '^#.*frozen_string_literal:'] + comment: + open: '\n#\n' + close: '#\n' + prefix: '# ' + +perl: + ext: ['.pl'] + after: ['^#!', '^#.*encoding:'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' + +# Support PEP 0263 comments: +# coding= +# -*- coding: -*- +# vim: set fileencoding= : +python: + ext: ['.py'] + after: ['^#!', '^#.*coding:', '^#.*coding=', '^#.*fileencoding='] + comment: + open: '\n' + close: '\n' + prefix: '# ' + +html: + ext: ['.html', '.htm', '.xhtml'] + comment: + open: '\n' + prefix: ' ' + +php: + ext: ['.php'] + after: [ '^#!' ] + comment: + open: '\n' + prefix: ' * ' + +javacript: + ext: ['.js', '.jsx'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +qml: + ext: ['.qml'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +qmake_project: + ext: ['.pro'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' + +css: + ext: ['.css'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +c: + ext: ['.c', '.h'] + comment: + open: '/*' + close: ' */\n\n' + prefix: ' * ' + +cpp: + ext: ['.cpp', '.hpp', '.cc', '.hh'] + comment: + open: '/*\n' + close: '*/\n\n' + prefix: ' ' + +java: + ext: ['.java'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +kotlin: + ext: ['.kt'] + comment: + open: '/*\n' + clone: ' */\n\n' + prefix: ' * ' + +golang: + ext: ['.go'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +groovy: + ext: ['.groovy'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +haml: + ext: ['.haml', '.hamlc'] + comment: + open: '-#\n' + close: '-#\n' + prefix: '-# ' + +coffee: + ext: ['.coffee'] + comment: + open: '###\n' + close: '###\n' + prefix: '' + +# M4 macro language, use #, not dnl +m4: + ext: ['.m4'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' + +# Most shells, really +shell: + ext: ['.sh'] + after: ['^#!'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' + +# Use "-- " to make sure e.g. MySQL understands it +sql: + ext: ['.sql'] + comment: + open: '-- \n' + close: '-- \n' + prefix: '-- ' + +# XML is *not* the same as HTML, and the comments need to go after a +# preprocessing directive, if present. +# FIXME: only supports single line directives +xml: + ext: ['.xml', '.xsd', '.mxml'] + after: ['^<\?'] + comment: + open: '\n' + prefix: ' ' + +yaml: + ext: ['.yml', '.yaml'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' + +action_script: + ext: ['.as'] + comment: + open: '//\n' + close: '//\n\n' + prefix: '// ' + +sass: + ext: ['.sass', '.scss'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +verilog: + ext: ['.v', '.sv'] + comment: + open: '//\n' + close: '//\n\n' + prefix: '// ' + +vhdl: + ext: ['.vhd'] + comment: + open: '--\n' + close: '--\n\n' + prefix: '-- ' + +elm: + ext: ['.elm'] + comment: + open: '{-\n' + close: '-}\n\n' + prefix: ' ' + +swift: + ext: ['.swift'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' + +rust: + ext: ['.rs'] + comment: + open: '/*\n' + close: ' */\n\n' + prefix: ' * ' +# Conf files i.e. apache config, splunk.conf files +conf: + ext: ['.conf'] + comment: + open: '#\n' + close: '#\n' + prefix: '# ' diff --git a/scripts/license/header b/scripts/license/header new file mode 100644 index 0000000000000000000000000000000000000000..8d3c73be61e52bea19e1e5f53301f2eb8a2df0f7 --- /dev/null +++ b/scripts/license/header @@ -0,0 +1,10 @@ +Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain +Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + +This software was partially supported by the +EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + +This software was partially supported by the +ADA-FS project under the SPPEXA project funded by the DFG. + +SPDX-License-Identifier: MIT diff --git a/scripts/shutdown_adafs.py b/scripts/shutdown_gkfs.py similarity index 94% rename from scripts/shutdown_adafs.py rename to scripts/shutdown_gkfs.py index 320fd5d0c3226732fb2eb25d8f74026dd1faf662..cca9265dbccd81a990b09319733bf9421fa7b0c1 100755 --- a/scripts/shutdown_adafs.py +++ b/scripts/shutdown_gkfs.py @@ -34,11 +34,11 @@ def check_dependencies(): def shutdown_system(daemon_pid_path, nodelist, sigkill): - """Shuts down ADAFS on specified nodes. + """Shuts down GekkoFS on specified nodes. Args: daemon_pid_path (str): Path to daemon pid file - nodelist (str): Comma-separated list of nodes where adafs is launched on + nodelist (str): Comma-separated list of nodes where daemons need to be launched sigkill (bool): If true force kills daemons """ global PSSH_PATH @@ -93,7 +93,7 @@ def shutdown_system(daemon_pid_path, nodelist, sigkill): time.sleep(1) print('Checking logs ...\n') - cmd_chk_str = '%s "tail -4 /tmp/adafs_daemon.log"' % pssh + cmd_chk_str = '%s "tail -4 /tmp/gkfs_daemon.log"' % pssh if PRETEND: print('Pretending: {}'.format(cmd_chk_str)) else: @@ -126,7 +126,7 @@ def shutdown_system(daemon_pid_path, nodelist, sigkill): if __name__ == "__main__": # Init parser - parser = argparse.ArgumentParser(description='This script stops adafs on multiple nodes', + parser = argparse.ArgumentParser(description='This script stops GekkoFS on multiple nodes', formatter_class=argparse.RawTextHelpFormatter) # positional arguments parser.add_argument('daemonpidpath', type=str, @@ -137,9 +137,9 @@ if __name__ == "__main__": # optional arguments parser.add_argument('-p', '--pretend', action='store_true', - help='Output adafs launch command and do not actually execute it') + help='Output launch command and do not actually execute it') parser.add_argument('-9', '--sigkill', action='store_true', - help='Force kill adafs_daemons') + help='Force kill daemons') parser.add_argument('-P', '--pssh', metavar='', type=str, default='', help='Path to parallel-ssh/pssh. Defaults to /usr/bin/{parallel-ssh,pssh}') parser.add_argument('-J', '--jobid', metavar='', type=str, default='', diff --git a/scripts/startup_adafs.py b/scripts/startup_gkfs.py similarity index 90% rename from scripts/startup_adafs.py rename to scripts/startup_gkfs.py index 98b8a6e3fd5a42b1a3d9a8416a1a40a18dbfd547..935bbfd5ea2bc1b3f097c59e46fddc987c4a36b9 100755 --- a/scripts/startup_adafs.py +++ b/scripts/startup_gkfs.py @@ -34,14 +34,14 @@ def check_dependencies(): def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, numactl): - """Initializes ADAFS on specified nodes. + """Initializes GekkoFS on specified nodes. Args: daemon_path (str): Path to daemon executable rootdir (str): Path to root directory for fs data metadir (str): Path to metadata directory where metadata is stored - mountdir (str): Path to mount directory where adafs is used in - nodelist (str): Comma-separated list of nodes where adafs is launched on + mountdir (str): Path to mount directory where is used in + nodelist (str): Comma-separated list of nodes where daemons need to be launched cleanroot (bool): if True, root and metadir is cleaned before daemon init numactl (str): numactl arguments for daemon init """ @@ -73,7 +73,7 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu # clean root and metadata dir if needed if cleanroot: - cmd_rm_str = '%s "rm -rf %s/* %s/* && truncate -s 0 /tmp/adafs_daemon.log /tmp/adafs_preload.log"' % (pssh, rootdir, metadir) + cmd_rm_str = '%s "rm -rf %s/* %s/* && truncate -s 0 /tmp/gkfs_daemon.log /tmp/gkfs_preload.log"' % (pssh, rootdir, metadir) if PRETEND: print('Pretending: {}'.format(cmd_rm_str)) else: @@ -95,18 +95,18 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu # Start deamons if nodefile: if len(numactl) == 0: - cmd_str = '%s "nohup %s -r %s -i %s -m %s --hostfile %s > /tmp/adafs_daemon.log 2>&1 &"' \ + cmd_str = '%s "nohup %s -r %s -i %s -m %s --hostfile %s > /tmp/gkfs_daemon.log 2>&1 &"' \ % (pssh, daemon_path, rootdir, metadir, mountdir, nodelist) else: - cmd_str = '%s "nohup numactl %s %s -r %s -i %s -m %s --hostfile %s > /tmp/adafs_daemon.log 2>&1 &"' \ + cmd_str = '%s "nohup numactl %s %s -r %s -i %s -m %s --hostfile %s > /tmp/gkfs_daemon.log 2>&1 &"' \ % (pssh, numactl, daemon_path, rootdir, metadir, mountdir, nodelist) else: if len(numactl) == 0: - cmd_str = '%s "nohup %s -r %s -i %s -m %s --hosts %s > /tmp/adafs_daemon.log 2>&1 &"' \ + cmd_str = '%s "nohup %s -r %s -i %s -m %s --hosts %s > /tmp/gkfs_daemon.log 2>&1 &"' \ % (pssh, daemon_path, rootdir, metadir, mountdir, nodelist) else: - cmd_str = '%s "nohup numactl %s %s -r %s -i %s -m %s --hosts %s > /tmp/adafs_daemon.log 2>&1 &"' \ + cmd_str = '%s "nohup numactl %s %s -r %s -i %s -m %s --hosts %s > /tmp/gkfs_daemon.log 2>&1 &"' \ % (pssh, numactl, daemon_path, rootdir, metadir, mountdir, nodelist) if PRETEND: @@ -124,7 +124,7 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu if not err: print('pssh daemon launch successfully executed. Checking for FS startup errors ...\n') else: - print('[ERR] with pssh. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!') + print('[ERR] with pssh. Aborting. Please run shutdown_gkfs.py to shut down orphan daemons!') exit(1) if not PRETEND: @@ -133,8 +133,8 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu print('{}\r'.format(WAITTIME - i)), time.sleep(1) - # Check adafs logs for errors - cmd_chk_str = '%s "head -5 /tmp/adafs_daemon.log"' % pssh + # Check logs for errors + cmd_chk_str = '%s "head -5 /tmp/gkfs_daemon.log"' % pssh if PRETEND: print('Pretending: {}'.format(cmd_chk_str)) else: @@ -160,13 +160,13 @@ def init_system(daemon_path, rootdir, metadir, mountdir, nodelist, cleanroot, nu if not err and not fs_err: print('pssh logging check successfully executed. Looks prime.') else: - print('[ERR] while checking fs logs. Aborting. Please run shutdown_adafs.py to shut down orphan adafs daemons!') + print('[ERR] while checking fs logs. Aborting. Please run shutdown_gkfs.py to shut down orphan daemons!') exit(1) if __name__ == "__main__": # Init parser - parser = argparse.ArgumentParser(description='This script launches adafs on multiple nodes', + parser = argparse.ArgumentParser(description='This script launches GekkoFS on multiple nodes', formatter_class=argparse.RawTextHelpFormatter) # positional arguments parser.add_argument('daemonpath', type=str, @@ -184,15 +184,15 @@ or a path to a nodefile (one node per line)''') help='''Path to separate metadir directory where metadata is stored. If not set, rootdir will be used instead.''') parser.add_argument('-p', '--pretend', action='store_true', - help='Output adafs launch command and do not actually execute it') + help='Output launch command and do not actually execute it') parser.add_argument('-P', '--pssh', metavar='', type=str, default='', help='Path to parallel-ssh/pssh. Defaults to /usr/bin/{parallel-ssh,pssh}') parser.add_argument('-J', '--jobid', metavar='', type=str, default='', help='Jobid for cluster batch system. Used for a unique hostfile used for pssh.') parser.add_argument('-c', '--cleanroot', action='store_true', - help='Removes contents of root and metadata directory before starting ADA-FS Daemon. Be careful!') + help='Removes contents of root and metadata directory before starting daemon. Be careful!') parser.add_argument('-n', '--numactl', metavar='', type=str, default='', - help='If adafs daemon should be pinned to certain cores, set numactl arguments here.') + help='If daemon should be pinned to certain cores, set numactl arguments here.') parser.add_argument('-H', '--pssh_hostfile', metavar='', type=str, default='/tmp/hostfile_pssh', help='''This script creates a hostfile to pass to MPI. This variable defines the path. Defaults to /tmp/hostfile_pssh''') diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 17b595f1beb284f6a71b37442a2a8722b2e31d18..ac665418d9ca01e372426948174c42fe8d7e5e21 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -37,9 +37,9 @@ set(PRELOAD_HEADERS ../../include/client/rpc/ld_rpc_metadentry.hpp ) -add_library(adafs_preload_client SHARED ${PRELOAD_SRC} ${PRELOAD_HEADERS}) +add_library(gkfs_intercept SHARED ${PRELOAD_SRC} ${PRELOAD_HEADERS}) -target_link_libraries(adafs_preload_client +target_link_libraries(gkfs_intercept # internal metadata distributor @@ -53,8 +53,15 @@ target_link_libraries(adafs_preload_client Threads::Threads ) -target_include_directories(adafs_preload_client +target_include_directories(gkfs_intercept PRIVATE ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) + +install(TARGETS gkfs_intercept + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gkfs +) + diff --git a/src/client/adafs_functions.cpp b/src/client/adafs_functions.cpp index dee0271d8b99b6bb47370db5b8dc2bf54e6267de..d84c240180bd97906ff53e819e0a53a5fd51abc1 100644 --- a/src/client/adafs_functions.cpp +++ b/src/client/adafs_functions.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include diff --git a/src/client/intcp_functions.cpp b/src/client/intcp_functions.cpp index 74b133a236487cf88f0301f765ea7d97ac871788..0235f184cbf5988fd0db68424cd7091077f350c4 100644 --- a/src/client/intcp_functions.cpp +++ b/src/client/intcp_functions.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + /** * All intercepted functions are defined here */ @@ -288,18 +301,6 @@ int fflush(FILE *stream) { return (reinterpret_cast(libc_fflush))(stream); } -int fpurge(FILE *stream) { - init_passthrough_if_needed(); - if(CTX->interception_enabled() && (stream != nullptr)) { - auto fd = file_to_fd(stream); - if(CTX->file_map()->exist(fd)) { - CTX->log()->trace("{}() called on fd {}", __func__, fd); - return 0; - } - } - return (reinterpret_cast(libc_fpurge))(stream); -} - void __fpurge(FILE *stream) { init_passthrough_if_needed(); if(CTX->interception_enabled() && (stream != nullptr)) { @@ -665,44 +666,6 @@ int faccessat(int dirfd, const char* cpath, int mode, int flags) noexcept { } -int stat(const char* path, struct stat* buf) noexcept { - init_passthrough_if_needed(); - if(!CTX->interception_enabled()) { - return LIBC_FUNC(stat, path, buf); - } - CTX->log()->trace("{}() called with path '{}'", __func__, path); - std::string rel_path; - if (!CTX->relativize_path(path, rel_path)) { - return LIBC_FUNC(stat, rel_path.c_str(), buf); - } - return adafs_stat(rel_path, buf); -} - -int fstat(int fd, struct stat* buf) noexcept { - init_passthrough_if_needed(); - if(CTX->interception_enabled()) { - CTX->log()->trace("{}() called with fd {}", __func__, fd); - if (CTX->file_map()->exist(fd)) { - auto path = CTX->file_map()->get(fd)->path(); - return adafs_stat(path, buf); - } - } - return LIBC_FUNC(fstat, fd, buf); -} - -int lstat(const char* path, struct stat* buf) noexcept { - init_passthrough_if_needed(); - if(!CTX->interception_enabled()) { - return LIBC_FUNC(lstat, path, buf); - } - CTX->log()->trace("{}() called with path '{}'", __func__, path); - std::string rel_path; - if (!CTX->relativize_path(path, rel_path, false)) { - return LIBC_FUNC(lstat, rel_path.c_str(), buf); - } - return adafs_stat(rel_path, buf, false); -} - int __xstat(int ver, const char* path, struct stat* buf) noexcept { init_passthrough_if_needed(); if(!CTX->interception_enabled()) { diff --git a/src/client/open_dir.cpp b/src/client/open_dir.cpp index d22cdea3cd7b25a0484b99fe7f8781c230913caa..a14eb02321958c11ba2bf9e67e9e875a8d5e3dc6 100644 --- a/src/client/open_dir.cpp +++ b/src/client/open_dir.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include diff --git a/src/client/open_file_map.cpp b/src/client/open_file_map.cpp index 54674905a43eced0c338696ecd70249fe49515e8..98cd98a5608cd8c5832f00332fdfc4529febb9e6 100644 --- a/src/client/open_file_map.cpp +++ b/src/client/open_file_map.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include @@ -168,7 +181,7 @@ int OpenFileMap::dup2(const int oldfd, const int newfd) { } /** - * Generate new file descriptor index to be used as an fd within one process in ADA-FS + * Generate new file descriptor index to be used as an fd within one process * @return fd_idx */ int OpenFileMap::generate_fd_idx() { diff --git a/src/client/passthrough.cpp b/src/client/passthrough.cpp index 33c7361bc0d32a85c30f8ea6f3e82fa70796d701..93aecb5136f65fa9872c9539828e33db471587ee 100644 --- a/src/client/passthrough.cpp +++ b/src/client/passthrough.cpp @@ -1,241 +1,263 @@ -/** - * All intercepted functions are mapped to a different function pointer prefixing - */ -#include +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +#include "client/passthrough.hpp" -#include +#include #include #include +#include +#include + + +/* We cannot use any fprintf here because it will call the write function internally + * that we are also intercepting, and this will lead to a deadlock. + * + * Instead the write syscall needs to be called directly. + */ +#define WRITE_STDERR(str) \ + do { \ + syscall(SYS_write, 2, str, sizeof(str)); \ + } while (0) + + +#define LIBC_FUNC_LOAD(FNAME) \ + do { \ + LIBC_FUNC_NAME(FNAME) = dlsym(glibc, #FNAME); \ + if (LIBC_FUNC_NAME(FNAME) == nullptr) { \ + WRITE_STDERR("GekkoFS ERROR: failed to load glibc func symbol: '" #FNAME "'\n"); \ + exit(EXIT_FAILURE); \ + } \ + } while (0) + static pthread_once_t init_lib_thread = PTHREAD_ONCE_INIT; -// function pointer for preloading -void* libc; - -void* libc_open; -void* libc_openat; - -void* libc_fopen; -void* libc_fopen64; -void* libc_fread; -void* libc_fwrite; -void* libc_fclose; -void* libc_clearerr; -void* libc_feof; -void* libc_ferror; -void* libc_fileno; -void* libc_fflush; -void* libc_fpurge; -void* libc___fpurge; - -void* libc_setbuf; -void* libc_setbuffer; -void* libc_setlinebuf; -void* libc_setvbuf; - -void* libc_putc; -void* libc_fputc; -void* libc_fputs; -void* libc_getc; -void* libc_fgetc; -void* libc_fgets; -void* libc_ungetc; - -void* libc_fseek; - -void* libc_mkdir; -void* libc_mkdirat; -void* libc_unlink; -void* libc_unlinkat; -void* libc_rmdir; - -void* libc_close; - -void* libc_access; -void* libc_faccessat; - -void* libc_stat; -void* libc_fstat; -void* libc_lstat; -void* libc___xstat; -void* libc___xstat64; -void* libc___fxstat; -void* libc___fxstat64; -void* libc___fxstatat; -void* libc___fxstatat64; -void* libc___lxstat; -void* libc___lxstat64; - -void* libc_statfs; -void* libc_fstatfs; -void* libc_statvfs; -void* libc_fstatvfs; - -void* libc_write; -void* libc_pwrite; -void* libc_pwrite64; -void* libc_writev; - -void* libc_read; -void* libc_pread; -void* libc_pread64; -void* libc_readv; - -void* libc_lseek; -void* libc_lseek64; - -void* libc_fsync; -void* libc_fdatasync; - -void* libc_truncate; -void* libc_ftruncate; - -void* libc_fcntl; - -void* libc_dup; -void* libc_dup2; -void* libc_dup3; - -void* libc_dirfd; -void* libc_opendir; -void* libc_fdopendir; -void* libc_readdir; -void* libc_closedir; - -void* libc_chmod; -void* libc_fchmod; -void* libc_fchmodat; - -void* libc_chdir; -void* libc_fchdir; - -void* libc_getcwd; -void* libc_get_current_dir_name; - -void* libc_link; -void* libc_linkat; -void* libc_symlinkat; - -void* libc_readlinkat; - -void* libc_realpath; + +void* LIBC_FUNC_NAME(open); +void* LIBC_FUNC_NAME(openat); + +void* LIBC_FUNC_NAME(fopen); +void* LIBC_FUNC_NAME(fopen64); +void* LIBC_FUNC_NAME(fread); +void* LIBC_FUNC_NAME(fwrite); +void* LIBC_FUNC_NAME(fclose); +void* LIBC_FUNC_NAME(clearerr); +void* LIBC_FUNC_NAME(feof); +void* LIBC_FUNC_NAME(ferror); +void* LIBC_FUNC_NAME(fileno); +void* LIBC_FUNC_NAME(fflush); +void* LIBC_FUNC_NAME(__fpurge); + +void* LIBC_FUNC_NAME(setbuf); +void* LIBC_FUNC_NAME(setbuffer); +void* LIBC_FUNC_NAME(setlinebuf); +void* LIBC_FUNC_NAME(setvbuf); + +void* LIBC_FUNC_NAME(putc); +void* LIBC_FUNC_NAME(fputc); +void* LIBC_FUNC_NAME(fputs); +void* LIBC_FUNC_NAME(getc); +void* LIBC_FUNC_NAME(fgetc); +void* LIBC_FUNC_NAME(fgets); +void* LIBC_FUNC_NAME(ungetc); + +void* LIBC_FUNC_NAME(fseek); + +void* LIBC_FUNC_NAME(mkdir); +void* LIBC_FUNC_NAME(mkdirat); +void* LIBC_FUNC_NAME(unlink); +void* LIBC_FUNC_NAME(unlinkat); +void* LIBC_FUNC_NAME(rmdir); + +void* LIBC_FUNC_NAME(close); + +void* LIBC_FUNC_NAME(access); +void* LIBC_FUNC_NAME(faccessat); + +void* LIBC_FUNC_NAME(__xstat); +void* LIBC_FUNC_NAME(__xstat64); +void* LIBC_FUNC_NAME(__fxstat); +void* LIBC_FUNC_NAME(__fxstat64); +void* LIBC_FUNC_NAME(__fxstatat); +void* LIBC_FUNC_NAME(__fxstatat64); +void* LIBC_FUNC_NAME(__lxstat); +void* LIBC_FUNC_NAME(__lxstat64); + +void* LIBC_FUNC_NAME(statfs); +void* LIBC_FUNC_NAME(fstatfs); +void* LIBC_FUNC_NAME(statvfs); +void* LIBC_FUNC_NAME(fstatvfs); + +void* LIBC_FUNC_NAME(write); +void* LIBC_FUNC_NAME(pwrite); +void* LIBC_FUNC_NAME(pwrite64); +void* LIBC_FUNC_NAME(writev); + +void* LIBC_FUNC_NAME(read); +void* LIBC_FUNC_NAME(pread); +void* LIBC_FUNC_NAME(pread64); +void* LIBC_FUNC_NAME(readv); + +void* LIBC_FUNC_NAME(lseek); +void* LIBC_FUNC_NAME(lseek64); + +void* LIBC_FUNC_NAME(fsync); +void* LIBC_FUNC_NAME(fdatasync); + +void* LIBC_FUNC_NAME(truncate); +void* LIBC_FUNC_NAME(ftruncate); + +void* LIBC_FUNC_NAME(fcntl); + +void* LIBC_FUNC_NAME(dup); +void* LIBC_FUNC_NAME(dup2); +void* LIBC_FUNC_NAME(dup3); + +void* LIBC_FUNC_NAME(dirfd); +void* LIBC_FUNC_NAME(opendir); +void* LIBC_FUNC_NAME(fdopendir); +void* LIBC_FUNC_NAME(readdir); +void* LIBC_FUNC_NAME(closedir); + +void* LIBC_FUNC_NAME(chmod); +void* LIBC_FUNC_NAME(fchmod); +void* LIBC_FUNC_NAME(fchmodat); + +void* LIBC_FUNC_NAME(chdir); +void* LIBC_FUNC_NAME(fchdir); + +void* LIBC_FUNC_NAME(getcwd); +void* LIBC_FUNC_NAME(get_current_dir_name); + +void* LIBC_FUNC_NAME(link); +void* LIBC_FUNC_NAME(linkat); +void* LIBC_FUNC_NAME(symlinkat); + +void* LIBC_FUNC_NAME(readlinkat); +void* LIBC_FUNC_NAME(realpath); void init_passthrough_() { - libc = dlopen("libc.so.6", RTLD_LAZY); - if(libc == nullptr){ - std::cerr << "ERROR: failed opening glibc shared object: " << dlerror() << std::endl; + auto glibc = dlopen("libc.so.6", RTLD_LAZY); + if(glibc == nullptr) { + WRITE_STDERR("GekkoFS ERROR: failed to load glibc\n"); \ exit(EXIT_FAILURE); } - libc_open = dlsym(libc, "open"); - libc_openat = dlsym(libc, "openat"); - - libc_fopen = dlsym(libc, "fopen"); - libc_fopen64 = dlsym(libc, "fopen64"); - libc_fread = dlsym(libc, "fread"); - libc_fwrite = dlsym(libc, "fwrite"); - libc_fclose = dlsym(libc, "fclose"); - libc_clearerr = dlsym(libc, "clearerr"); - libc_feof = dlsym(libc, "feof"); - libc_ferror = dlsym(libc, "ferror"); - libc_fileno = dlsym(libc, "fileno"); - libc_fflush = dlsym(libc, "fflush"); - libc_fpurge = dlsym(libc, "fpurge"); - libc___fpurge = dlsym(libc, "__fpurge"); - - libc_setbuf = dlsym(libc, "setbuf"); - libc_setbuffer = dlsym(libc, "setbuffer"); - libc_setlinebuf = dlsym(libc, "setlinebuf"); - libc_setvbuf = dlsym(libc, "setvbuf"); - - libc_putc = dlsym(libc, "putc"); - libc_fputc = dlsym(libc, "fputc"); - libc_fputs = dlsym(libc, "fputs"); - libc_getc = dlsym(libc, "getc"); - libc_fgetc = dlsym(libc, "fgetc"); - libc_fgets = dlsym(libc, "fgets"); - libc_ungetc = dlsym(libc, "ungetc"); - - libc_fseek = dlsym(libc, "fseek"); - - libc_mkdir = dlsym(libc, "mkdir"); - libc_mkdirat = dlsym(libc, "mkdirat"); - - libc_unlink = dlsym(libc, "unlink"); - libc_unlinkat = dlsym(libc, "unlinkat"); - libc_rmdir = dlsym(libc, "rmdir"); - - libc_close = dlsym(libc, "close"); - - libc_access = dlsym(libc, "access"); - libc_faccessat = dlsym(libc, "faccessat"); - - libc_stat = dlsym(libc, "stat"); - libc_fstat = dlsym(libc, "fstat"); - libc_lstat = dlsym(libc, "lstat"); - libc___xstat = dlsym(libc, "__xstat"); - libc___xstat64 = dlsym(libc, "__xstat64"); - libc___fxstat = dlsym(libc, "__fxstat"); - libc___fxstat64 = dlsym(libc, "__fxstat64"); - libc___fxstatat = dlsym(libc, "__fxstatat"); - libc___fxstatat64 = dlsym(libc, "__fxstatat64"); - libc___lxstat = dlsym(libc, "__lxstat"); - libc___lxstat64 = dlsym(libc, "__lxstat64"); - - libc_statfs = dlsym(libc, "statfs"); - libc_fstatfs = dlsym(libc, "fstatfs"); - libc_statvfs = dlsym(libc, "statvfs"); - libc_fstatvfs = dlsym(libc, "fstatvfs"); - - libc_write = dlsym(libc, "write"); - libc_pwrite = dlsym(libc, "pwrite"); - libc_pwrite64 = dlsym(libc, "pwrite64"); - libc_writev = dlsym(libc, "writev"); - - libc_read = dlsym(libc, "read"); - libc_pread = dlsym(libc, "pread"); - libc_pread64 = dlsym(libc, "pread64"); - libc_readv = dlsym(libc, "readv"); - - libc_lseek = dlsym(libc, "lseek"); - libc_lseek64 = dlsym(libc, "lseek64"); - libc_fsync = dlsym(libc, "fsync"); - libc_fdatasync = dlsym(libc, "fdatasync"); - - libc_truncate = dlsym(libc, "truncate"); - libc_ftruncate = dlsym(libc, "ftruncate"); - - libc_fcntl = dlsym(libc, "fcntl"); - - libc_dup = dlsym(libc, "dup"); - libc_dup2 = dlsym(libc, "dup2"); - libc_dup3 = dlsym(libc, "dup3"); - - libc_dirfd = dlsym(libc, "dirfd"); - libc_opendir = dlsym(libc, "opendir"); - libc_fdopendir = dlsym(libc, "fdopendir"); - libc_readdir = dlsym(libc, "readdir"); - libc_closedir = dlsym(libc, "closedir"); - - libc_chmod = dlsym(libc, "chmod"); - libc_fchmod = dlsym(libc, "fchmod"); - libc_fchmodat = dlsym(libc, "fchmodat"); - - libc_chdir = dlsym(libc, "chdir"); - libc_fchdir = dlsym(libc, "fchdir"); - - libc_getcwd = dlsym(libc, "getcwd"); - libc_get_current_dir_name = dlsym(libc, "get_current_dir_name"); - - libc_link = dlsym(libc, "link"); - libc_linkat = dlsym(libc, "linkat"); - libc_symlinkat = dlsym(libc, "symlinkat"); - - libc_readlinkat = dlsym(libc, "readlinkat"); - - libc_realpath = dlsym(libc, "realpath"); - + LIBC_FUNC_LOAD(open); + LIBC_FUNC_LOAD(openat); + + LIBC_FUNC_LOAD(fopen); + LIBC_FUNC_LOAD(fopen64); + LIBC_FUNC_LOAD(fread); + LIBC_FUNC_LOAD(fwrite); + LIBC_FUNC_LOAD(fclose); + LIBC_FUNC_LOAD(clearerr); + LIBC_FUNC_LOAD(feof); + LIBC_FUNC_LOAD(ferror); + LIBC_FUNC_LOAD(fileno); + LIBC_FUNC_LOAD(fflush); + LIBC_FUNC_LOAD(__fpurge); + + LIBC_FUNC_LOAD(setbuf); + LIBC_FUNC_LOAD(setbuffer); + LIBC_FUNC_LOAD(setlinebuf); + LIBC_FUNC_LOAD(setvbuf); + + LIBC_FUNC_LOAD(putc); + LIBC_FUNC_LOAD(fputc); + LIBC_FUNC_LOAD(fputs); + LIBC_FUNC_LOAD(getc); + LIBC_FUNC_LOAD(fgetc); + LIBC_FUNC_LOAD(fgets); + LIBC_FUNC_LOAD(ungetc); + + LIBC_FUNC_LOAD(fseek); + + LIBC_FUNC_LOAD(mkdir); + LIBC_FUNC_LOAD(mkdirat); + + LIBC_FUNC_LOAD(unlink); + LIBC_FUNC_LOAD(unlinkat); + LIBC_FUNC_LOAD(rmdir); + + LIBC_FUNC_LOAD(close); + + LIBC_FUNC_LOAD(access); + LIBC_FUNC_LOAD(faccessat); + + LIBC_FUNC_LOAD(__xstat); + LIBC_FUNC_LOAD(__xstat64); + LIBC_FUNC_LOAD(__fxstat); + LIBC_FUNC_LOAD(__fxstat64); + LIBC_FUNC_LOAD(__fxstatat); + LIBC_FUNC_LOAD(__fxstatat64); + LIBC_FUNC_LOAD(__lxstat); + LIBC_FUNC_LOAD(__lxstat64); + + LIBC_FUNC_LOAD(statfs); + LIBC_FUNC_LOAD(fstatfs); + LIBC_FUNC_LOAD(statvfs); + LIBC_FUNC_LOAD(fstatvfs); + + LIBC_FUNC_LOAD(write); + LIBC_FUNC_LOAD(pwrite); + LIBC_FUNC_LOAD(pwrite64); + LIBC_FUNC_LOAD(writev); + + LIBC_FUNC_LOAD(read); + LIBC_FUNC_LOAD(pread); + LIBC_FUNC_LOAD(pread64); + LIBC_FUNC_LOAD(readv); + + LIBC_FUNC_LOAD(lseek); + LIBC_FUNC_LOAD(lseek64); + LIBC_FUNC_LOAD(fsync); + LIBC_FUNC_LOAD(fdatasync); + + LIBC_FUNC_LOAD(truncate); + LIBC_FUNC_LOAD(ftruncate); + + LIBC_FUNC_LOAD(fcntl); + + LIBC_FUNC_LOAD(dup); + LIBC_FUNC_LOAD(dup2); + LIBC_FUNC_LOAD(dup3); + + LIBC_FUNC_LOAD(dirfd); + LIBC_FUNC_LOAD(opendir); + LIBC_FUNC_LOAD(fdopendir); + LIBC_FUNC_LOAD(readdir); + LIBC_FUNC_LOAD(closedir); + + LIBC_FUNC_LOAD(chmod); + LIBC_FUNC_LOAD(fchmod); + LIBC_FUNC_LOAD(fchmodat); + + LIBC_FUNC_LOAD(chdir); + LIBC_FUNC_LOAD(fchdir); + + LIBC_FUNC_LOAD(getcwd); + LIBC_FUNC_LOAD(get_current_dir_name); + + LIBC_FUNC_LOAD(link); + LIBC_FUNC_LOAD(linkat); + LIBC_FUNC_LOAD(symlinkat); + + LIBC_FUNC_LOAD(readlinkat); + + LIBC_FUNC_LOAD(realpath); } void init_passthrough_if_needed() { diff --git a/src/client/preload.cpp b/src/client/preload.cpp index 6c5adee63a677a12c309a431e627237846baa764..6386708c6d0a5b3179eb39e775408e888be47ea5 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include @@ -218,7 +231,7 @@ void init_preload() { init_cwd(); CTX->log()->debug("Current working directory: '{}'", CTX->cwd()); if (get_daemon_pid() == -1 || CTX->mountdir().empty()) { - std::cerr << "ADA-FS daemon not running or mountdir could not be loaded. Check adafs_preload.log" << std::endl; + std::cerr << "GekkoFS daemon not running or mountdir could not be loaded. Check logs for more details" << std::endl; CTX->log()->error("{}() Daemon not running or mountdir not set", __func__); exit(EXIT_FAILURE); } else { diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index 9643a134a20436832f7fab68e7a8c052df758156..0a7ef22f39e790cb4f7e77e9e299706fdd689c38 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include diff --git a/src/client/preload_util.cpp b/src/client/preload_util.cpp index a292dadfcc9839bf080b078aa18e60f8eba5e764..57b8260cd1fb9e9e69e21f9fbbd8fc5e66756a79 100644 --- a/src/client/preload_util.cpp +++ b/src/client/preload_util.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include @@ -78,15 +91,15 @@ int get_daemon_pid() { if (getline(ifs, adafs_daemon_pid_s) && !adafs_daemon_pid_s.empty()) adafs_daemon_pid = ::stoi(adafs_daemon_pid_s); else { - cerr << "ADA-FS daemon pid not found. Daemon not running?" << endl; + cerr << "GekkoFS daemon pid not found. Daemon not running?" << endl; CTX->log()->error("{}() Unable to read daemon pid from pid file", __func__); ifs.close(); return -1; } // check that daemon is running if (kill(adafs_daemon_pid, 0) != 0) { - cerr << "ADA-FS daemon process with pid " << adafs_daemon_pid << " not found. Daemon not running?" << endl; - CTX->log()->error("{}() ADA-FS daemon pid {} not found. Daemon not running?", __func__, adafs_daemon_pid); + cerr << "GekkoFS daemon process with pid " << adafs_daemon_pid << " not found. Daemon not running?" << endl; + CTX->log()->error("{}() daemon pid {} not found. Daemon not running?", __func__, adafs_daemon_pid); ifs.close(); return -1; } @@ -95,7 +108,7 @@ int get_daemon_pid() { if (getline(ifs, daemon_addr) && !daemon_addr.empty()) { CTX->daemon_addr_str(daemon_addr); } else { - CTX->log()->error("{}() ADA-FS daemon pid file contains no daemon address. Exiting ...", __func__); + CTX->log()->error("{}() daemon pid file contains no daemon address. Exiting ...", __func__); ifs.close(); return -1; } @@ -103,13 +116,13 @@ int get_daemon_pid() { if (getline(ifs, mountdir) && !mountdir.empty()) { CTX->mountdir(mountdir); } else { - CTX->log()->error("{}() ADA-FS daemon pid file contains no mountdir path. Exiting ...", __func__); + CTX->log()->error("{}() daemon pid file contains no mountdir path. Exiting ...", __func__); ifs.close(); return -1; } } else { - cerr << "No permission to open pid file at " << daemon_pid_path() - << " or ADA-FS daemon pid file not found. Daemon not running?" << endl; + cerr << "Failed to to open pid file at '" << daemon_pid_path() + << "'. Daemon not running?" << endl; CTX->log()->error( "{}() Failed to open pid file '{}'. Error: {}", __func__, daemon_pid_path(), std::strerror(errno)); diff --git a/src/client/resolve.cpp b/src/client/resolve.cpp index 9d578d0fa016c69a1a915aafa03eafbc07f2e2ab..5b57d976642b22f4af69c26669f0cd6d2d71b148 100644 --- a/src/client/resolve.cpp +++ b/src/client/resolve.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include @@ -5,9 +18,13 @@ #include #include "global/path_util.hpp" +#include "global/configure.hpp" #include "client/passthrough.hpp" #include "client/preload.hpp" + +constexpr static const char * ENV_NAME_CWD = ENV_PREFIX "CWD"; + /* Match components in path * * Returns the number of consecutive components at start of `path` @@ -206,7 +223,7 @@ void set_sys_cwd(const std::string& path) { void set_env_cwd(const std::string& path) { CTX->log()->debug("{}() to '{}'", __func__, path); - if(setenv("ADAFS_CWD", path.c_str(), 1)) { + if(setenv(ENV_NAME_CWD, path.c_str(), 1)) { CTX->log()->error("{}() failed to set environment current working directory: {}", __func__, std::strerror(errno)); throw std::system_error(errno, @@ -217,7 +234,7 @@ void set_env_cwd(const std::string& path) { void unset_env_cwd() { CTX->log()->debug("{}()", __func__); - if(unsetenv("ADAFS_CWD")) { + if(unsetenv(ENV_NAME_CWD)) { CTX->log()->error("{}() failed to unset environment current working directory: {}", __func__, std::strerror(errno)); throw std::system_error(errno, @@ -227,7 +244,7 @@ void unset_env_cwd() { } void init_cwd() { - const char* env_cwd = std::getenv("ADAFS_CWD"); + const char* env_cwd = std::getenv(ENV_NAME_CWD); if (env_cwd != nullptr) { CTX->cwd(env_cwd); } else { diff --git a/src/client/rpc/ld_rpc_data_ws.cpp b/src/client/rpc/ld_rpc_data_ws.cpp index 4ee79449e608c9c53844dc7742d11fbf97d76d4b..0f19786e36fb589995933298bdcdb1d575c6a4e2 100644 --- a/src/client/rpc/ld_rpc_data_ws.cpp +++ b/src/client/rpc/ld_rpc_data_ws.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include diff --git a/src/client/rpc/ld_rpc_management.cpp b/src/client/rpc/ld_rpc_management.cpp index 2168b9f0012a2125b7da733c4ad9f3d7f7630b56..db14608d912f1834f7eb5b9c0623a9dbca7a5da5 100644 --- a/src/client/rpc/ld_rpc_management.cpp +++ b/src/client/rpc/ld_rpc_management.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include "client/rpc/ld_rpc_management.hpp" #include "global/rpc/rpc_types.hpp" #include diff --git a/src/client/rpc/ld_rpc_metadentry.cpp b/src/client/rpc/ld_rpc_metadentry.cpp index e68a8126215bf5d852677e296c38523d2726ab78..bf1f1b95464e704716977b5f5b6efdf95a933f3e 100644 --- a/src/client/rpc/ld_rpc_metadentry.cpp +++ b/src/client/rpc/ld_rpc_metadentry.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include "client/preload.hpp" diff --git a/src/daemon/CMakeLists.txt b/src/daemon/CMakeLists.txt index 79ad10a5d43228a48c7ae3ec6141e08f2d2deece..042370827492ce555a1fd08e3b64a0c2f5475e2b 100644 --- a/src/daemon/CMakeLists.txt +++ b/src/daemon/CMakeLists.txt @@ -5,8 +5,8 @@ set(DAEMON_SRC ../global/rpc/rpc_utils.cpp ../global/global_func.cpp ../global/path_util.cpp - adafs_daemon.cpp - adafs_ops/metadentry.cpp + main.cpp + ops/metadentry.cpp classes/fs_data.cpp classes/rpc_data.cpp handler/h_metadentry.cpp @@ -14,20 +14,21 @@ set(DAEMON_SRC handler/h_preload.cpp ) set(DAEMON_HEADERS + ../../include/version.hpp ../../include/global/configure.hpp ../../include/global/global_defs.hpp ../../include/global/global_func.hpp ../../include/global/rpc/rpc_types.hpp ../../include/global/rpc/rpc_utils.hpp ../../include/global/path_util.hpp - ../../include/daemon/adafs_daemon.hpp - ../../include/daemon/adafs_ops/metadentry.hpp + ../../include/daemon/main.hpp + ../../include/daemon/ops/metadentry.hpp ../../include/daemon/classes/fs_data.hpp ../../include/daemon/classes/rpc_data.hpp ../../include/daemon/handler/rpc_defs.hpp ) -add_executable(adafs_daemon ${DAEMON_SRC} ${DAEMON_HEADERS}) -target_link_libraries(adafs_daemon +add_executable(gkfs_daemon ${DAEMON_SRC} ${DAEMON_HEADERS}) +target_link_libraries(gkfs_daemon # internal libs metadata metadata_db @@ -46,8 +47,12 @@ target_link_libraries(adafs_daemon Threads::Threads ) -target_include_directories(adafs_daemon +target_include_directories(gkfs_daemon PRIVATE ${ABT_INCLUDE_DIRS} ${MARGO_INCLUDE_DIRS} ) + +install(TARGETS gkfs_daemon + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +) diff --git a/src/daemon/backend/data/CMakeLists.txt b/src/daemon/backend/data/CMakeLists.txt index 1b333a82508db14aa9a5c013faea077988b79daa..41f1ec0f998a2e31fd26fcf595312e1706ff239e 100644 --- a/src/daemon/backend/data/CMakeLists.txt +++ b/src/daemon/backend/data/CMakeLists.txt @@ -5,7 +5,6 @@ target_sources(storage ${INCLUDE_DIR}/daemon/backend/data/chunk_storage.hpp PRIVATE ${INCLUDE_DIR}/global/path_util.hpp - ${INCLUDE_DIR}/extern/spdlog/spdlog.h ${CMAKE_CURRENT_LIST_DIR}/chunk_storage.cpp ) diff --git a/src/daemon/backend/data/chunk_storage.cpp b/src/daemon/backend/data/chunk_storage.cpp index e331ce9976fc0c09132f7b2ade979f1d70e3bcf8..cdc839f04e450554fe3efc92f26953468d6b0e51 100644 --- a/src/daemon/backend/data/chunk_storage.cpp +++ b/src/daemon/backend/data/chunk_storage.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include diff --git a/src/daemon/backend/metadata/db.cpp b/src/daemon/backend/metadata/db.cpp index df31964c810edd61acf63cbb618fb2b2c3c4ab3f..f41a69a7b00d52528d8dad70c1e1c7bd99d2b1c1 100644 --- a/src/daemon/backend/metadata/db.cpp +++ b/src/daemon/backend/metadata/db.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include @@ -169,7 +182,6 @@ void MetadataDB::iterate_all() { for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { key = iter->key().ToString(); val = iter->value().ToString(); - //TODO ADAFS_DATA->spdlogger()->trace("key '{}' value '{}'", key, val); } } diff --git a/src/daemon/backend/metadata/merge.cpp b/src/daemon/backend/metadata/merge.cpp index 63e8a49171c4958d17cf30f9936611b1169d532e..9ef371124c16f233704bd8f2a5bea248fc5887c7 100644 --- a/src/daemon/backend/metadata/merge.cpp +++ b/src/daemon/backend/metadata/merge.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include diff --git a/src/daemon/classes/fs_data.cpp b/src/daemon/classes/fs_data.cpp index ee0e2b08dc37d66f82efde4371f5d64d3d331108..3e96988797a302fb51edf0d6f6f60cf2f67f5f13 100644 --- a/src/daemon/classes/fs_data.cpp +++ b/src/daemon/classes/fs_data.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include diff --git a/src/daemon/classes/rpc_data.cpp b/src/daemon/classes/rpc_data.cpp index 0462731faba96d0b9e3396fb0c92fd6e0493c3a9..b04b4c2b5cd3cfd761ebc80bafb2cb42acf8e5ed 100644 --- a/src/daemon/classes/rpc_data.cpp +++ b/src/daemon/classes/rpc_data.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include diff --git a/src/daemon/handler/h_data.cpp b/src/daemon/handler/h_data.cpp index b26a5b17775386a812b321311b8e8cefbb0f2b1a..869dd666d30377ae49d529ee66c2422ca46d6e3c 100644 --- a/src/daemon/handler/h_data.cpp +++ b/src/daemon/handler/h_data.cpp @@ -1,10 +1,23 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include #include #include -#include +#include #include diff --git a/src/daemon/handler/h_metadentry.cpp b/src/daemon/handler/h_metadentry.cpp index 4e62747a150a82ad0477a1a4cb21d0e263ac1a06..ce2b4bef45ead8430215f62bbb7f6937598297fb 100644 --- a/src/daemon/handler/h_metadentry.cpp +++ b/src/daemon/handler/h_metadentry.cpp @@ -1,10 +1,23 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include #include -#include +#include using namespace std; diff --git a/src/daemon/handler/h_preload.cpp b/src/daemon/handler/h_preload.cpp index 9219ee2ad04c858634b2078c6bc7ea8b8dbaa117..1f6914ff2af43a6d65768ed14c90b26b86bc94c2 100644 --- a/src/daemon/handler/h_preload.cpp +++ b/src/daemon/handler/h_preload.cpp @@ -1,5 +1,18 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany -#include + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + + +#include #include #include "global/rpc/rpc_types.hpp" diff --git a/src/daemon/adafs_daemon.cpp b/src/daemon/main.cpp similarity index 93% rename from src/daemon/adafs_daemon.cpp rename to src/daemon/main.cpp index dc59ebbadedeed2778b427b264b6f2e0d83140fb..51267e7d42f3cf09efe1e4d0f5428a3f3debd653 100644 --- a/src/daemon/adafs_daemon.cpp +++ b/src/daemon/main.cpp @@ -1,11 +1,25 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany -#include + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + + +#include +#include "version.hpp" #include #include #include #include #include -#include +#include #include #include @@ -132,7 +146,7 @@ void destroy_enviroment() { ADAFS_DATA->spdlogger()->info("{}() Closing DB...", __func__); ADAFS_DATA->close_mdb(); - ADAFS_DATA->spdlogger()->info("All services shut down. ADA-FS shutdown complete."); + ADAFS_DATA->spdlogger()->info("Shutdown complete"); } bool init_io_tasklet_pool() { @@ -258,7 +272,7 @@ void register_daemon_proc() { } auto pid_file = daemon_pid_path(); - // check if a pid file exists from another adafs_daemon + // check if a pid file exists from another daemon if (bfs::exists(pid_file)) { throw runtime_error( fmt::format("Pid file already exists, " @@ -338,14 +352,15 @@ int main(int argc, const char* argv[]) { desc.add_options() ("help,h", "Help message") ("mountdir,m", po::value()->required(), "User Fuse mountdir") - ("rootdir,r", po::value()->required(), "ADA-FS data directory") - ("metadir,i", po::value(), "ADA-FS metadata directory, if not set rootdir is used for metadata ") + ("rootdir,r", po::value()->required(), "data directory") + ("metadir,i", po::value(), "metadata directory, if not set rootdir is used for metadata ") ("listen,l", po::value(), "Address or interface to bind the daemon on. Default: local hostname") ("port,p", po::value()->default_value(DEFAULT_RPC_PORT), "Port to bind the server on. Default: 4433") ("hostfile", po::value(), "Path to the hosts_file for all fs participants") ("hosts,h", po::value(), "Comma separated list of hosts_ for all fs participants") ("hostname-suffix,s", po::value()->default_value(""), "Suffix that is added to each given host. Consult /etc/hosts for allowed suffixes") - ("lookup-file,k", po::value(), "Shared file used by deamons to register their enpoints. (Needs to be on a shared filesystem)"); + ("lookup-file,k", po::value(), "Shared file used by deamons to register their enpoints. (Needs to be on a shared filesystem)") + ("version,h", "print version and exit"); po::variables_map vm; po::store(po::parse_command_line(argc, argv, desc), vm); @@ -354,6 +369,18 @@ int main(int argc, const char* argv[]) { return 1; } + if (vm.count("version")) { + cout << GKFS_VERSION_STRING << endl; +#ifndef NDEBUG + cout << "Debug: ON" << endl; +#else + cout << "Debug: OFF" << endl; +#endif + cout << "RPC protocol: " << RPC_PROTOCOL << endl; + cout << "Chunk size: " << CHUNKSIZE << " bytes" << endl; + return 0; + } + try { po::notify(vm); } catch (po::required_option& e) { diff --git a/src/daemon/adafs_ops/metadentry.cpp b/src/daemon/ops/metadentry.cpp similarity index 83% rename from src/daemon/adafs_ops/metadentry.cpp rename to src/daemon/ops/metadentry.cpp index de085c93ca24990c6addc23ba993de522779a4ef..73d1d43ce4d2bc3931b77ed616f59f6025c5f608 100644 --- a/src/daemon/adafs_ops/metadentry.cpp +++ b/src/daemon/ops/metadentry.cpp @@ -1,5 +1,18 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany -#include + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + + +#include #include #include diff --git a/src/global/global_func.cpp b/src/global/global_func.cpp index 5058010dffe382a2fa6293d50ad197218521b899..8ce3eb82c372e5476b7305f50933b4c5c67214c2 100644 --- a/src/global/global_func.cpp +++ b/src/global/global_func.cpp @@ -1,7 +1,20 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include using namespace std; string daemon_pid_path() { - return (DAEMON_AUX_PATH + "/adafs_daemon.pid"s); + return (DAEMON_AUX_PATH + "/gkfs_daemon.pid"s); } diff --git a/src/global/log_util.cpp b/src/global/log_util.cpp index eab71758210f1896eba1d9343fc4b0f640b919d3..8105b8691b46c7a6782491b5f7cf5cb011ad9ffa 100644 --- a/src/global/log_util.cpp +++ b/src/global/log_util.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include diff --git a/src/global/metadata.cpp b/src/global/metadata.cpp index 3d9a057a3f7757c4c14234627e5b49391872070d..c33ac34569b90a2d2b49f802427d4faf631c060e 100644 --- a/src/global/metadata.cpp +++ b/src/global/metadata.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include "global/metadata.hpp" #include "global/configure.hpp" diff --git a/src/global/path_util.cpp b/src/global/path_util.cpp index 015b11ef63faf6bf48e513ef9c2ea865e9acc3e8..4f123749a146654d13646a0ef89a3f766d12dfbd 100644 --- a/src/global/path_util.cpp +++ b/src/global/path_util.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include #include diff --git a/src/global/rpc/distributor.cpp b/src/global/rpc/distributor.cpp index 6074eb23b0c5f9d8221eccda3b15c2f382ded19a..68c8ea592d9c6e38f46fcc16a64eec53e06bb429 100644 --- a/src/global/rpc/distributor.cpp +++ b/src/global/rpc/distributor.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include SimpleHashDistributor:: diff --git a/src/global/rpc/rpc_utils.cpp b/src/global/rpc/rpc_utils.cpp index 9b6dee9a92682067297bdbcfa40816620242dc91..3b25642c73003289ea57394ccc5d81d10ad593cc 100644 --- a/src/global/rpc/rpc_utils.cpp +++ b/src/global/rpc/rpc_utils.cpp @@ -1,3 +1,16 @@ +/* + Copyright 2018-2019, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2019, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + #include #include