Merge remote-tracking branch... (8ea12c9e) · Commits · David Auer / gekkofs

.gitlab-ci.yml

+2 −1

Original line number	Diff line number	Diff line
		@@ -17,7 +17,8 @@ variables:
		# Configuration variables
		GKFS_LOG_LEVEL: "1"
		GKFS_DAEMON_LOG_PATH: "${CI_PROJECT_DIR}/logs/daemon.log"
		GKFS_PRELOAD_LOG_PATH: "${CI_PROJECT_DIR}/logs/preload.log"
		LIBGKFS_LOG: "all"
		LIBGKFS_LOG_OUTPUT: "${CI_PROJECT_DIR}/logs/gkfs_client.log"
		GIT_SUBMODULE_STRATEGY: recursive

		image: gekkofs/gekkofs:build_env

CMake/FindDate.cmake

0 → 100644

+40 −0

Original line number	Diff line number	Diff line
		find_path(DATE_INCLUDE_DIR
		NAMES date/date.h
		)

		find_path(TZ_INCLUDE_DIR
		NAMES date/tz.h
		)

		find_library(TZ_LIBRARY
		NAMES tz
		)

		include(FindPackageHandleStandardArgs)
		find_package_handle_standard_args( Date
		DEFAULT_MSG
		DATE_INCLUDE_DIR
		TZ_INCLUDE_DIR
		TZ_LIBRARY
		)

		if(Date_FOUND)
		set(DATE_INCLUDE_DIRS ${DATE_INCLUDE_DIR})
		set(TZ_INCLUDE_DIRS ${TZ_INCLUDE_DIR})
		set(TZ_LIBRARIES ${TZ_LIBRARY})

		if(NOT TARGET Date::TZ)
		add_library(Date::TZ UNKNOWN IMPORTED)
		set_target_properties(Date::TZ PROPERTIES
		IMPORTED_LOCATION "${TZ_LIBRARY}"
		INTERFACE_COMPILE_DEFINITIONS "USE_OS_TZDB=1"
		INTERFACE_INCLUDE_DIRECTORIES "${TZ_INCLUDE_DIR}"
		)
		endif()
		endif()

		mark_as_advanced(
		DATE_INCLUDE_DIR
		TZ_INCLUDE_DIR
		TZ_LIBRARY
		)

CMakeLists.txt

+27 −17

Original line number	Diff line number	Diff line
		@@ -27,9 +27,13 @@ message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
		# Compiler flags for various cmake build types
		set(WARNINGS_FLAGS "-Wall -Wextra --pedantic -Wno-unused-parameter -Wno-missing-field-initializers")
		set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DNDEBUG -O3")
		set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${WARNINGS_FLAGS} -g -O0")
		set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${WARNINGS_FLAGS} -g -O0 -DGKFS_DEBUG_BUILD")
		set(CMAKE_CXX_FLAGS_MEMCHECK "${WARNINGS_FLAGS} -g -O0 -fsanitize=address -fno-omit-frame-pointer")
		set(CMAKE_CXX_FLAGS_MAINTAINER "${WARNINGS_FLAGS} -g -O0 -pg -no-pie")
		set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -DNDEBUG -O3")
		set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${WARNINGS_FLAGS} -g -O0 -DGKFS_DEBUG_BUILD")
		set(CMAKE_C_FLAGS_MEMCHECK "${WARNINGS_FLAGS} -g -O0 -fsanitize=address -fno-omit-frame-pointer")
		set(CMAKE_C_FLAGS_MAINTAINER "${WARNINGS_FLAGS} -g -O0 -pg -no-pie")
		mark_as_advanced(CMAKE_CXX_FLAGS_MAINTAINER)

		# Project version
		@@ -94,6 +98,8 @@ find_package(Boost 1.53 REQUIRED

		find_package(Threads REQUIRED)

		find_package(Date REQUIRED)

		set(RPC_PROTOCOL "ofi+tcp" CACHE STRING "Communication plugin used for RPCs")
		set_property(CACHE RPC_PROTOCOL PROPERTY STRINGS
		"bmi+tcp"
		@@ -112,9 +118,10 @@ if(SYMLINK_SUPPORT)
		endif()
		message(STATUS "Symlink support: ${SYMLINK_SUPPORT}")

		option(USE_BITSET_FOR_INTERNAL_FDS "Use std::bitset to track internal fds" ON)
		if(USE_BITSET_FOR_INTERNAL_FDS)
		add_definitions(-DUSE_BITSET_FOR_INTERNAL_FDS)
		set(MAX_INTERNAL_FDS 256 CACHE STRING "Number of file descriptors reserved for internal use")
		add_definitions(-DMAX_INTERNAL_FDS=${MAX_INTERNAL_FDS})
		message(STATUS "File descriptors reserved for internal use: ${MAX_INTERNAL_FDS}")

		execute_process(COMMAND getconf OPEN_MAX
		OUTPUT_VARIABLE GETCONF_MAX_FDS
		OUTPUT_STRIP_TRAILING_WHITESPACE
		@@ -123,13 +130,16 @@ if(USE_BITSET_FOR_INTERNAL_FDS)
		set(GETCONF_MAX_FDS=512)
		endif()
		add_definitions(-DMAX_OPEN_FDS=${GETCONF_MAX_FDS})
		endif()

		message(STATUS "Use std::bitset for internal fd tracking: ${USE_BITSET_FOR_INTERNAL_FDS}")

		if(USE_BITSET_FOR_INTERNAL_FDS)
		message(STATUS "Max open files: ${GETCONF_MAX_FDS}")
		option(ENABLE_CLIENT_LOG "Enable logging messages" ON)
		if(ENABLE_CLIENT_LOG)
		set(CLIENT_LOG_MESSAGE_SIZE 1024 CACHE STRING "Maximum size of a log message in the client library")
		add_definitions(-DLIBGKFS_LOG_MESSAGE_SIZE=${CLIENT_LOG_MESSAGE_SIZE})
		message(STATUS "Maximum log message size in the client library: ${CLIENT_LOG_MESSAGE_SIZE}")
		else()
		add_definitions(-DGKFS_DISABLE_LOGGING)
		endif()
		message(STATUS "Client logging output: ${ENABLE_CLIENT_LOGGING}")

		configure_file(include/global/configure.hpp.in include/global/configure.hpp)

README.md

+64 −19

Original line number	Diff line number	Diff line
		@@ -42,7 +42,7 @@ This is a file system.
		(execute the script for help):

		```bash
		usage: dl_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>]
		usage: dl_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>]
		source_path


		@@ -54,18 +54,23 @@ positional arguments:

		optional arguments:
		-h, --help shows this help message and exits
		-l, --list-dependencies
		list dependencies available for download
		-n <NAPLUGIN>, --na <NAPLUGIN>
		network layer that is used for communication. Valid: {bmi,ofi,all}
		defaults to 'all'
		-c <CLUSTER>, --cluster <CLUSTER>
		additional configurations for specific compute clusters
		supported clusters: {mogon1,fh2}
		supported clusters: {mogon1,mogon2,fh2}
		-d <DEPENDENCY>, --dependency <DEPENDENCY>
		download a specific dependency. If unspecified
		all dependencies are built and installed.
		```
		- Now use the install script to compile them and install them to the desired directory. You can choose the according
		na_plugin (execute the script for help):

		```bash
		usage: compile_dep.sh [-h] [-n <NAPLUGIN>] [-c <CLUSTER>] [-j <COMPILE_CORES>]
		usage: compile_dep.sh [-h] [-l] [-n <NAPLUGIN>] [-c <CLUSTER>] [-d <DEPENDENCY>] [-j <COMPILE_CORES>]
		source_path install_path


		@@ -78,15 +83,21 @@ positional arguments:

		optional arguments:
		-h, --help shows this help message and exits
		-l, --list-dependencies
		list dependencies available for building and installation
		-n <NAPLUGIN>, --na <NAPLUGIN>
		network layer that is used for communication. Valid: {bmi,ofi,all}
		defaults to 'all'
		-c <CLUSTER>, --cluster <CLUSTER>
		additional configurations for specific compute clusters
		supported clusters: {mogon1,mogon2,fh2}
		-d <DEPENDENCY>, --dependency <DEPENDENCY>
		build and install a specific dependency. If unspecified
		all dependencies are built and installed.
		-j <COMPILE_CORES>, --compilecores <COMPILE_CORES>
		number of cores that are used to compile the depdencies
		number of cores that are used to compile the dependencies
		defaults to number of available cores
		-t, --test Perform libraries tests.
		```

		## Compile GekkoFS
		@@ -127,11 +138,45 @@ Run the application with the preload library: `LD_PRELOAD=<path>/build/lib/libio
		an MPI application use the `{mpirun, mpiexec} -x` argument.

		### Logging
		To enable logging the following environment variables are used:
		GKFS_PRELOAD_LOG_PATH="<path/to/file>" to set the path to the logging file of the client library.
		GKFS_DAEMON_LOG_PATH="<path/to/file>" to set the path to the logging file of the daemon.
		GKFS_LOG_LEVEL={off,critical,err,warn,info,debug,trace} to set the trace level verbosity.
		Numbers from 0-6 may also be used where as 0 is off and 6 represents trace.
		The following environment variables can be used to enable logging in the client
		library: `LIBGKFS_LOG=<module>` and `LIBGKFS_LOG_OUTPUT=<path/to/file>` to
		configure the output module and set the path to the log file of the client
		library. If not path is specified in `LIBGKFS_LOG_OUTPUT`, the client library
		will send log messages to `/tmp/gkfs_client.log`.

		The following modules are available:

		- `none`: don't print any messages
		- `syscalls`: Trace system calls: print the name of each system call, its
		arguments, and its return value. All system calls are printed after being
		executed save for those that may not return, such as `execve()`,
		`execve_at()`, `exit()`, and `exit_group()`. This module will only be
		available if the client library is built in `Debug` mode.
		- `syscalls_at_entry`: Trace system calls: print the name of each system call
		and its arguments. All system calls are printed before being executed and
		therefore their return values are not available in the log. This module will
		only be available if the client library is built in `Debug` mode.
		- `info`: Print information messages.
		- `critical`: Print critical errors.
		- `errors`: Print errors.
		- `warnings`: Print warnings.
		- `mercury`: Print Mercury messages.
		- `debug`: Print debug messages. This module will only be available if the
		client library is built in `Debug` mode.
		- `most`: All previous options combined except `syscalls_at_entry`. This
		module will only be available if the client library is built in `Debug`
		mode.
		- `all`: All previous options combined.
		- `help`: Print a help message and exit.

		Additionally, setting the `LIBGKFS_LOG_OUTPUT_TRUNC` environment variable with
		a value different from `0` will instruct the logging subsystem to truncate
		the file used for logging, rather than append to it.

		For the daemon, the `GKFS_DAEMON_LOG_PATH=<path/to/file>` environment variable
		can be provided to set the path to the log file, and the log module can be
		selected with the `GKFS_LOG_LEVEL={off,critical,err,warn,info,debug,trace}`
		environment variable.


		### Acknowledgment

hermes @ 2e578554

Original line number	Diff line number	Diff line
		Subproject commit 38b6bbfe77806b527f4e4b4157feea658f555be6
		Subproject commit 2e578554d52d734eec83b5def2602dde7b6ce570