Merge branch '89-task-daemon-doxygen-documentation' into 'master' (61faf7ac) · Commits · hpc / gekkofs

.gitignore

+1 −0

Original line number	Diff line number	Diff line
		@@ -64,6 +64,7 @@

		# .idea
		*.idea/
		.run/

		# OS generated files
		.DS_Store

include/daemon/backend/data/chunk_storage.hpp

+96 −8

Original line number	Diff line number	Diff line
		@@ -25,6 +25,10 @@

		SPDX-License-Identifier: GPL-3.0-or-later
		*/
		/**
		* @brief Chunk storage declarations handles all interactions with the
		* node-local storage system.
		*/

		#ifndef GEKKOFS_CHUNK_STORAGE_HPP
		#define GEKKOFS_CHUNK_STORAGE_HPP
		@@ -47,56 +51,140 @@ struct ChunkStat {
		unsigned long chunk_size;
		unsigned long chunk_total;
		unsigned long chunk_free;
		};
		}; //!< Struct for attaining current usage of storage backend

		/**
		* @brief Generic exception for ChunkStorage
		*/
		class ChunkStorageException : public std::system_error {
		public:
		ChunkStorageException(const int err_code, const std::string& s)
		: std::system_error(err_code, std::generic_category(), s){};
		};

		/**
		* @brief ChunkStorage class handles _all_ interaction with node-local storage
		* system and is run as a single instance within the GekkoFS daemon.
		*/
		class ChunkStorage {
		private:
		std::shared_ptr<spdlog::logger> log_;
		std::shared_ptr<spdlog::logger> log_; //!< Class logger

		std::string root_path_;
		size_t chunksize_;
		std::string root_path_; //!< Path to GekkoFS root directory
		size_t chunksize_; //!< File system chunksize. TODO Why does that exist?

		inline std::string
		/**
		* @brief Converts an internal gkfs path under the root dir to the absolute
		* path of the system.
		* @param internal_path E.g., /foo/bar
		* @return Absolute path, e.g., /tmp/rootdir/<pid>/data/chunks/foo:bar
		*/
		[[nodiscard]] inline std::string
		absolute(const std::string& internal_path) const;

		/**
		* @brief Returns the chunk dir directory for a given path which is expected
		* to be absolute.
		* @param file_path
		* @return Chunk dir path
		*/
		static inline std::string
		get_chunks_dir(const std::string& file_path);

		/**
		* @brief Returns the backend chunk file path for a given internal path.
		* @param file_path Internal file path, e.g., /foo/bar
		* @param chunk_id Number of chunk id
		* @return Chunk file path, e.g., /foo/bar
		* /tmp/rootdir/<pid>>/data/chunks/foo:bar/0
		*/
		static inline std::string
		get_chunk_path(const std::string& file_path, gkfs::rpc::chnk_id_t chunk_id);

		/**
		* @brief Initializes the chunk space for a GekkoFS file, creating its
		* directory on the local file system.
		* @param file_path Chunk file path, e.g., /foo/bar
		*/
		void
		init_chunk_space(const std::string& file_path) const;

		public:
		/**
		* @brief Initializes the ChunkStorage object on daemon launch.
		* @param path Root directory where all data is placed on the local FS.
		* @param chunksize Used chunksize in this GekkoFS instance.
		* @throws ChunkStorageException on launch failure
		*/
		ChunkStorage(std::string& path, size_t chunksize);

		/**
		* @brief Removes chunk directory with all its files which is a recursive
		* remove operation on the chunk directory.
		* @param file_path Chunk file path, e.g., /foo/bar
		* @throws ChunkStorageException
		*/
		void
		destroy_chunk_space(const std::string& file_path) const;

		/**
		* @brief Writes a single chunk file and is usually called by an Argobots
		* tasklet.
		* @param file_path Chunk file path, e.g., /foo/bar
		* @param chunk_id Number of chunk id
		* @param buf Buffer to write to chunk
		* @param size Amount of bytes to write to the chunk file
		* @param offset Offset where to write to the chunk file
		* @return The amount of bytes written
		* @throws ChunkStorageException with its error code
		*/
		ssize_t
		write_chunk(const std::string& file_path, gkfs::rpc::chnk_id_t chunk_id,
		const char* buf, size_t size, off64_t offset) const;

		/**
		* @brief Reads a single chunk file and is usually called by an Argobots
		* tasklet.
		* @param file_path Chunk file path, e.g., /foo/bar
		* @param chunk_id Number of chunk id
		* @param buf Buffer to read to from chunk
		* @param size Amount of bytes to read to the chunk file
		* @param offset Offset where to read from the chunk file
		* @return The amount of bytes read
		* @throws ChunkStorageException with its error code
		*/
		ssize_t
		read_chunk(const std::string& file_path, gkfs::rpc::chnk_id_t chunk_id,
		char* buf, size_t size, off64_t offset) const;

		/**
		* @brief Delete all chunks starting with chunk a chunk id.
		* @param file_path Chunk file path, e.g., /foo/bar
		* @param chunk_start Number of chunk id
		* @throws ChunkStorageException with its error code
		*/
		void
		trim_chunk_space(const std::string& file_path,
		gkfs::rpc::chnk_id_t chunk_start);

		/**
		* @brief Truncates a single chunk file to a given byte length.
		* @param file_path Chunk file path, e.g., /foo/bar
		* @param chunk_id Number of chunk id
		* @param length Length of bytes to truncate the chunk to
		* @throws ChunkStorageException
		*/
		void
		truncate_chunk_file(const std::string& file_path,
		gkfs::rpc::chnk_id_t chunk_id, off_t length);

		ChunkStat
		/**
		* @brief Calls statfs on the chunk directory to get statistic on its used
		* storage space.
		* @return ChunkStat struct
		* @throws ChunkStorageException
		*/
		[[nodiscard]] ChunkStat
		chunk_stat() const;
		};

include/daemon/backend/data/data_module.hpp

+22 −4

Original line number	Diff line number	Diff line
		@@ -25,6 +25,10 @@

		SPDX-License-Identifier: GPL-3.0-or-later
		*/
		/**
		* @brief Declaration for GekkoFS data module object that is run once per daemon
		* instance.
		*/

		#ifndef GEKKOFS_DAEMON_DATA_LOGGING_HPP
		#define GEKKOFS_DAEMON_DATA_LOGGING_HPP
		@@ -33,12 +37,16 @@

		namespace gkfs::data {

		/**
		* @brief The data module class providing the data backend for the daemon as a
		* singleton.
		*/
		class DataModule {

		private:
		DataModule() {}
		DataModule() = default;

		std::shared_ptr<spdlog::logger> log_;
		std::shared_ptr<spdlog::logger> log_; ///< Logging instance for data backend

		public:
		static constexpr const char* LOGGER_NAME = "DataModule";
		@@ -54,16 +62,26 @@ public:
		void
		operator=(DataModule const&) = delete;

		const std::shared_ptr<spdlog::logger>&
		/**
		* @brief Returns the data module log handle.
		* @return Pointer to the spdlog instance
		*/
		[[nodiscard]] const std::shared_ptr<spdlog::logger>&
		log() const;

		/**
		* @brief Attaches a logging instance to the data module.
		* @param log spdlog shared pointer instance
		*/
		void
		log(const std::shared_ptr<spdlog::logger>& log);
		};

		#define GKFS_DATA_MOD \
		(static_cast<gkfs::data::DataModule*>( \
		gkfs::data::DataModule::getInstance()))
		gkfs::data::DataModule::getInstance())) ///< macro to access the
		///< DataModule singleton
		///< across the daemon

		} // namespace gkfs::data

include/daemon/backend/data/file_handle.hpp

+26 −11

Original line number	Diff line number	Diff line
		@@ -25,7 +25,10 @@

		SPDX-License-Identifier: GPL-3.0-or-later
		*/

		/**
		* @brief Definitions for the file handle abstraction layer used for the backend
		* file system.
		*/
		#ifndef GEKKOFS_DAEMON_FILE_HANDLE_HPP
		#define GEKKOFS_DAEMON_FILE_HANDLE_HPP

		@@ -38,21 +41,21 @@ extern "C" {
		namespace gkfs::data {

		/**
		* File handle to encapsulate a file descriptor, allowing RAII closing of the
		* file descriptor
		* @brief File handle class to encapsulate a file descriptor, allowing RAII
		* closing of the file descriptor.
		*/
		class FileHandle {

		private:
		constexpr static const int init_value{-1};
		constexpr static const int init_value{-1}; ///< initial file descriptor

		int fd_{init_value};
		std::string path_{};
		int fd_{init_value}; ///< file descriptor
		std::string path_{}; ///< chunk file path

		public:
		FileHandle() = default;

		explicit FileHandle(int fd, std::string path) noexcept : fd_(fd) {}
		explicit FileHandle(int fd, const std::string& path) noexcept : fd_(fd) {}

		FileHandle(FileHandle&& rhs) = default;

		@@ -73,19 +76,28 @@ public:
		return !valid();
		}

		bool
		/**
		* @brief Checks for valid file descriptor value.
		* @return boolean if valid file descriptor
		*/
		[[nodiscard]] bool
		valid() const noexcept {
		return fd_ != init_value;
		}

		int
		/**
		* @brief Retusn the file descriptor value used in this file handle
		* operation.
		* @return file descriptor value
		*/
		[[nodiscard]] int
		native() const noexcept {
		return fd_;
		}

		/**
		* Closes file descriptor and resets it to initial value
		* @return
		* @brief Closes file descriptor and resets it to initial value
		* @return boolean if file descriptor was successfully closed
		*/
		bool
		close() noexcept {
		@@ -101,6 +113,9 @@ public:
		return true;
		}

		/**
		* @brief Destructor implicitly closes the internal file descriptor.
		*/
		~FileHandle() {
		if(fd_ != init_value)
		close();

include/daemon/backend/metadata/db.hpp

+104 −10

Original line number	Diff line number	Diff line
		@@ -25,7 +25,10 @@

		SPDX-License-Identifier: GPL-3.0-or-later
		*/

		/**
		* @brief Class declaration for MetadataDB class which uses RocksDB and provides
		* a single instance within the daemon.
		*/
		#ifndef GEKKOFS_METADATA_DB_HPP
		#define GEKKOFS_METADATA_DB_HPP

		@@ -39,54 +42,145 @@ namespace rdb = rocksdb;

		namespace gkfs::metadata {

		/**
		* @brief MetadataDB class providing an abstraction layer to the KV store
		* RocksDB.
		*/
		class MetadataDB {
		private:
		std::unique_ptr<rdb::DB> db;
		rdb::Options options;
		rdb::WriteOptions write_opts;
		std::string path;
		std::shared_ptr<spdlog::logger> log_;

		std::unique_ptr<rdb::DB> db; ///< RocksDB instance
		rdb::Options options; ///< RocksDB configuration
		rdb::WriteOptions write_opts; ///< RocksDB write configuration
		std::string path; ///< Path to where RocksDB persists its data
		std::shared_ptr<spdlog::logger> log_; ///< MetadataDB internal logger

		/**
		* @brief Sets up specific settings to optimize RocksDB instance to
		* environment on launch.
		* @param options RocksDB configurations
		*/
		static void
		optimize_rocksdb_options(rdb::Options& options);

		public:
		/**
		* @brief Exception wrapper on Status object.
		* @param s RocksDB status
		* @throws Throws NotFoundException if s.IsNotFound(), general DBException
		* otherwise
		*/
		static inline void
		throw_rdb_status_excpt(const rdb::Status& s);

		/**
		* @brief Constructor, called when daemon is started and connects to KV
		* store.
		* @param path Path to where RocksDB persists its data
		*/
		explicit MetadataDB(const std::string& path);

		std::string
		/**
		* @brief Gets the KV store value for a key.
		* @param key KV store key
		* @return KV store value
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		[[nodiscard]] std::string
		get(const std::string& key) const;

		/**
		* @brief Puts an entry into the KV store.
		* @param key KV store key
		* @param val KV store value
		* @throws DBException
		*/
		void
		put(const std::string& key, const std::string& val);

		/**
		* @brief Puts an entry into the KV store if it doesn't exist.
		* @param key KV store key
		* @param val KV store value
		* @throws DBException on failure, ExistException if entry already exists
		*/
		void
		put_no_exist(const std::string& key, const std::string& val);

		/**
		* @brief Removes an entry from the KV store.
		* @param key KV store key
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		void
		remove(const std::string& key);

		/**
		* @brief Checks for existence of an entry.
		* @param key KV store key
		* @return true if exists
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		bool
		exists(const std::string& key);

		/**
		* Updates a metadata entry atomically and also allows to change keys.
		* @param old_key KV store key to be replaced
		* @param new_key new KV store key
		* @param val KV store value
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		void
		update(const std::string& old_key, const std::string& new_key,
		const std::string& val);

		/**
		* @brief Increases only the size part of the metadata entry via a RocksDB
		* Operand.
		* @param key KV store key
		* @param size new size for entry
		* @param append
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		void
		increase_size(const std::string& key, size_t size, bool append);

		/**
		* @brief Decreases only the size part of the metadata entry via a RocksDB
		* Operand/
		* @param key KV store key
		* @param size new size for entry
		* @throws DBException on failure, NotFoundException if entry doesn't exist
		*/
		void
		decrease_size(const std::string& key, size_t size);

		std::vector<std::pair<std::string, bool>>
		/**
		* @brief Return all file names and modes for the first-level entries of the
		* given directory.
		* @param dir directory prefix string
		* @return vector of pair <std::string name, bool is_dir>,
		* where name is the name of the entries and is_dir
		* is true in the case the entry is a directory.
		*/
		[[nodiscard]] std::vector<std::pair<std::string, bool>>
		get_dirents(const std::string& dir) const;

		std::vector<std::tuple<std::string, bool, size_t, time_t>>
		/**
		* @brief Return all file names and modes for the first-level entries of the
		* given directory including their sizes and creation time.
		* @param dir directory prefix string
		* @return vector of pair <std::string name, bool is_dir - size - ctime>,
		* where name is the name of the entries and is_dir
		* is true in the case the entry is a directory.
		*/
		[[nodiscard]] std::vector<std::tuple<std::string, bool, size_t, time_t>>
		get_dirents_extended(const std::string& dir) const;

		/**
		* @brief Iterate over complete database, note ONLY used for debugging and
		* is therefore unused.
		*/
		void
		iterate_all();
		};