server: Add posix_file module (da593ead) · Commits · hpc / cargo

src/CMakeLists.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -26,6 +26,7 @@ add_subdirectory(utils)
		add_subdirectory(config)
		add_subdirectory(logger)
		add_subdirectory(net)
		add_subdirectory(posix_file)

		## The main executable for the Cargo data stager
		add_executable(cargo_server)
		@@ -57,6 +58,7 @@ target_link_libraries(
		Boost::program_options
		Boost::serialization
		Boost::mpi
		posix_file
		)

		set_target_properties(cargo_server PROPERTIES OUTPUT_NAME "cargo")

src/posix_file/CMakeLists.txt

0 → 100644

+42 −0

Original line number	Diff line number	Diff line
		################################################################################
		# Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain #
		# #
		# This software was partially supported by the EuroHPC-funded project ADMIRE #
		# (Project ID: 956748, https://www.admire-eurohpc.eu). #
		# #
		# This file is part of Cargo. #
		# #
		# Cargo is free software: you can redistribute it and/or modify #
		# it under the terms of the GNU General Public License as published by #
		# the Free Software Foundation, either version 3 of the License, or #
		# (at your option) any later version. #
		# #
		# Cargo is distributed in the hope that it will be useful, #
		# but WITHOUT ANY WARRANTY; without even the implied warranty of #
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
		# GNU General Public License for more details. #
		# #
		# You should have received a copy of the GNU General Public License #
		# along with Cargo. If not, see <https://www.gnu.org/licenses/>. #
		# #
		# SPDX-License-Identifier: GPL-3.0-or-later #
		################################################################################

		add_library(posix_file INTERFACE)

		target_sources(
		posix_file
		INTERFACE posix_file/types.hpp
		posix_file/file.hpp
		posix_file/ranges.hpp
		posix_file/views.hpp
		posix_file/math.hpp
		posix_file/views/block_iterator.hpp
		posix_file/views/strided_iterator.hpp
		)

		target_include_directories(posix_file INTERFACE ${CMAKE_CURRENT_SOURCE_DIR})

		set_property(TARGET posix_file PROPERTY POSITION_INDEPENDENT_CODE ON)

		target_link_libraries(posix_file INTERFACE fmt::fmt tl::expected)

src/posix_file/posix_file/file.hpp

0 → 100644

+275 −0

Original line number	Diff line number	Diff line
		/******************************************************************************
		* Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain
		*
		* This software was partially supported by the EuroHPC-funded project ADMIRE
		* (Project ID: 956748, https://www.admire-eurohpc.eu).
		*
		* This file is part of Cargo.
		*
		* Cargo is free software: you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation, either version 3 of the License, or
		* (at your option) any later version.
		*
		* Cargo is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with Cargo. If not, see <https://www.gnu.org/licenses/>.
		*
		* SPDX-License-Identifier: GPL-3.0-or-later
		*****************************************************************************/

		#ifndef POSIX_FILE_FILE_HPP
		#define POSIX_FILE_FILE_HPP

		#include "types.hpp"

		#include <filesystem>
		#include <utility>
		#include <fcntl.h>
		#include <tl/expected.hpp>

		extern "C" {
		#include <unistd.h>
		};

		namespace posix_file {

		class file_handle {

		private:
		constexpr static const int init_value{-1}; ///< initial file descriptor

		int m_fd{init_value}; ///< file descriptor

		public:
		file_handle() = default;

		explicit file_handle(int fd) noexcept : m_fd(fd) {}

		file_handle(file_handle&& rhs) noexcept {
		this->m_fd = rhs.m_fd;
		rhs.m_fd = init_value;
		}

		file_handle(const file_handle& other) = delete;

		file_handle&
		operator=(file_handle&& rhs) noexcept {
		this->m_fd = rhs.m_fd;
		rhs.m_fd = init_value;
		return *this;
		}

		file_handle&
		operator=(const file_handle& other) = delete;

		explicit operator bool() const noexcept {
		return valid();
		}

		bool
		operator!() const noexcept {
		return !valid();
		}

		/**
		* @brief Checks for valid file descriptor value.
		* @return boolean if valid file descriptor
		*/
		[[nodiscard]] bool
		valid() const noexcept {
		return m_fd != init_value;
		}

		/**
		* @brief Retusn the file descriptor value used in this file handle
		* operation.
		* @return file descriptor value
		*/
		[[nodiscard]] int
		native() const noexcept {
		return m_fd;
		}

		/**
		* @brief Closes file descriptor and resets it to initial value
		* @return boolean if file descriptor was successfully closed
		*/
		bool
		close() noexcept {
		if(m_fd != init_value) {
		if(::close(m_fd) < 0) {
		return false;
		}
		}
		m_fd = init_value;
		return true;
		}

		/**
		* @brief Destructor implicitly closes the internal file descriptor.
		*/
		~file_handle() {
		if(m_fd != init_value) {
		close();
		}
		}
		};

		class file {

		public:
		explicit file(std::filesystem::path filepath) noexcept
		: m_path(std::move(filepath)) {}

		file(std::filesystem::path filepath, int fd) noexcept
		: m_path(std::move(filepath)), m_handle(fd) {}

		std::filesystem::path
		path() const noexcept {
		return m_path;
		}

		posix_file::offset
		eof() const noexcept {
		return static_cast<posix_file::offset>(size());
		}

		std::size_t
		size() const noexcept {
		return std::filesystem::file_size(m_path);
		}

		auto
		remove() noexcept {
		return std::filesystem::remove(m_path);
		}

		tl::expected<void, std::error_code>
		fallocate(int mode, offset offset, std::size_t len) const noexcept {

		if(!m_handle) {
		return tl::make_unexpected(
		std::error_code{EBADF, std::generic_category()});
		}

		int ret = ::fallocate(m_handle.native(), mode, offset,
		static_cast<off_t>(len));

		if(ret == -1) {
		return tl::make_unexpected(
		std::error_code{errno, std::generic_category()});
		}

		return {};
		}

		template <typename MemoryBuffer>
		tl::expected<std::size_t, std::error_code>
		pread(MemoryBuffer&& buf, offset offset, std::size_t size) const noexcept {

		assert(buf.size() >= size);

		if(!m_handle) {
		return tl::make_unexpected(
		std::error_code{EBADF, std::generic_category()});
		}

		std::size_t bytes_read = 0;
		std::size_t bytes_left = size;

		while(bytes_read < size) {

		ssize_t n = ::pread(m_handle.native(), buf.data() + bytes_read,
		bytes_left, offset + bytes_read);

		if(n == 0) {
		// EOF
		return 0;
		}

		if(n == -1) {
		// Interrupted by a signal, retry
		if(errno == EINTR) {
		continue;
		}

		// Some other error condition, report
		return tl::make_unexpected(
		std::error_code{errno, std::generic_category()});
		}

		bytes_read += n;
		bytes_left -= n;
		}

		return bytes_read;
		}

		template <typename MemoryBuffer>
		tl::expected<std::size_t, std::error_code>
		pwrite(MemoryBuffer&& buf, offset offset, std::size_t size) const noexcept {

		assert(buf.size() >= size);

		if(!m_handle) {
		return tl::make_unexpected(
		std::error_code{EBADF, std::generic_category()});
		}

		std::size_t bytes_written = 0;
		std::size_t bytes_left = size;

		while(bytes_written < size) {

		ssize_t n = ::pwrite(m_handle.native(), buf.data() + bytes_written,
		bytes_left, offset + bytes_written);

		if(n == -1) {
		// Interrupted by a signal, retry
		if(errno == EINTR) {
		continue;
		}

		// Some other error condition, report
		return tl::make_unexpected(
		std::error_code{errno, std::generic_category()});
		}

		bytes_written += n;
		bytes_left -= n;
		}

		return bytes_written;
		}

		protected:
		const std::filesystem::path m_path;
		file_handle m_handle;
		};

		static inline tl::expected<file, std::error_code>
		open(const std::filesystem::path& filepath, int flags, ::mode_t mode = 0) {

		int fd = ::open(filepath.c_str(), flags, mode);

		if(fd == -1) {
		return tl::make_unexpected(
		std::error_code{errno, std::generic_category()});
		}

		return file{filepath, fd};
		}

		static inline tl::expected<file, std::error_code>
		create(const std::filesystem::path& filepath, int flags, ::mode_t mode) {
		return open(filepath, O_CREAT \| flags, mode);
		}

		} // namespace posix_file

		#endif // POSIX_FILE_FILE_HPP

src/posix_file/posix_file/math.hpp

0 → 100644

+216 −0

Original line number	Diff line number	Diff line
		/*
		Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
		Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany

		This software was partially supported by the
		EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).

		This software was partially supported by the
		ADA-FS project under the SPPEXA project funded by the DFG.

		This file is part of GekkoFS.

		GekkoFS is free software: you can redistribute it and/or modify
		it under the terms of the GNU General Public License as published by
		the Free Software Foundation, either version 3 of the License, or
		(at your option) any later version.

		GekkoFS is distributed in the hope that it will be useful,
		but WITHOUT ANY WARRANTY; without even the implied warranty of
		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		GNU General Public License for more details.

		You should have received a copy of the GNU General Public License
		along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.

		SPDX-License-Identifier: GPL-3.0-or-later
		*/

		#ifndef POSIX_FILE_MATH_HPP
		#define POSIX_FILE_MATH_HPP

		#include <cstdint>
		#include <unistd.h>
		#include <cassert>

		namespace posix_file::math {

		/**
		* Check whether integer `n` is a power of 2.
		*
		* @param [in] n the number to check.
		* @returns `true` if `n` is a power of 2; `false` otherwise.
		*/
		constexpr bool
		is_power_of_2(uint64_t n) {
		return n && (!(n & (n - 1u)));
		}

		/**
		* Compute the base2 logarithm for 64 bit integers.
		*
		* @param [in] n the number from which to compute the log2.
		* @returns the base 2 logarithm of `n`.
		*/
		constexpr std::size_t
		log2(uint64_t n) {
		return 8u * sizeof(uint64_t) - __builtin_clzll(n) - 1;
		}

		/**
		* Check whether @n is aligned to a block boundary, i.e. if it is divisible by
		* @block_size.
		*
		* @note This function assumes that block_size is a power of 2.
		*
		* @param [in] n the number to check.
		* @param [in] block_size
		* @returns true if @n is divisible by @block_size; false otherwise.
		*/
		constexpr bool
		is_aligned(const uint64_t n, const size_t block_size) {
		using math::log2;
		assert(is_power_of_2(block_size));
		return !(n & ((1u << log2(block_size)) - 1));
		}

		/**
		* Given a file @offset and a @block_size, align the @offset to its
		* closest left-side block boundary.
		*
		* @note This function assumes that block_size is a power of 2.
		*
		* @param [in] offset the offset to align.
		* @param [in] block_size the block size used to compute boundaries.
		* @returns an offset aligned to the left-side block boundary.
		*/
		constexpr uint64_t
		align_left(const uint64_t offset, const size_t block_size) {
		// This check is automatically removed in release builds
		assert(is_power_of_2(block_size));
		return offset & ~(block_size - 1u);
		}

		/**
		* Given a file @offset and a @block_size, align the @offset to its
		* closest right-side block boundary.
		*
		* @note This function assumes that block_size is a power of 2.
		*
		* @param [in] offset the offset to align.
		* @param [in] block_size the block size used to compute boundaries.
		* @returns an offset aligned to the right-side block boundary.
		*/
		constexpr uint64_t
		align_right(const uint64_t offset, const size_t block_size) {
		// This check is automatically removed in release builds
		assert(is_power_of_2(block_size));
		return align_left(offset, block_size) + block_size;
		}


		/**
		* Return the overrun bytes that separate @offset from the closest left side
		* block boundary.
		*
		* @note This function assumes that block_size is a power of 2.
		*
		* @param [in] offset the offset for which the overrun distance should be
		* computed.
		* @param [in] block_size the block size used to compute boundaries.
		* @returns the distance in bytes between the left-side boundary of @offset
		*/
		constexpr size_t
		block_overrun(const uint64_t offset, const size_t block_size) {
		// This check is automatically removed in release builds
		assert(is_power_of_2(block_size));
		return offset & (block_size - 1u);
		}


		/**
		* Return the underrun bytes that separate @offset from the closest right side
		* block boundary.
		*
		* @note This function assumes that block_size is a power of 2.
		*
		* @param [in] offset the offset for which the overrun distance should be
		* computed.
		* @param [in] block_size the block size used to compute boundaries.
		* @returns the distance in bytes between the right-side boundary of @offset
		*/
		constexpr size_t
		block_underrun(const uint64_t offset, const size_t block_size) {
		// This check is automatically removed in release builds
		assert(is_power_of_2(block_size));
		return align_right(offset, block_size) - offset;
		}


		/**
		* Given an @offset and a @block_size, compute the block index to which @offset
		* belongs.
		*
		* @note Block indexes are (conceptually) computed by dividing @offset
		* by @block_size, with index 0 referring to block [0, block_size - 1],
		* index 1 to block [block_size, 2 * block_size - 1], and so on up to
		* a maximum index FILE_LENGTH / block_size.
		*
		* @note This function assumes that @block_size is a power of 2.
		*
		* @param [in] offset the offset for which the block index should be computed.
		* @param [in] block_size the block_size that should be used to compute the
		* index.
		* @returns the index of the block containing @offset.
		*/
		constexpr uint64_t
		block_index(const uint64_t offset, const size_t block_size) {

		using math::log2;

		// This check is automatically removed in release builds
		assert(is_power_of_2(block_size));
		return align_left(offset, block_size) >> log2(block_size);
		}

		/**
		* Compute the number of blocks involved in an operation affecting the
		* regions from [@offset, to @offset + @count).
		*
		* @note This function assumes that @block_size is a power of 2.
		* @note This function assumes that @offset + @count does not
		* overflow.
		*
		* @param [in] offset the operation's initial offset.
		* @param [in] size the number of bytes affected by the operation.
		* @param [in] block_size the block size that should be used to compute the
		* number of blocks.
		* @returns the number of blocks affected by the operation.
		*/
		constexpr std::size_t
		block_count(const uint64_t offset, const size_t size, const size_t block_size) {

		using math::log2;

		// These checks are automatically removed in release builds
		assert(is_power_of_2(block_size));

		#if defined(__GNUC__) && !defined(__clang__)
		assert(!__builtin_add_overflow_p(offset, size, uint64_t{0}));
		#else
		assert(offset + size > offset);
		#endif

		const uint64_t first_block = align_left(offset, block_size);
		const uint64_t final_block = align_left(offset + size, block_size);
		const size_t mask = -!!size; // this is either 0 or ~0

		return (((final_block >> log2(block_size)) -
		(first_block >> log2(block_size)) +
		!is_aligned(offset + size, block_size))) &
		mask;
		}

		} // namespace posix_file::math

		#endif // POSIX_FILE_MATH_HPP

src/posix_file/posix_file/ranges.hpp

0 → 100644

+214 −0

Original line number	Diff line number	Diff line
		/******************************************************************************
		* Copyright 2022-2023, Barcelona Supercomputing Center (BSC), Spain
		*
		* This software was partially supported by the EuroHPC-funded project ADMIRE
		* (Project ID: 956748, https://www.admire-eurohpc.eu).
		*
		* This file is part of Cargo.
		*
		* Cargo is free software: you can redistribute it and/or modify
		* it under the terms of the GNU General Public License as published by
		* the Free Software Foundation, either version 3 of the License, or
		* (at your option) any later version.
		*
		* Cargo is distributed in the hope that it will be useful,
		* but WITHOUT ANY WARRANTY; without even the implied warranty of
		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
		* GNU General Public License for more details.
		*
		* You should have received a copy of the GNU General Public License
		* along with Cargo. If not, see <https://www.gnu.org/licenses/>.
		*
		* SPDX-License-Identifier: GPL-3.0-or-later
		*****************************************************************************/

		#ifndef POSIX_FILE_RANGES_HPP
		#define POSIX_FILE_RANGES_HPP

		#include <cstddef>
		#include <utility>

		namespace posix_file::ranges {

		/**
		* An iterator for a file's offsets
		*/
		class offset_iterator {

		public:
		using iterator_category = std::random_access_iterator_tag;
		using value_type = posix_file::offset;
		using reference = value_type const&;
		using pointer = value_type const*;
		using difference_type = std::ptrdiff_t;

		constexpr offset_iterator() : m_current(0) {}

		constexpr explicit offset_iterator(posix_file::offset offset)
		: m_current(offset) {}

		// Forward iterator requirements
		// (We don't need a strictly conforming implementation since this is a
		// read only view that only returns integral values)
		constexpr value_type
		operator*() const {
		return m_current;
		}

		constexpr offset_iterator&
		operator++() {
		++m_current;
		return *this;
		}

		constexpr offset_iterator // NOLINT
		operator++(int) {
		offset_iterator tmp = *this;
		++(*this);
		return tmp;
		}

		// Bidirectional iterator requirements
		constexpr offset_iterator&
		operator--() {
		--m_current;
		return *this;
		}

		constexpr offset_iterator // NOLINT
		operator--(int) {
		offset_iterator tmp = *this;
		--(*this);
		return tmp;
		}

		// Random access iterator requirements
		constexpr offset_iterator&
		operator+=(difference_type n) {
		m_current += n;
		return *this;
		}

		constexpr offset_iterator
		operator+(difference_type n) const {
		return offset_iterator{m_current + n};
		}

		constexpr offset_iterator&
		operator-=(difference_type n) {
		m_current -= n;
		return *this;
		}

		constexpr offset_iterator
		operator-(difference_type n) const {
		return offset_iterator{m_current - n};
		}

		constexpr friend bool
		operator==(const offset_iterator& lhs, const offset_iterator& rhs) {
		return lhs.m_current == rhs.m_current;
		};

		friend bool
		operator!=(const offset_iterator& lhs, const offset_iterator& rhs) {
		return !(lhs == rhs);
		};

		friend constexpr difference_type
		operator-(const offset_iterator& lhs, const offset_iterator& rhs) {
		return static_cast<difference_type>(lhs.m_current - rhs.m_current);
		}

		private:
		value_type m_current;
		};

		/**
		* A file range defined by [offset, offset+size).
		*/
		class range {

		public:
		constexpr range(posix_file::offset offset, std::size_t size) noexcept
		: m_offset(offset), m_size(size) {}

		constexpr posix_file::offset
		offset() const noexcept {
		return static_cast<posix_file::offset>(m_offset);
		}

		constexpr std::size_t
		size() const noexcept {
		return m_size;
		}

		friend constexpr bool
		operator==(const range& lhs, const range& rhs) noexcept {
		return lhs.m_offset == rhs.m_offset && lhs.m_size == rhs.m_size;
		}

		constexpr auto
		begin() const noexcept {
		return offset_iterator{m_offset};
		}

		constexpr auto
		end() const noexcept {
		return offset_iterator{m_offset + m_size};
		}

		private:
		posix_file::offset m_offset;
		std::size_t m_size;
		};

		template <typename Iterator>
		struct iterator_range {

		using base_iterator = Iterator;

		constexpr iterator_range(Iterator&& begin_iterator, Iterator&& end_iterator)
		: m_begin_iterator(std::forward<Iterator>(begin_iterator)),
		m_end_iterator(std::forward<Iterator>(end_iterator)) {}

		constexpr std::size_t
		size() const {
		return std::distance(m_begin_iterator, m_end_iterator);
		}

		constexpr auto
		begin() const {
		return m_begin_iterator;
		}

		constexpr auto
		end() const {
		return m_end_iterator;
		}

		Iterator m_begin_iterator;
		Iterator m_end_iterator;
		};

		} // namespace posix_file::ranges

		#ifdef POSIX_FILE_HAVE_FMT

		#include <fmt/format.h>

		template <>
		struct fmt::formatter<posix_file::ranges::range> : formatter<std::string_view> {
		// parse is inherited from formatter<string_view>.
		template <typename FormatContext>
		auto
		format(const posix_file::ranges::range& r, FormatContext& ctx) const {
		const auto str =
		fmt::format("{{offset: {}, size: {}}}", r.offset(), r.size());
		return formatter<std::string_view>::format(str, ctx);
		}
		};

		#endif // POSIX_FILE_HAVE_FMT

		#endif // POSIX_FILE_RANGES_HPP