diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 68eaca4ed3a2a944e32510b744c9294bf0e4d417..3786a35f70ef06e31dd87d4a33d493555974cc0e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,14 +24,14 @@ variables:
GIT_SUBMODULE_STRATEGY: recursive
# base image
-image: gekkofs/core:0.9.2
+image: gekkofs/core:0.9.3-exp
################################################################################
## Validating
################################################################################
check format:
stage: lint
- image: gekkofs/linter:0.9.2
+ image: gekkofs/linter:0.9.3-exp
needs: []
script:
- ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format:
################################################################################
gkfs:
stage: build
- image: gekkofs/deps:0.9.2
+ image: gekkofs/deps:0.9.3-exp
interruptible: true
needs: []
script:
@@ -67,7 +67,7 @@ gkfs:
gkfwd:
stage: build
- image: gekkofs/deps:0.9.2
+ image: gekkofs/deps:0.9.3-exp
interruptible: true
needs: []
script:
@@ -89,7 +89,7 @@ gkfwd:
## == tests for scripts ====================
scripts:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
needs: []
script:
- mkdir -p ${BUILD_PATH}/tests/scripts
@@ -105,7 +105,7 @@ scripts:
## == integration tests for gkfs ===========
gkfs:integration:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
interruptible: true
needs: ['gkfs']
parallel:
@@ -154,7 +154,7 @@ gkfs:integration:
## == integration tests for gkfwd ==========
gkfwd:integration:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
interruptible: true
needs: ['gkfwd']
parallel:
@@ -204,7 +204,7 @@ gkfwd:integration:
## == unit tests for gkfs ==================
gkfs:unit:
stage: test
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
needs: ['gkfs']
script:
## Add path to mkfs.kreon
@@ -242,7 +242,7 @@ gkfs:unit:
################################################################################
documentation:
stage: docs
- image: gekkofs/docs:0.9.2
+ image: gekkofs/docs:0.9.3-exp
needs: []
rules:
# we only build the documentation automatically if we are on the
@@ -272,7 +272,7 @@ documentation:
## == coverage baseline ====================
coverage:baseline:
stage: report
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
interruptible: true
needs: ['gkfs', 'gkfwd']
@@ -298,7 +298,7 @@ coverage:baseline:
coverage:
stage: report
- image: gekkofs/testing:0.9.2
+ image: gekkofs/testing:0.9.3-exp
# needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',
# 'gkfs:unit' ]
needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ]
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f360b69b91ab61433facc33aaad5bc321d549b8b..e7b1302c8666b7310e22029d2b46566eb90a0c3f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
-
+- Replication without using the server. NUM_REPL (0 < NUM_REPL < num_servers) env variable defines the number of
+replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
+- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.
+- Added reattemp support in get_fs_config to other servers, when the initial server fails.
+- Added support for Erasure codes ([!168]) using Jerasure lib and adding support for Read error injection.
+- Added support for on demand erasure codes calculation with `gkfs_ec_ondemand(fd)`
### New
- Additional tests to increase code coverage ([!141](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
diff --git a/CMake/FindGF_complete.cmake b/CMake/FindGF_complete.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..4af6878ea932d475f7345967a67d5c5256c42aca
--- /dev/null
+++ b/CMake/FindGF_complete.cmake
@@ -0,0 +1,68 @@
+################################################################################
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+#
+# - Try to find GF_complete library
+# This will define
+# GF_complete_FOUND
+# GF_complete_INCLUDE_DIR
+# GF_complete_LIBRARIES
+#
+
+find_path(GF_complete_INCLUDE_DIR
+ NAMES gf_complete.h
+ )
+
+find_library(GF_complete_LIBRARY
+ NAMES gf_complete
+ )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args( GF_complete
+ DEFAULT_MSG
+ GF_complete_INCLUDE_DIR
+ GF_complete_LIBRARY
+)
+
+if(GF_complete_FOUND)
+ set(GF_complete_INCLUDE_DIRS ${GF_complete_INCLUDE_DIR})
+ set(GF_complete_LIBRARIES ${GF_complete_LIBRARY})
+
+ if(NOT TARGET GF_complete::GF_complete)
+ add_library(GF_complete::GF_complete UNKNOWN IMPORTED)
+ set_target_properties(GF_complete::GF_complete PROPERTIES
+ IMPORTED_LOCATION "${GF_complete_LIBRARY}"
+ INTERFACE_INCLUDE_DIRECTORIES "${GF_complete_INCLUDE_DIR}"
+ )
+ endif()
+endif()
+
+mark_as_advanced(
+ GF_complete_INCLUDE_DIR
+ GF_complete_LIBRARY
+)
diff --git a/CMake/FindJerasure.cmake b/CMake/FindJerasure.cmake
new file mode 100644
index 0000000000000000000000000000000000000000..8117b47e6147de9f1ef32eda02e07f38cc963c39
--- /dev/null
+++ b/CMake/FindJerasure.cmake
@@ -0,0 +1,68 @@
+################################################################################
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany #
+# #
+# This software was partially supported by the #
+# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
+# #
+# This software was partially supported by the #
+# ADA-FS project under the SPPEXA project funded by the DFG. #
+# #
+# This file is part of GekkoFS. #
+# #
+# GekkoFS is free software: you can redistribute it and/or modify #
+# it under the terms of the GNU General Public License as published by #
+# the Free Software Foundation, either version 3 of the License, or #
+# (at your option) any later version. #
+# #
+# GekkoFS is distributed in the hope that it will be useful, #
+# but WITHOUT ANY WARRANTY; without even the implied warranty of #
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
+# GNU General Public License for more details. #
+# #
+# You should have received a copy of the GNU General Public License #
+# along with GekkoFS. If not, see . #
+# #
+# SPDX-License-Identifier: GPL-3.0-or-later #
+################################################################################
+
+#
+# - Try to find Jerasure library
+# This will define
+# Jerasure_FOUND
+# Jerasure_INCLUDE_DIR
+# Jerasure_LIBRARIES
+#
+
+find_path(Jerasure_INCLUDE_DIR
+ NAMES jerasure/jerasure.h
+ )
+
+find_library(Jerasure_LIBRARY
+ NAMES Jerasure
+ )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Jerasure
+ DEFAULT_MSG
+ Jerasure_INCLUDE_DIR
+ Jerasure_LIBRARY
+)
+
+if(Jerasure_FOUND)
+ set(Jerasure_INCLUDE_DIRS ${Jerasure_INCLUDE_DIR})
+ set(Jerasure_LIBRARIES ${Jerasure_LIBRARY})
+
+ if(NOT TARGET Jerasure::Jerasure)
+ add_library(Jerasure::Jerasure UNKNOWN IMPORTED)
+ set_target_properties(Jerasure::Jerasure PROPERTIES
+ IMPORTED_LOCATION "${Jerasure_LIBRARY}"
+ INTERFACE_INCLUDE_DIRECTORIES "${Jerasure_INCLUDE_DIR}"
+ )
+ endif()
+endif()
+
+mark_as_advanced(
+ Jerasure_INCLUDE_DIR
+ Jerasure_LIBRARY
+)
\ No newline at end of file
diff --git a/CMake/gkfs-options.cmake b/CMake/gkfs-options.cmake
index 598de903ca9a82d9dd4496a2bd96a704792fe2ad..775b9275c00adda90078d0297396b2fbadc47c06 100644
--- a/CMake/gkfs-options.cmake
+++ b/CMake/gkfs-options.cmake
@@ -370,4 +370,20 @@ gkfs_define_option(
DESCRIPTION "If GKFS_ENABLE_FORWARDING is ON, use AGIOS for scheduling I/Os"
)
+################################################################################
+# Erasure codes
+################################################################################
+gkfs_define_option(
+ GKFS_ENABLE_EC
+ HELP_TEXT "Enable Erasure Code"
+ DEFAULT_VALUE OFF
+ DESCRIPTION "Use Jerasure for erasure codes reliability"
+)
+
+gkfs_define_option(
+ GKFS_ENABLE_READ_ERRORS
+ HELP_TEXT "Enable Read Errors using replication"
+ DEFAULT_VALUE OFF
+ DESCRIPTION "Inject read errors"
+)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 71e7c842f6c73bc814feaff9203b6c97048f8058..d61d8a675ecaffb397b7ffed281137f5308929f1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -203,6 +203,18 @@ if(GKFS_ENABLE_PROMETHEUS)
find_package(prometheus-cpp REQUIRED) # >= 1.0.0
endif()
+### Jerasure: needed for the calculation of Erasure codes
+if(GKFS_ENABLE_EC)
+ message(STATUS "[${PROJECT_NAME}] Checking for Jerasure")
+ add_compile_definitions(GKFS_ENABLE_EC)
+ find_package(GF_complete REQUIRED)
+ find_package(Jerasure REQUIRED)
+endif()
+
+if(GKFS_ENABLE_READ_ERRORS)
+ add_compile_definitions(GKFS_ENABLE_READ_ERRORS)
+endif()
+
### Other stuff that can be found out using find_package:
# determine the thread library of the system
diff --git a/CMakePresets.json b/CMakePresets.json
index 88fc9ce80df87c14a5217b3334b7e30e1dd8bfcb..819894f00182724c1a9239aea0a89fcfb56b867f 100644
--- a/CMakePresets.json
+++ b/CMakePresets.json
@@ -86,7 +86,8 @@
"GKFS_ENABLE_PROMETHEUS": true,
"GKFS_RENAME_SUPPORT": true,
"MAX_OPEN_FDS": "10000",
- "MAX_INTERNAL_FDS": "1024"
+ "MAX_INTERNAL_FDS": "1024",
+ "GKFS_ENABLE_EC": true
}
},
{
diff --git a/README.md b/README.md
index 6f571eff69a818ce00c1c4ecfb64a9f6876bb9ba..2da692a2a26bfe92d1facb99e8aa426121d4e130 100644
--- a/README.md
+++ b/README.md
@@ -319,6 +319,24 @@ Support for fstat in renamed files is included.
This is disabled by default.
+### Replication
+The user can enable the data replication feature by setting the replication environment variable:
+`LIBGKFS_NUM_REPL=`.
+The number of replicas should go from 0 to the number of servers-1.
+The replication environment variable can be set up for each client, independently.
+
+### Erasure codes
+The user can enable resilience with erasure codes with `-DGKFS_ENABLE_EC`
+Using `LIBGKFS_NUM_REPL=`, the user can define the number of EC servers.
+The total servers available for data will be -> total servervs - num_repl
+
+Additionaly the user can enable the `-DGKFS_ENABLE_READ_ERRORS` to inject a
+50% of read errors into the workflow.
+
+We also included a function to call the EC calculation on demand over a open file:
+`gkfs_ec_ondemand(int fd)`, to use it requires the environment variable `LIBGKFS_EC_ONDEMAND=1`
+
+The function is useful to calculate the ec only when the file is completed, like checkpoints.
## Acknowledgment
This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).
diff --git a/docker/0.9.3-exp/core/Dockerfile b/docker/0.9.3-exp/core/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..a55434924b2cfe720195c371de6736ac049826f7
--- /dev/null
+++ b/docker/0.9.3-exp/core/Dockerfile
@@ -0,0 +1,40 @@
+FROM debian:bullseye-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ git \
+ curl \
+ ca-certificates \
+ libtool \
+ pkg-config \
+ make \
+ automake \
+ gcc \
+ g++ \
+ ninja-build \
+ procps \
+ # AGIOS dependencies
+ libconfig-dev \
+ # Mercury dependencies
+ libltdl-dev \
+ lbzip2 \
+ # Margo dependencies \
+ libjson-c-dev \
+ # RocksDB dependencies
+ liblz4-dev \
+ # syscall_intercept dependencies
+ libcapstone-dev \
+ # GekkoFS dependencies
+ libboost-program-options-dev \
+ uuid-dev && \
+ # install cmake 3.14+ since it's needed for some dependencies
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+ chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+ ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean && \
+ rm ./cmake-3.25.2-Linux-x86_64.sh
diff --git a/docker/0.9.3-exp/core/Makefile b/docker/0.9.3-exp/core/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..e11d6cd4bcf5eb26c09aa45029a4a12fdd0b3364
--- /dev/null
+++ b/docker/0.9.3-exp/core/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/core:0.9.3-exp .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/core:0.9.3-exp .
+
+all:
+ docker build -t gekkofs/core:0.9.3-exp .
diff --git a/docker/0.9.3-exp/deps/Dockerfile b/docker/0.9.3-exp/deps/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..0ea72dfa9096d155295a59f36412329606fe736c
--- /dev/null
+++ b/docker/0.9.3-exp/deps/Dockerfile
@@ -0,0 +1,38 @@
+FROM gekkofs/core:0.9.3-exp
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH /opt/gkfs
+ENV GKFS_VERSION 0.9.3-exp
+
+ENV SCRIPTS_PATH ${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH ${GKFS_PATH}/deps_src
+ENV INSTALL_PATH /usr/local
+
+COPY scripts/dl_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches ${SCRIPTS_PATH}/patches
+COPY scripts/profiles ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ python3 \
+ python3-pip \
+ python3-dev \
+ python3-venv \
+ python3-setuptools \
+ libnuma-dev libyaml-dev libcurl4-openssl-dev \
+ procps && \
+ python3 -m pip install --upgrade pip && \
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && apt-get autoclean
+
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+ /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+ /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+ cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\
+ rm -rf ${DEPS_SRC_PATH} && \
+ rm -rf ${SCRIPTS_PATH} && \
+ rmdir ${GKFS_PATH} && \
+ ldconfig
diff --git a/docker/0.9.3-exp/deps/Makefile b/docker/0.9.3-exp/deps/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..41a10361ee08f8d1b2c242c8e0ca102bb139bb8b
--- /dev/null
+++ b/docker/0.9.3-exp/deps/Makefile
@@ -0,0 +1,23 @@
+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+ cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+ docker build -t gekkofs/deps:0.9.3-exp .
+
+build-amd64: copy-scripts
+ docker build --platform amd64 -t gekkofs/deps:0.9.3-exp .
+
+build-aarch64: copy-scripts
+ docker build --platform aarch64 -t gekkofs/deps:0.9.3-exp .
+
+remove-scripts:
+ - rm -rf $(CWD)/scripts
diff --git a/docker/0.9.3-exp/docs/Dockerfile b/docker/0.9.3-exp/docs/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..55a44fdc1332af58f99c3f46c43cadbdf3ccdf6b
--- /dev/null
+++ b/docker/0.9.3-exp/docs/Dockerfile
@@ -0,0 +1,34 @@
+FROM gekkofs/deps:0.9.3-exp
+
+LABEL Description="Debian-based environment suitable to build GekkoFS' documentation"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # install dependencies for Doxygen
+ python \
+ flex \
+ bison \
+ graphviz && \
+ # install doxygen (repo version is kind of old)
+ cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \
+ tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \
+ mkdir -p /tmp/doxygen-1.9.2/build && \
+ cd /tmp/doxygen-1.9.2/build && \
+ cmake -G "Unix Makefiles" .. && \
+ make -j8 install && \
+ # install sphinx, breathe and exhale
+ pip3 install \
+ 'sphinx==4.4.0' \
+ sphinx_rtd_theme \
+ 'breathe==4.33.1' \
+ 'exhale==0.3.1' \
+ 'sphinx-copybutton==0.5.0' \
+ 'sphinx-multiversion==0.2.4' \
+ 'myst_parser==0.17.0' \
+ attrs && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && \
+ rm -rf /tmp/doxygen-1.9.2 && \
+ rm /tmp/doxygen-1.9.2.src.tar.gz && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.3-exp/docs/Makefile b/docker/0.9.3-exp/docs/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..e7608ea74952aa7565432790c1391a3ae1f2270a
--- /dev/null
+++ b/docker/0.9.3-exp/docs/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/docs:0.9.3-exp .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/docs:0.9.3-exp .
+
+all:
+ docker build -t gekkofs/docs:0.9.3-exp .
\ No newline at end of file
diff --git a/docker/0.9.3-exp/linter/Dockerfile b/docker/0.9.3-exp/linter/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..82d87f86d99604d7d725dd2324e7cb9c8690e622
--- /dev/null
+++ b/docker/0.9.3-exp/linter/Dockerfile
@@ -0,0 +1,19 @@
+FROM gekkofs/core:0.9.3-exp
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # clang 15 deps for clang-format
+ lsb-release \
+ wget \
+ software-properties-common \
+ gnupg2 && \
+ # add clang-15 repos
+ wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \
+ # install clang-format
+ apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \
+ # Clean apt cache to reduce image layer size
+ rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+ # Clean apt caches of packages
+ apt-get clean && apt-get autoclean
diff --git a/docker/0.9.3-exp/linter/Makefile b/docker/0.9.3-exp/linter/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..98f344150a289781171e9e118fb2354b2e1b5f2d
--- /dev/null
+++ b/docker/0.9.3-exp/linter/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/linter:0.9.3-exp .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/linter:0.9.3-exp .
+
+all:
+ docker build -t gekkofs/linter:0.9.3-exp .
diff --git a/docker/0.9.3-exp/testing/Dockerfile b/docker/0.9.3-exp/testing/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..250fea63c48a79e185ad1c40b9b38eeb1cb8c3b6
--- /dev/null
+++ b/docker/0.9.3-exp/testing/Dockerfile
@@ -0,0 +1,32 @@
+FROM gekkofs/deps:0.9.3-exp
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+ apt-get update && \
+ apt-get install -y --no-install-recommends \
+ # required by lcov's genhtml
+ libgd-perl && \
+ # install cmake 3.21+ since we need to produce JUnit XML files
+ curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+ chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+ ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+ # install loguru
+ # (required by several of our scripts)
+ pip3 install loguru && \
+ # install lcov_cobertura
+ # (required to produce Cobertura XML reports)
+ pip3 install lcov_cobertura && \
+ # install lcov
+ # (required to produce partial coverage reports in parallel runs)
+ curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \
+ tar xfz lcov-1.16.tar.gz && \
+ cd lcov-1.16 && \
+ make install && \
+ cd .. && \
+ # cleanup
+ rm -rf /var/lib/apt/lists/* && \
+ apt-get clean && \
+ apt-get autoclean && \
+ rm ./cmake-3.25.2-Linux-x86_64.sh && \
+ rm -rf ./lcov-1.16.*
diff --git a/docker/0.9.3-exp/testing/Makefile b/docker/0.9.3-exp/testing/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..26551ad435ada88b301cc52742d501df5ce8a05b
--- /dev/null
+++ b/docker/0.9.3-exp/testing/Makefile
@@ -0,0 +1,10 @@
+.PHONY: all
+
+amd64:
+ docker build --platform amd64 -t gekkofs/testing:0.9.3-exp .
+
+aarch64:
+ docker build --platform aarch64 -t gekkofs/testing:0.9.3-exp .
+
+all:
+ docker build -t gekkofs/testing:0.9.3-exp .
diff --git a/include/client/env.hpp b/include/client/env.hpp
index c35ae1ad30a857ac037d593dba1cba979c08b787..7f408e5f36d70570e49b0482ca1c16b5cfd04744 100644
--- a/include/client/env.hpp
+++ b/include/client/env.hpp
@@ -52,6 +52,10 @@ static constexpr auto HOSTS_FILE = ADD_PREFIX("HOSTS_FILE");
static constexpr auto FORWARDING_MAP_FILE = ADD_PREFIX("FORWARDING_MAP_FILE");
#endif
+static constexpr auto NUM_REPL = ADD_PREFIX("NUM_REPL");
+static constexpr auto EC_ONDEMAND = ADD_PREFIX("EC_ONDEMAND");
+
+
} // namespace gkfs::env
#undef ADD_PREFIX
diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp
index d005d2fb950ff0563553c08b5ea6cd4c112e5693..66e2c755b4fac11b17077b4d95fb8e4f9cbe00f7 100644
--- a/include/client/gkfs_functions.hpp
+++ b/include/client/gkfs_functions.hpp
@@ -159,4 +159,10 @@ gkfs_rename(const std::string& old_path, const std::string& new_path);
extern "C" int
gkfs_getsingleserverdir(const char* path, struct dirent_extended* dirp,
unsigned int count, int server);
+
+#ifdef GKFS_ENABLE_EC
+extern "C" int
+gkfs_ec_ondemand(const unsigned int fd);
+#endif
+
#endif // GEKKOFS_GKFS_FUNCTIONS_HPP
diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp
index 26875435b3f1546b9dd455160d845811ade38166..9fc4dd1c9a020b0643e5b3edb543257553bb4090 100644
--- a/include/client/preload_context.hpp
+++ b/include/client/preload_context.hpp
@@ -104,6 +104,8 @@ private:
bool internal_fds_must_relocate_;
std::bitset protected_fds_;
std::string hostname;
+ int replicas_;
+ bool ec_ondemand_;
public:
static PreloadContext*
@@ -215,6 +217,18 @@ public:
std::string
get_hostname();
+
+ void
+ set_replicas(const int repl);
+
+ int
+ get_replicas();
+
+ bool
+ get_ec_ondemand();
+
+ void
+ set_ec_ondemand(const bool ec_ondemand);
};
} // namespace preload
diff --git a/include/client/rpc/forward_data.hpp b/include/client/rpc/forward_data.hpp
index 8518e997f61de93818c112ec951f44e181713fc1..5248e90bd890c94f8ba921c1cfe9ee7bdceead11 100644
--- a/include/client/rpc/forward_data.hpp
+++ b/include/client/rpc/forward_data.hpp
@@ -30,6 +30,9 @@
#ifndef GEKKOFS_CLIENT_FORWARD_DATA_HPP
#define GEKKOFS_CLIENT_FORWARD_DATA_HPP
+#include
+#include
+#include
namespace gkfs::rpc {
struct ChunkStat {
@@ -41,20 +44,36 @@ struct ChunkStat {
// TODO once we have LEAF, remove all the error code returns and throw them as
// an exception.
+std::pair
+ecc_forward_read(const std::string& path, void* buf, const size_t read_size,
+ const int8_t server);
+
+std::pair
+ecc_forward_write(const std::string& path, const void* buf,
+ const size_t write_size, const int8_t server);
+
std::pair
forward_write(const std::string& path, const void* buf, off64_t offset,
- size_t write_size);
+ size_t write_size, const int8_t num_copy = 0);
std::pair
forward_read(const std::string& path, void* buf, off64_t offset,
- size_t read_size);
+ size_t read_size, const int8_t num_copies,
+ std::set& failed);
int
-forward_truncate(const std::string& path, size_t current_size, size_t new_size);
+forward_truncate(const std::string& path, size_t current_size, size_t new_size,
+ const int8_t num_copies);
std::pair
forward_get_chunk_stat();
+std::pair
+calc_op_chunks(const std::string& path, const bool append_flag,
+ const off64_t in_offset, const size_t write_size,
+ const int64_t updated_metadentry_size);
+
+
} // namespace gkfs::rpc
#endif // GEKKOFS_CLIENT_FORWARD_DATA_HPP
diff --git a/include/client/rpc/forward_metadata.hpp b/include/client/rpc/forward_metadata.hpp
index 564dc903b592ead71947a4696331214a323e685b..97849a5757cfa3718cc80a9c3d24e0d855200860 100644
--- a/include/client/rpc/forward_metadata.hpp
+++ b/include/client/rpc/forward_metadata.hpp
@@ -50,10 +50,10 @@ class Metadata;
namespace rpc {
int
-forward_create(const std::string& path, mode_t mode);
+forward_create(const std::string& path, mode_t mode, const int copy);
int
-forward_stat(const std::string& path, std::string& attr);
+forward_stat(const std::string& path, std::string& attr, const int copy);
#ifdef HAS_RENAME
int
@@ -62,22 +62,24 @@ forward_rename(const std::string& oldpath, const std::string& newpath,
#endif // HAS_RENAME
int
-forward_remove(const std::string& path);
+forward_remove(const std::string& path, const int8_t num_copies);
int
-forward_decr_size(const std::string& path, size_t length);
+forward_decr_size(const std::string& path, size_t length, const int copy);
int
-forward_update_metadentry(
- const std::string& path, const gkfs::metadata::Metadata& md,
- const gkfs::metadata::MetadentryUpdateFlags& md_flags);
+forward_update_metadentry(const std::string& path,
+ const gkfs::metadata::Metadata& md,
+ const gkfs::metadata::MetadentryUpdateFlags& md_flags,
+ const int copy);
std::pair
forward_update_metadentry_size(const std::string& path, size_t size,
- off64_t offset, bool append_flag);
+ off64_t offset, bool append_flag,
+ const int num_copies);
std::pair
-forward_get_metadentry_size(const std::string& path);
+forward_get_metadentry_size(const std::string& path, const int copy);
std::pair>
forward_get_dirents(const std::string& path);
diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp
index 5993a31c300f0c03bfe55b0eca45eeee94836a38..bc8afd67651afd067b988b2cd0552533d8029835 100644
--- a/include/client/rpc/rpc_types.hpp
+++ b/include/client/rpc/rpc_types.hpp
@@ -1469,11 +1469,11 @@ struct write_data {
public:
input(const std::string& path, int64_t offset, uint64_t host_id,
- uint64_t host_size, uint64_t chunk_n, uint64_t chunk_start,
- uint64_t chunk_end, uint64_t total_chunk_size,
- const hermes::exposed_memory& buffers)
+ uint64_t host_size, const std::string& wbitset, uint64_t chunk_n,
+ uint64_t chunk_start, uint64_t chunk_end,
+ uint64_t total_chunk_size, const hermes::exposed_memory& buffers)
: m_path(path), m_offset(offset), m_host_id(host_id),
- m_host_size(host_size), m_chunk_n(chunk_n),
+ m_host_size(host_size), m_wbitset(wbitset), m_chunk_n(chunk_n),
m_chunk_start(chunk_start), m_chunk_end(chunk_end),
m_total_chunk_size(total_chunk_size), m_buffers(buffers) {}
@@ -1512,6 +1512,11 @@ struct write_data {
return m_chunk_n;
}
+ std::string
+ wbitset() const {
+ return m_wbitset;
+ }
+
uint64_t
chunk_start() const {
return m_chunk_start;
@@ -1535,15 +1540,16 @@ struct write_data {
explicit input(const rpc_write_data_in_t& other)
: m_path(other.path), m_offset(other.offset),
m_host_id(other.host_id), m_host_size(other.host_size),
- m_chunk_n(other.chunk_n), m_chunk_start(other.chunk_start),
- m_chunk_end(other.chunk_end),
+ m_wbitset(other.wbitset), m_chunk_n(other.chunk_n),
+ m_chunk_start(other.chunk_start), m_chunk_end(other.chunk_end),
m_total_chunk_size(other.total_chunk_size),
m_buffers(other.bulk_handle) {}
explicit operator rpc_write_data_in_t() {
- return {m_path.c_str(), m_offset, m_host_id,
- m_host_size, m_chunk_n, m_chunk_start,
- m_chunk_end, m_total_chunk_size, hg_bulk_t(m_buffers)};
+ return {m_path.c_str(), m_offset, m_host_id,
+ m_host_size, m_wbitset.c_str(), m_chunk_n,
+ m_chunk_start, m_chunk_end, m_total_chunk_size,
+ hg_bulk_t(m_buffers)};
}
private:
@@ -1551,6 +1557,7 @@ struct write_data {
int64_t m_offset;
uint64_t m_host_id;
uint64_t m_host_size;
+ std::string m_wbitset;
uint64_t m_chunk_n;
uint64_t m_chunk_start;
uint64_t m_chunk_end;
@@ -1647,11 +1654,11 @@ struct read_data {
public:
input(const std::string& path, int64_t offset, uint64_t host_id,
- uint64_t host_size, uint64_t chunk_n, uint64_t chunk_start,
- uint64_t chunk_end, uint64_t total_chunk_size,
- const hermes::exposed_memory& buffers)
+ uint64_t host_size, const std::string& wbitset, uint64_t chunk_n,
+ uint64_t chunk_start, uint64_t chunk_end,
+ uint64_t total_chunk_size, const hermes::exposed_memory& buffers)
: m_path(path), m_offset(offset), m_host_id(host_id),
- m_host_size(host_size), m_chunk_n(chunk_n),
+ m_host_size(host_size), m_wbitset(wbitset), m_chunk_n(chunk_n),
m_chunk_start(chunk_start), m_chunk_end(chunk_end),
m_total_chunk_size(total_chunk_size), m_buffers(buffers) {}
@@ -1685,6 +1692,11 @@ struct read_data {
return m_host_size;
}
+ std::string
+ wbitset() const {
+ return m_wbitset;
+ }
+
uint64_t
chunk_n() const {
return m_chunk_n;
@@ -1713,15 +1725,16 @@ struct read_data {
explicit input(const rpc_read_data_in_t& other)
: m_path(other.path), m_offset(other.offset),
m_host_id(other.host_id), m_host_size(other.host_size),
- m_chunk_n(other.chunk_n), m_chunk_start(other.chunk_start),
- m_chunk_end(other.chunk_end),
+ m_wbitset(other.wbitset), m_chunk_n(other.chunk_n),
+ m_chunk_start(other.chunk_start), m_chunk_end(other.chunk_end),
m_total_chunk_size(other.total_chunk_size),
m_buffers(other.bulk_handle) {}
explicit operator rpc_read_data_in_t() {
- return {m_path.c_str(), m_offset, m_host_id,
- m_host_size, m_chunk_n, m_chunk_start,
- m_chunk_end, m_total_chunk_size, hg_bulk_t(m_buffers)};
+ return {m_path.c_str(), m_offset, m_host_id,
+ m_host_size, m_wbitset.c_str(), m_chunk_n,
+ m_chunk_start, m_chunk_end, m_total_chunk_size,
+ hg_bulk_t(m_buffers)};
}
private:
@@ -1729,6 +1742,7 @@ struct read_data {
int64_t m_offset;
uint64_t m_host_id;
uint64_t m_host_size;
+ std::string m_wbitset;
uint64_t m_chunk_n;
uint64_t m_chunk_start;
uint64_t m_chunk_end;
diff --git a/include/common/metadata.hpp b/include/common/metadata.hpp
index d64f94921439be0d738c6dda01b7059f94457dbf..25ed22f71428323099f461b85315a5357d12ab60 100644
--- a/include/common/metadata.hpp
+++ b/include/common/metadata.hpp
@@ -37,7 +37,6 @@
#include
#include
-
namespace gkfs::metadata {
constexpr mode_t LINK_MODE = ((S_IRWXU | S_IRWXG | S_IRWXO) | S_IFLNK);
diff --git a/include/common/rpc/distributor.hpp b/include/common/rpc/distributor.hpp
index b42d1ae30b90697e8fe449562525695f27d55531..697a1df6c5ebf21756081dbd0851f083e5466fed 100644
--- a/include/common/rpc/distributor.hpp
+++ b/include/common/rpc/distributor.hpp
@@ -36,7 +36,6 @@
#include
#include
#include