.gitlab-ci.yml

@@ -24,14 +24,14 @@ variables:
   GIT_SUBMODULE_STRATEGY:       recursive
 
 # base image
-image: gekkofs/core:0.9.2
+image: gekkofs/core:0.9.3
 
 ################################################################################
 ## Validating
 ################################################################################
 check format:
   stage: lint
-  image: gekkofs/linter:0.9.2
+  image: gekkofs/linter:0.9.3
   needs: []
   script:
     - ${SCRIPTS_DIR}/check_format.sh
@@ -45,7 +45,7 @@ check format:
 ################################################################################
 gkfs:
   stage: build
-  image: gekkofs/deps:0.9.2
+  image: gekkofs/deps:0.9.3
   interruptible: true
   needs: []
   script:
@@ -65,23 +65,6 @@ gkfs:
       - ${INSTALL_PATH}
     expire_in: 1 week
 
-gkfwd:
-  stage: build
-  image: gekkofs/deps:0.9.2
-  interruptible: true
-  needs: []
-  script:
-    - cmake --preset ci-forwarding-coverage
-    - cmake --build ${BUILD_PATH} -j $(nproc) --target install
-    # reduce artifacts size
-    - ${CI_SCRIPTS_DIR}/trim_build_artifacts.sh ${BUILD_PATH}
-  artifacts:
-    paths:
-      - ${BUILD_PATH}
-      - ${INSTALL_PATH}
-    expire_in: 1 week
-
-
 ################################################################################
 ## Testing
 ################################################################################
@@ -89,7 +72,7 @@ gkfwd:
 ## == tests for scripts ====================
 scripts:
   stage: test
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
   needs: []
   script:
     - mkdir -p ${BUILD_PATH}/tests/scripts
@@ -105,7 +88,7 @@ scripts:
 ## == integration tests for gkfs ===========
 gkfs:integration:
   stage: test
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
   interruptible: true
   needs: ['gkfs']
   parallel:
@@ -154,14 +137,14 @@ gkfs:integration:
 ## == integration tests for gkfwd ==========
 gkfwd:integration:
   stage: test
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
   interruptible: true
-  needs: ['gkfwd']
+  needs: ['gkfs']
   parallel:
     matrix:
       - SUBTEST: [ forwarding ]
-  rules:
-    - when: never
+ # rules:
+ #   - when: never
 
   script:
     ## run tests
@@ -204,7 +187,7 @@ gkfwd:integration:
 ## == unit tests for gkfs ==================
 gkfs:unit:
   stage: test
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
   needs: ['gkfs']
   script:
     ## Add path to mkfs.kreon
@@ -242,7 +225,7 @@ gkfs:unit:
 ################################################################################
 documentation:
   stage: docs
-  image: gekkofs/docs:0.9.2
+  image: gekkofs/docs:0.9.3
   needs: []
   rules:
     # we only build the documentation automatically if we are on the
@@ -272,9 +255,9 @@ documentation:
 ## == coverage baseline ====================
 coverage:baseline:
   stage: report
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
   interruptible: true
-  needs: ['gkfs', 'gkfwd']
+  needs: ['gkfs']
 
   script:
     ## capture initial coverage information to establish a baseline
@@ -298,10 +281,10 @@ coverage:baseline:
 
 coverage:
   stage: report
-  image: gekkofs/testing:0.9.2
+  image: gekkofs/testing:0.9.3
 #  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfwd:integration',
 #           'gkfs:unit' ]
-  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit' ]
+  needs: [ 'coverage:baseline', 'gkfs:integration', 'gkfs:unit', 'gkfwd:integration']
   script:
     - cd ${CI_PROJECT_DIR}
     - cmake

.gitmodules

 [submodule "external/hermes"]
 	path = external/hermes
-	url = https://github.com/bsc-ssrg/hermes.git
+	url = https://github.com/gekkofs/hermes.git
 [submodule "tests/scripts/bats"]
 	path = tests/scripts/bats
 	url = https://github.com/bats-core/bats-core.git

CHANGELOG.md

@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+- Replication without using the server. NUM_REPL (0 < NUM_REPL < num_servers) env variable defines the number of 
+replicas ([!166](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
+- Modified write and reads to use a bitset instead of the traditional hash per chunk in the server.
+- Added reattemp support in get_fs_config to other servers, when the initial server fails.
+- Fused GekkoFWD and GekkoFS. GekkoFWD is enabled with the `--enable-following` in the server configuration and the ENV variable
+`LIBGKFS_FORWARDING_MAP_FILE` in the clients. ([!170](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/170)).
+- FMT10 and date removal, several dependencies updated. ([!172](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_request/172)).
 
 ### New
 ### Changed

CMake/FindMercury.cmake

@@ -177,7 +177,7 @@ endmacro()
 
 include(SelectLibraryConfigurations)
 
-set(_mercury_components na mercury_util mercury_hl)
+set(_mercury_components na mercury_util)
 
 # prevent repeating work if the main CMakeLists.txt already called
 # find_package(PkgConfig)

CMake/gkfs-options.cmake

@@ -227,6 +227,14 @@ gkfs_define_option(
   DEFAULT_VALUE OFF
 )
 
+
+# use old resolve function
+gkfs_define_option(
+	GKFS_USE_OLD_PATH_RESOLVE
+  HELP_TEXT "Use the old implementation of the resolve function"
+  DEFAULT_VALUE OFF
+)
+
 cmake_dependent_option(GKFS_INSTALL_TESTS "Install GekkoFS self tests" OFF "GKFS_BUILD_TESTS" OFF)
 
 
@@ -258,6 +266,14 @@ gkfs_define_option(
   DESCRIPTION "Compile with support for rename ops (experimental)"
 )
 
+## external link support
+gkfs_define_option(
+  GKFS_FOLLOW_EXTERNAL_SYMLINKS
+  HELP_TEXT "Enable support for following external links for resolving the path"
+  DEFAULT_VALUE OFF
+  DESCRIPTION "Compile with lstat usage in path resolve"
+)
+
 
 ################################################################################
 # Options and variables that control how GekkoFS behaves internally
@@ -348,14 +364,6 @@ gkfs_define_option(
 # I/O forwarding
 ################################################################################
 
-## Forwarding support
-gkfs_define_option(
-  GKFS_ENABLE_FORWARDING
-  HELP_TEXT "Enable I/O forwarding mode"
-  DEFAULT_VALUE OFF
-  DESCRIPTION "Use ${PROJECT_NAME} as an I/O forwarding layer"
-)
-
 ## Scheduling in I/O forwarding mode
 gkfs_define_option(
   GKFS_ENABLE_AGIOS

CMakeLists.txt

@@ -32,16 +32,17 @@ cmake_minimum_required(VERSION 3.13)
 project(
     GekkoFS
     VERSION 0.9.2
+    LANGUAGES ASM CXX C
 )
 
 enable_testing()
 
-if (NOT CMAKE_COMPILER_IS_GNUCC)
-    message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
-endif ()
-if (NOT CMAKE_COMPILER_IS_GNUCXX)
-    message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
-endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCC)
+#   message(FATAL_ERROR "The choosen C compiler is not gcc and is not supported")
+#endif ()
+#if (NOT CMAKE_COMPILER_IS_GNUCXX)
+#    message(FATAL_ERROR "The choosen C++ compiler is not g++ and is not supported")
+#endif ()
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -160,10 +161,6 @@ find_package(Margo 0.9.6 REQUIRED)
 message(STATUS "[${PROJECT_NAME}] Checking for syscall_intercept")
 find_package(Syscall_intercept REQUIRED)
 
-### Date (required by the libgkfs_intercept.so)
-message(STATUS "[${PROJECT_NAME}] Checking for Date")
-find_package(Date REQUIRED)
-
 ### AGIOS: required for scheduling I/O requests
 if (GKFS_ENABLE_AGIOS)
     message(STATUS "[${PROJECT_NAME}] Checking for Agios")
@@ -225,7 +222,7 @@ include_from_source(fmt
   MESSAGE "[${PROJECT_NAME}] Searching for {fmt}"
   SOURCE_DIR ${GKFS_DEPENDENCIES_PATH}/fmt
   GIT_REPOSITORY https://github.com/fmtlib/fmt
-  GIT_TAG f94b7364b9409f05207c3af3fa4666730e11a854 # v6.1.2
+  GIT_TAG e57ca2e3685b160617d3d95fcd9e789c4e06ca88 # v10.1.0
 )
 
 # ensure that fmt is linked as PIC
@@ -259,6 +256,14 @@ if (GKFS_SYMLINK_SUPPORT)
     add_definitions(-DHAS_SYMLINKS)
 endif ()
 
+if (GKFS_USE_OLD_PATH_RESOLVE)
+    add_definitions(-DGKFS_USE_OLD_PATH_RESOLVE)
+endif ()
+
+if (GKFS_FOLLOW_EXTERNAL_SYMLINKS)
+    add_definitions(-DGKFS_FOLLOW_EXTERNAL_SYMLINKS)
+endif ()
+
 if (GKFS_RENAME_SUPPORT)
     # Rename depends on symlink support
     add_definitions(-DHAS_SYMLINKS)
@@ -338,11 +343,7 @@ if (GKFS_BUILD_TESTS)
     message(STATUS "[gekkofs] Network interface for tests: ${GKFS_TESTS_INTERFACE}")
 
     message(STATUS "[gekkofs] Check for forwarding tests...")
-    if (GKFS_ENABLE_FORWARDING)
-        set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
-    else ()
-        set(GKFS_TESTS_FORWARDING "OFF" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
-    endif ()
+    set(GKFS_TESTS_FORWARDING "ON" CACHE STRING "Enable I/O forwarding tests (default: OFF)")
     message(STATUS "[gekkofs] Forwarding tests: ${GKFS_TESTS_FORWARDING}")
     message(STATUS "[gekkofs] Check for guided distributor tests...")
     if (GKFS_USE_GUIDED_DISTRIBUTION)

CMakePresets.json

@@ -27,14 +27,6 @@
         "deprecated": true
       }
     },
-    {
-      "name": "forwarding",
-      "inherits": "default",
-      "hidden": true,
-      "cacheVariables": {
-        "GKFS_ENABLE_FORWARDING": true
-      }
-    },
     {
       "name": "release",
       "hidden": true,
@@ -113,30 +105,6 @@
         "release"
       ]
     },
-    {
-      "name": "forwarding-debug",
-      "displayName": "Forwarding gekkofs (debug)",
-      "inherits": [
-        "forwarding",
-        "debug"
-      ]
-    },
-    {
-      "name": "forwarding-coverage",
-      "displayName": "Forwarding gekkofs (coverage)",
-      "inherits": [
-        "forwarding",
-        "coverage"
-      ]
-    },
-    {
-      "name": "forwarding-release",
-      "displayName": "Forwarding gekkofs (release)",
-      "inherits": [
-        "forwarding",
-        "release"
-      ]
-    },
     {
       "name": "ci-debug",
       "displayName": "Default gekkofs (debug, CI flags)",
@@ -171,33 +139,6 @@
         "default",
         "release"
       ]
-    },
-    {
-      "name": "ci-forwarding-debug",
-      "displayName": "Forwarding gekkofs (debug, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "debug"
-      ]
-    },
-    {
-      "name": "ci-forwarding-coverage",
-      "displayName": "Forwarding gekkofs (coverage, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "coverage"
-      ]
-    },
-    {
-      "name": "ci-forwarding-release",
-      "displayName": "Forwarding gekkofs (release, CI flags)",
-      "inherits": [
-        "ci",
-        "forwarding",
-        "release"
-      ]
     }
   ]
 }

README.md

@@ -320,6 +320,13 @@ Support for fstat in renamed files is included.
 
 This is disabled by default.
 
+### Replication
+
+The user can enable the data replication feature by setting the replication environment variable:
+`LIBGKFS_NUM_REPL=<num repl>`.
+The number of replicas should go from `0` to the `number of servers - 1`. The replication environment variable can be
+set up for each client independently.
+
 ## Acknowledgment
 
 This software was partially supported by the EC H2020 funded NEXTGenIO project (Project ID: 671951, www.nextgenio.eu).
@@ -330,7 +337,8 @@ the DFG.
 This software is partially supported by the FIDIUM project funded by the DFG.
 
 This work was partially funded by the European Union’s Horizon 2020 and the German Ministry of Education and Research (
-BMBF) under the ``Adaptive multi-tier intelligent data manager for Exascale (ADMIRE)'' project (https://www.admire-eurohpc.eu/); Grant Agreement number:
+BMBF) under the ``Adaptive multi-tier intelligent data manager for Exascale (ADMIRE)''
+project (https://www.admire-eurohpc.eu/); Grant Agreement number:
 956748-ADMIRE-H2020-JTI-EuroHPC-2019-1. Further, this work was partially supported by the Spanish Ministry of Economy
 and Competitiveness (MINECO) under grants PID2019-107255GB, and the Generalitat de Catalunya under contract
 2021-SGR-00412. This publication is part of the project ADMIRE PCI2021-121952, funded by MCIN/AEI/10.13039/501100011033.

docker/0.9.3/core/Dockerfile

+FROM debian:bullseye-slim
+
+LABEL Description="Debian-based environment suitable to build GekkoFS and its dependencies"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		git \
+		curl \
+		ca-certificates \
+		libtool \
+		pkg-config \
+		make \
+		automake \
+		gcc \
+		g++ \
+		ninja-build \
+		procps \
+		# AGIOS dependencies
+		libconfig-dev \
+		# Mercury dependencies
+		libltdl-dev \
+		lbzip2 \
+        # Margo dependencies \
+        libjson-c-dev \
+		# RocksDB dependencies
+		liblz4-dev \
+		# syscall_intercept dependencies
+		libcapstone-dev \
+		# GekkoFS dependencies
+		libboost-program-options-dev \
+		uuid-dev && \
+    # install cmake 3.14+ since it's needed for some dependencies
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+    chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+    ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean && \
+    rm ./cmake-3.25.2-Linux-x86_64.sh

docker/0.9.3/core/Makefile

+.PHONY: all
+
+amd64:
+	docker buildx build --platform amd64 -t gekkofs/core:0.9.3 .
+
+aarch64:
+	docker buildx build --platform aarch64 -t gekkofs/core:0.9.3 .
+
+all:
+	docker buildx build -t gekkofs/core:0.9.3 .

docker/0.9.3/deps/Dockerfile

+FROM gekkofs/core:0.9.3
+
+LABEL Description="Debian-based environment to build GekkoFS"
+
+ENV GKFS_PATH	/opt/gkfs
+ENV GKFS_VERSION 0.9.3
+
+ENV SCRIPTS_PATH	${GKFS_PATH}/scripts
+ENV DEPS_SRC_PATH	${GKFS_PATH}/deps_src
+ENV INSTALL_PATH	/usr/local
+
+COPY scripts/dl_dep.sh		${SCRIPTS_PATH}/
+COPY scripts/compile_dep.sh ${SCRIPTS_PATH}/
+COPY scripts/patches        ${SCRIPTS_PATH}/patches
+COPY scripts/profiles       ${SCRIPTS_PATH}/profiles
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		python3 \
+		python3-pip \
+		python3-dev \
+		python3-venv \
+		python3-setuptools \
+    libnuma-dev libyaml-dev libcurl4-openssl-dev \
+    procps && \
+    python3 -m pip install --upgrade pip && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && apt-get autoclean
+ 
+# Download and build dependencies
+RUN cd ${SCRIPTS_PATH} && \
+    /bin/bash ./dl_dep.sh -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} && \
+    /bin/bash ./compile_dep.sh -j 8 -p ci:${GKFS_VERSION} ${DEPS_SRC_PATH} ${INSTALL_PATH} && \
+    cp ${DEPS_SRC_PATH}/parallax/lib/include/parallax/structures.h ${INSTALL_PATH}/include/ &&\
+    rm -rf ${DEPS_SRC_PATH} && \
+    rm -rf ${SCRIPTS_PATH} && \
+    rmdir ${GKFS_PATH} && \
+    ldconfig

docker/0.9.3/deps/Makefile

+CWD:=$(shell pwd)
+GIT_ROOT:=$(shell git rev-parse --show-toplevel)
+
+.PHONY: all build mount-scripts umount-scripts
+
+all: build mount-scripts remove-scripts
+amd64: build-amd64 mount-scripts remove-scripts
+aarch64: build-aarch64 mount-scripts remove-scripts
+
+copy-scripts:
+	cp -R $(GIT_ROOT)/scripts $(CWD)/scripts
+
+build: copy-scripts
+	docker build -t gekkofs/deps:0.9.3 .
+
+build-amd64: copy-scripts
+	docker build --platform amd64 -t gekkofs/deps:0.9.3 .
+
+build-aarch64: copy-scripts
+	docker build --platform aarch64 -t gekkofs/deps:0.9.3 .
+
+remove-scripts:
+	- rm -rf $(CWD)/scripts

docker/0.9.3/docs/Dockerfile

+FROM gekkofs/deps:0.9.3
+
+LABEL Description="Debian-based environment suitable to build GekkoFS' documentation"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        # install dependencies for Doxygen
+        python \
+        flex \
+        bison \
+		graphviz && \
+	# install doxygen (repo version is kind of old)
+	cd /tmp && curl -L https://sourceforge.net/projects/doxygen/files/rel-1.9.2/doxygen-1.9.2.src.tar.gz/download --output doxygen-1.9.2.src.tar.gz && \
+	tar xvfz /tmp/doxygen-1.9.2.src.tar.gz && \
+	mkdir -p /tmp/doxygen-1.9.2/build && \
+	cd /tmp/doxygen-1.9.2/build && \
+	cmake -G "Unix Makefiles" .. && \
+	make -j8 install && \
+    # install sphinx, breathe and exhale
+    pip3 install \
+        'sphinx==4.4.0' \
+        sphinx_rtd_theme \
+        'breathe==4.33.1' \
+        'exhale==0.3.1' \
+        'sphinx-copybutton==0.5.0' \
+        'sphinx-multiversion==0.2.4' \
+        'myst_parser==0.17.0'  \
+        attrs && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && \
+    rm -rf /tmp/doxygen-1.9.2 && \
+    rm /tmp/doxygen-1.9.2.src.tar.gz && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean

docker/0.9.3/docs/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/docs:0.9.3 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/docs:0.9.3 .
+
+all:
+	docker build -t gekkofs/docs:0.9.3 .
\ No newline at end of file

docker/0.9.3/linter/Dockerfile

+FROM gekkofs/core:0.9.3
+
+LABEL Description="Debian-based environment to check the formatting of GekkoFS code"
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+		# clang 15 deps for clang-format
+		lsb-release \
+		wget \
+		software-properties-common \
+		gnupg2 && \
+    # add clang-15 repos
+    wget https://apt.llvm.org/llvm.sh -P /tmp && chmod +x /tmp/llvm.sh && /tmp/llvm.sh 15 && \
+    # install clang-format
+    apt-get update && apt-get install -y --no-install-recommends clang-format-15 && \
+    # Clean apt cache to reduce image layer size
+    rm -rf /var/lib/apt/lists/* && rm /tmp/llvm.sh && \
+    # Clean apt caches of packages
+    apt-get clean && apt-get autoclean

docker/0.9.3/linter/Makefile

+.PHONY: all
+
+amd64:
+	docker buildx build --platform amd64 -t gekkofs/linter:0.9.3 .
+
+aarch64:
+	docker buildx build --platform aarch64 -t gekkofs/linter:0.9.3 .
+
+all:
+	docker buildx build -t gekkofs/linter:0.9.3 .

docker/0.9.3/testing/Dockerfile

+FROM gekkofs/deps:0.9.3
+
+LABEL Description="Debian-based environment to test GekkoFS"
+
+RUN \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+        # required by lcov's genhtml
+        libgd-perl && \
+    # install cmake 3.21+ since we need to produce JUnit XML files
+    curl -OL https://github.com/Kitware/CMake/releases/download/v3.25.2/cmake-3.25.2-Linux-x86_64.sh && \
+    chmod u+x ./cmake-3.25.2-Linux-x86_64.sh && \
+    ./cmake-3.25.2-Linux-x86_64.sh --skip-license --prefix=/usr && \
+    # install loguru
+    # (required by several of our scripts)
+    pip3 install loguru && \
+    # install lcov_cobertura
+    # (required to produce Cobertura XML reports)
+    pip3 install lcov_cobertura && \
+    # install lcov
+    # (required to produce partial coverage reports in parallel runs)
+    curl -OL https://github.com/linux-test-project/lcov/releases/download/v1.16/lcov-1.16.tar.gz && \
+    tar xfz lcov-1.16.tar.gz && \
+    cd lcov-1.16 && \
+    make install && \
+    cd .. && \
+    # cleanup
+    rm -rf /var/lib/apt/lists/* && \
+    apt-get clean && \
+    apt-get autoclean && \
+    rm ./cmake-3.25.2-Linux-x86_64.sh && \
+    rm -rf ./lcov-1.16.*

docker/0.9.3/testing/Makefile

+.PHONY: all
+
+amd64:
+	docker build --platform amd64 -t gekkofs/testing:0.9.3 .
+
+aarch64:
+	docker build --platform aarch64 -t gekkofs/testing:0.9.3 .
+
+all:
+	docker build -t gekkofs/testing:0.9.3 .

docs/sphinx/users/dependencies.rst

@@ -43,4 +43,4 @@ Required
 Optional
 --------
 
-- `agios <https://github.com/francielizanon/agios>`_ (commit c26a654 or newer) to enable GekkoFWD mode.
+- `agios <https://github.com/francielizanon/agios>`_ (commit c26a654 or newer) to enable GekkoFWD mode scheduling.

docs/sphinx/users/forwarding.rst

@@ -24,13 +24,11 @@ PFS for storage, instead of a local store available at the compute nodes.
 Enabling GekkoFWD
 ------------------
 
-To enable the I/O forwarding mode of GekkoFS, the
-:code:`GKFS_ENABLE_FORWARDING` CMake option should be enabled, :ref:`when
-configuring <building_gekkofs>` the build:
-
-.. code-block:: console
-
-    $ cmake -DENABLE_FORWARDING:BOOL=ON
+To enable the I/O forwarding mode of GekkoFS, an environment 
+variable named :code:`LIBGKFS_FORWARDING_MAP_FILE` is provided
+to allow users to identify the map file on each client.
+This environment variable will enable GekkoFS forwarding, then we 
+only need to use :code:`--enable-forwarding`` option in the servers.
 
 I/O Scheduling
 --------------