diff --git a/.dockerignore b/.dockerignore index 7c1a871517e17a256b7f93563a6e40dd34522d98..e1aedd0812859e24feac01bc6e25fb29bd13a315 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,18 @@ -.git -.gitignore -build -/archive_old_fs_versions +build/ +.git/ +.vscode/ +.idea/ +cmake-build-*/ +*.o +*.a +*.so +GKFS_DATA/ +GKFS_METADATA/ +builds/ +.gitlab-ci-local/ +install/ +gkfs +ci_output.txt +log/ +logs/ +.cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5de91be752eaf7ee49c46bbe740b691240a54c24..1579e6364d5663f07c8bc03be8f8fefa4819a690 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Dirents buffer size control (`LIBGKFS_DIRENTS_BUFF_SIZE`). - New sfind filtering in the server side - Added new tests (and enabling failing ones) to increase coverage + - Added docker image (release) for 0.9.6 ([!279](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/279)) ### Changed diff --git a/CMakeLists.txt b/CMakeLists.txt index 9451f5f736d543c9569f7a40bd14c2878a620233..dde0ff34bcc7944bbcb10036a7d2f59bc6741a2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -182,6 +182,19 @@ endif () if (GKFS_ENABLE_ROCKSDB) message(STATUS "[${PROJECT_NAME}] Checking for RocksDB") add_compile_definitions(GKFS_ENABLE_ROCKSDB) + + # If liburing is available in the system, RocksDB will have been built + # with liburing support. If so, its native CMake configuration will + # expect a uring::uring target. + # Since liburing exports a pkg-config .pc file, we can use it + # to retrieve its details and create the target if it's missing. + find_package(PkgConfig REQUIRED) + pkg_check_modules(liburing IMPORTED_TARGET liburing) + if (liburing_FOUND AND NOT TARGET uring::uring) + add_library(uring::uring INTERFACE IMPORTED) + target_link_libraries(uring::uring INTERFACE PkgConfig::liburing) + endif () + find_package(RocksDB REQUIRED) message(STATUS "[${PROJECT_NAME}] RocksDB version ${RocksDB_VERSION}") message(STATUS "[${PROJECT_NAME}] RocksDB location ${RocksDB_DIR}") diff --git a/docker/0.9.6/release/Dockerfile b/docker/0.9.6/release/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..218325ddae1db7202d87d545fbe61b3944c443e9 --- /dev/null +++ b/docker/0.9.6/release/Dockerfile @@ -0,0 +1,108 @@ +# syntax=docker/dockerfile:1 + +# Builder stage +FROM debian:bookworm-slim AS builder + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + git \ + curl \ + ca-certificates \ + libtool \ + autoconf \ + automake \ + pkg-config \ + bison \ + flex \ + libzstd-dev \ + liblz4-dev \ + libcap-dev \ + libzmq3-dev \ + liburing-dev \ + libssl-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + zlib1g-dev \ + python3 \ + perl \ + patch \ + && rm -rf /var/lib/apt/lists/* + +# Set up environment variables for the build +ENV GKFS_PATH=/usr/local/gekkofs +ENV GKFS_DEPS_PATH=/usr/local/gekkofs_deps +ENV GKFS_SRC=/usr/src/gekkofs +ENV LD_LIBRARY_PATH=${GKFS_DEPS_PATH}/lib:${GKFS_DEPS_PATH}/lib64:${LD_LIBRARY_PATH:-} +ENV LIBRARY_PATH=${GKFS_DEPS_PATH}/lib:${GKFS_DEPS_PATH}/lib64:${LIBRARY_PATH:-} + + +# Create directories +RUN mkdir -p ${GKFS_DEPS_PATH} ${GKFS_SRC} + +# Copy scripts and source +COPY scripts ${GKFS_SRC}/scripts + +# Build dependencies +# We use the default profile but might need to adjust if specific versions are required. +# Assuming compile_dep.sh handles downloading/building dependencies. +WORKDIR ${GKFS_SRC} +RUN scripts/dl_dep.sh \ + -P ${GKFS_SRC}/scripts/profiles \ + -p docker:latest \ + /usr/src/gekkofs_deps_src + +RUN scripts/compile_dep.sh \ + -P ${GKFS_SRC}/scripts/profiles \ + -p docker:latest \ + -j $(nproc) \ + /usr/src/gekkofs_deps_src \ + ${GKFS_DEPS_PATH} + +# Copy rest of source (invalidates cache for GekkoFS build only) +COPY . ${GKFS_SRC} + +# Build GekkoFS +WORKDIR ${GKFS_SRC}/build +RUN cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=${GKFS_DEPS_PATH} \ + -DCMAKE_INSTALL_PREFIX=${GKFS_PATH} \ + -DGKFS_RENAME_SUPPORT=ON \ + -DGKFS_SYMLINK_SUPPORT=ON \ + -DGKFS_ENABLE_ROCKSDB=ON \ + -DGKFS_ENABLE_PROMETHEUS=ON \ + -DGKFS_ENABLE_CLIENT_METRICS=ON \ + -DGKFS_BUILD_TESTS=OFF \ + -DGKFS_INSTALL_TESTS=OFF \ + -DGKFS_BUILD_EXAMPLES=OFF \ + .. && \ + make -j$(nproc) && \ + make install + +# Runtime stage +FROM debian:bookworm-slim + +# Install runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + libstdc++6 \ + libgcc-s1 \ + libcurl4 \ + liblz4-1 \ + libzstd1 \ + libcap2 \ + libzmq5 \ + liburing2 \ + && rm -rf /var/lib/apt/lists/* + +# Copy GekkoFS artifacts +COPY --from=builder /usr/local/gekkofs /usr/local/gekkofs +COPY --from=builder /usr/local/gekkofs_deps/lib /usr/local/gekkofs_deps/lib + +# Set up environment variables +ENV LD_LIBRARY_PATH=/usr/local/gekkofs/lib:/usr/local/gekkofs_deps/lib:/usr/local/lib +ENV PATH=/usr/local/gekkofs/bin:${PATH} + +# Default command +CMD ["gkfs_daemon", "--help"] diff --git a/docker/0.9.6/release/Makefile b/docker/0.9.6/release/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..a3061dc6ee1065e19ab3a0867f376b4e82aa7a52 --- /dev/null +++ b/docker/0.9.6/release/Makefile @@ -0,0 +1,10 @@ +.PHONY: all + +amd64: + docker build --platform amd64 -t gekkofs/release:0.9.6 -f Dockerfile ../../.. + +aarch64: + docker build --platform aarch64 -t gekkofs/release:0.9.6 -f Dockerfile ../../.. + +all: + docker build -t gekkofs/release:0.9.6 -f Dockerfile ../../.. diff --git a/scripts/profiles/0.9.6-dev/docker.specs b/scripts/profiles/0.9.6-dev/docker.specs new file mode 100644 index 0000000000000000000000000000000000000000..6b6ea44135c26f776c75f33062ba07a2985b6e81 --- /dev/null +++ b/scripts/profiles/0.9.6-dev/docker.specs @@ -0,0 +1,29 @@ +declare -A wgetdeps clonedeps clonedeps_args clonedeps_patches extra_install_args +declare -a order +comment="Dependencies for Docker Release" +wgetdeps=( + ["zstd"]="1.5.7" + ["lz4"]="1.9.4" + ["capstone"]="6.0.0-Alpha1" + ["argobots"]="1.2" + ["rocksdb"]="10.4.2" + ["json-c"]="0.17-20230812" + ["cereal"]="1.3.2" + ["libzmq"]="4.3.5" + ["cppzmq"]="4.10.0" + ["prometheus-cpp"]="v1.0.0" +) +clonedeps=( + ["libfabric"]="HEAD@v2.2.0" + ["mercury"]="v2.4.1" + ["margo"]="v0.21.0" + ["thallium"]="v0.15.1" + ["syscall_intercept"]="d8b2a69961921ed123625c79a609331fc56a8931" +) +clonedeps_args=( + ["mercury"]="--recurse-submodules" +) +order=( + "lz4" "zstd" "capstone" "json-c" "cereal" "libfabric" "mercury" "argobots" "margo" "thallium" "rocksdb" "syscall_intercept" "libzmq" "cppzmq" "prometheus-cpp" +) +extra_install_args=() diff --git a/scripts/profiles/0.9.6-dev/install/zstd.install b/scripts/profiles/0.9.6-dev/install/zstd.install index 0152130593884aeb2bd6ed34d8ba4f0e80f11977..439feb5f24a81d11fc83ba40ed14aa41b901517f 100644 --- a/scripts/profiles/0.9.6-dev/install/zstd.install +++ b/scripts/profiles/0.9.6-dev/install/zstd.install @@ -59,7 +59,7 @@ pkg_install() { -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. make -j"${CORES}" - make DESTDIR="${INSTALL_DIR}" PREFIX="" install + make install diff --git a/src/common/msgpack_util.cpp b/src/common/msgpack_util.cpp index 687bacf503fbe30091b1f95a9e33aa9a8186e6e5..88f1bd6af62f6f27e0f4838420794aed4fd5bf87 100644 --- a/src/common/msgpack_util.cpp +++ b/src/common/msgpack_util.cpp @@ -208,7 +208,7 @@ ClientMetrics::path() const { return flush_path_; } void -ClientMetrics::path(const string& path, const string prefix) { +ClientMetrics::path(const string& path, const string& prefix) { const std::time_t t = std::chrono::system_clock::to_time_t(init_t_); std::stringstream init_t_stream; init_t_stream << std::put_time(std::localtime(&t), "%F_%T");