From d7ac698bb432244577614505ab3ba947c7c4c24f Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 11:32:48 +0200 Subject: [PATCH 1/5] Updated spack for 0.9.4 --- scripts/spack/packages/bmi/package.py | 2 +- scripts/spack/packages/capstone/package.py | 34 +++++++++++++++++++ scripts/spack/packages/gekkofs/package.py | 29 ++++++++++------ scripts/spack/packages/rocksdb/package.py | 23 +++++++++++++ .../packages/syscall-intercept/package.py | 29 +++++++++------- 5 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 scripts/spack/packages/capstone/package.py diff --git a/scripts/spack/packages/bmi/package.py b/scripts/spack/packages/bmi/package.py index a5049ba4c..fff255fb3 100644 --- a/scripts/spack/packages/bmi/package.py +++ b/scripts/spack/packages/bmi/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * class Bmi(AutotoolsPackage): diff --git a/scripts/spack/packages/capstone/package.py b/scripts/spack/packages/capstone/package.py new file mode 100644 index 000000000..9d1743bf0 --- /dev/null +++ b/scripts/spack/packages/capstone/package.py @@ -0,0 +1,34 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin.build_systems.cmake import CMakePackage + +from spack.package import * + + +class Capstone(CMakePackage): + """Capstone is a lightweight multi-platform, + multi-architecture disassembly framework.""" + + homepage = "https://www.capstone-engine.org/" + url = "https://github.com/capstone-engine/capstone/archive/4.0.1.tar.gz" + git = "https://github.com/capstone-engine/capstone.git" + + license("BSD-3-Clause-Clear") + + version("6.0.0-Alpha1", sha256="6d617aa91818b7eece0f2bea59b93b1f87dda2f65c60fdb429b9811d8cd04b52") + version("5.0.1", sha256="2b9c66915923fdc42e0e32e2a9d7d83d3534a45bb235e163a70047951890c01a") + version("4.0.2", sha256="7c81d798022f81e7507f1a60d6817f63aa76e489aa4e7055255f21a22f5e526a") + version("4.0.1", sha256="79bbea8dbe466bd7d051e037db5961fdb34f67c9fac5c3471dd105cfb1e05dc7") + + depends_on("c", type="build") + depends_on("cxx", type="build") + + def cmake_args(self): + """Set up syscall intercept CMake arguments""" + args = [ + self.define('BUILD_SHARED_LIBS', 'ON'), + self.define('CMAKE_POSITION_INDEPENDENT_CODE', 'ON'), + ] + return args \ No newline at end of file diff --git a/scripts/spack/packages/gekkofs/package.py b/scripts/spack/packages/gekkofs/package.py index 85c131a8f..69cf5e9c3 100755 --- a/scripts/spack/packages/gekkofs/package.py +++ b/scripts/spack/packages/gekkofs/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * # for Clion. Comment out when using spack @@ -20,7 +20,7 @@ class Gekkofs(CMakePackage): isolation from each other with regards to I/O, which reduces interferences and improves performance.""" homepage = "https://storage.bsc.es/gitlab/hpc/gekkofs" git = "https://storage.bsc.es/gitlab/hpc/gekkofs.git" - url = "https://storage.bsc.es/projects/gekkofs/releases/gekkofs-v0.9.3.tar.gz" + url = "https://storage.bsc.es/projects/gekkofs/releases/gekkofs-v0.9.4.tar.gz" maintainers = ['marc_vef', 'ramon_nou'] # set various versions @@ -28,7 +28,8 @@ class Gekkofs(CMakePackage): version('0.9.0', sha256='f6f7ec9735417d71d68553b6a4832e2c23f3e406d8d14ffb293855b8aeec4c3a', deprecated=True) version('0.9.1', sha256='1772b8a9d4777eca895f88cea6a1b4db2fda62e382ec9f73508e38e9d205d5f7', deprecated=True) version('0.9.2', sha256='30e0fb225e890b89eaddd930a10845d549c8f5be7aa4670e2cb97d4aaa3eb459', deprecated=True) - version('0.9.3', sha256='3ebc62eccbcf91e9077f6c76ed06e76eb1b5c67ae1d0a43cd36300c0b52b4844') + version('0.9.3', sha256='3ebc62eccbcf91e9077f6c76ed06e76eb1b5c67ae1d0a43cd36300c0b52b4844', deprecated=True) + version('0.9.4', sha256='e45a50b158736888192c0cd4c58ecbf7c0392eb0532ea8f84b0745b1ae45db5a') # apply patches patch('date-tz.patch', when='@0.9.0,0.9.2') # set arguments @@ -43,17 +44,19 @@ class Gekkofs(CMakePackage): variant('guided_distributor', default=False, description='Enables the guided distributor.') variant('prometheus', default=False, description='Enables Prometheus support for statistics.') variant('parallax', default=False, description='Enables Parallax key-value database.', when='@latest') - variant('rename', default=False, description='Enables experimental rename support.', when='@latest') + variant('rename', default=True, description='Enables experimental rename support.', when='@0.9.4,latest') + variant('symlinks', default=True, description='Enables experimental symlink support.', when='@0.9.4,latest') variant('dedicated_psm2', default=False, description='Use dedicated _non-system_ opa-psm2 version 11.2.185.') - variant('compile', default='x86', multi=False, values=('x86', 'powerpc', 'arm'), + variant('compile', default='x86', multi=False, values=('x86', 'powerpc', 'arm', 'riscv'), description='Architecture to compile syscall intercept.') # general dependencies - depends_on('cmake@3.6.0:', type='build') + depends_on('cmake@4.0.2: -doc -ncurses -qtgui', type='build') depends_on('lz4') depends_on('argobots') depends_on('syscall-intercept@arm', when='compile=arm') depends_on('syscall-intercept@powerpc', when='compile=powerpc') depends_on('syscall-intercept@x86', when='compile=x86') + depends_on('syscall-intercept@riscv', when='compile=riscv') depends_on('opa-psm2@11.2.185', when='+dedicated_psm2') # 0.9.0 specific depends_on('date cxxstd=14 +shared +tz tzdb=system', when='@0.9.0,0.9.1,0.9.2') @@ -64,12 +67,16 @@ class Gekkofs(CMakePackage): # 0.9.1 specific depends_on('rocksdb@6.26.1 -shared +static +lz4 -snappy -zlib -zstd -bz2 +rtti', when='@0.9.1') # 0.9.2 specific - depends_on('rocksdb@8.10.2 -shared +static +lz4 -snappy -zlib -zstd -bz2 +rtti', when='@0.9.2,0.9.3,latest') + depends_on('rocksdb@8.10.2 -shared +static +lz4 -snappy -zlib -zstd -bz2 +rtti', when='@0.9.2,0.9.3') # 0.9.3 specific - depends_on('libfabric@1.20.1', when='@0.9.3,latest') - depends_on('mercury@2.3.1 -debug +ofi -mpi -bmi +sm +shared +boostsys -checksum', when='@0.9.3,latest') + depends_on('libfabric@1.20.1 fabrics=sockets,tcp,udp', when='@0.9.3,0.9.4,latest') + depends_on('mercury@2.3.1 -debug +ofi -mpi -bmi +sm +shared +boostsys -checksum', when='@0.9.3') depends_on('mochi-margo@0.15.0', when='@0.9.3,latest') - + #0.9.4 specific + depends_on('mercury@2.4.0 -debug +ofi -mpi -bmi +sm +shared +boostsys -checksum', when='@0.9.4,latest') + depends_on('mochi-margo@0.18.3', when='@0.9.4,latest') + depends_on('gcc@14.3.1', when='@0.9.4,latest') + depends_on('rocksdb@9.4.0 -shared +static +lz4 -snappy -zlib -zstd -bz2 +rtti', when='@0.9.4,latest') # Additional features # Agios I/O forwarding @@ -93,6 +100,8 @@ class Gekkofs(CMakePackage): self.define_from_variant('GKFS_ENABLE_PROMETHEUS', 'prometheus'), self.define_from_variant('GKFS_USE_PARALLAX', 'parallax'), self.define_from_variant('GKFS_RENAME_SUPPORT', 'rename'), + self.define_from_variant('GKFS_SYMLINK_SUPPORT', 'symlinks'), + ] return args diff --git a/scripts/spack/packages/rocksdb/package.py b/scripts/spack/packages/rocksdb/package.py index f94434aad..7845e1653 100644 --- a/scripts/spack/packages/rocksdb/package.py +++ b/scripts/spack/packages/rocksdb/package.py @@ -17,6 +17,7 @@ class Rocksdb(MakefilePackage): version("master", git=git, branch="master", submodules=True) version("9.2.1", sha256="bb20fd9a07624e0dc1849a8e65833e5421960184f9c469d508b58ed8f40a780f") + version("9.4.0", sha256="1f829976aa24b8ba432e156f52c9e0f0bd89c46dc0cc5a9a628ea70571c1551c") version("8.10.2", sha256="44b6ec2f4723a0d495762da245d4a59d38704e0d9d3d31c45af4014bee853256") version("8.6.7", sha256="cdb2fc3c6a556f20591f564cb8e023e56828469aa3f76e1d9535c443ba1f0c1a") version("8.1.1", sha256="9102704e169cfb53e7724a30750eeeb3e71307663852f01fa08d5a320e6155a8") @@ -61,6 +62,28 @@ class Rocksdb(MakefilePackage): phases = ["install"] def patch(self): + # For rocksdb version 9.4.0, add the missing include. + if self.spec.version == Version("9.4.0"): + # Find the line '#include ' and replace it with + # itself, a newline, and then '#include '. + filter_file( + r"#include ", + "#include \n#include ", + "db/blob/blob_file_meta.h" + ) + filter_file( + r"#include \"rocksdb/attribute_groups.h\"", + "#include \"rocksdb/attribute_groups.h\"\n#include ", + "include/rocksdb/write_batch_base.h" + ) + + filter_file( + r"#include ", + "#include \n#include ", + "include/rocksdb/trace_record.h" + ) + + filter_file("-march=native", "", join_path("build_tools", "build_detect_platform")) def install(self, spec, prefix): diff --git a/scripts/spack/packages/syscall-intercept/package.py b/scripts/spack/packages/syscall-intercept/package.py index 0243adc7d..dad87b532 100644 --- a/scripts/spack/packages/syscall-intercept/package.py +++ b/scripts/spack/packages/syscall-intercept/package.py @@ -3,8 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * - +from spack.package import * class SyscallIntercept(CMakePackage): """FIXME: Put a proper description of your package here.""" @@ -15,18 +14,24 @@ class SyscallIntercept(CMakePackage): maintainers = ['jeanbez'] version('304404', commit='304404581c57d43478438d175099d20260bae74e') - version('x86', branch='master') + version('x86', branch='x86') version('powerpc', branch='powerpc') version('arm', branch='arm') - - depends_on('capstone@4.0.1') - - with when('@arm'): - patch('syscall-intercept-arm.patch') - - with when('@x86'): - patch('syscall-intercept.patch') - patch('syscall-intercept-clone.patch') + version('riscv', branch='riscv') + + depends_on('capstone@6.0.0-Alpha1') + + def cmake_args(self): + """Set up syscall intercept CMake arguments""" + args = [ + self.define('CMAKE_PREFIX_PATH', self.prefix), + self.define('CMAKE_INSTALL_PREFIX', self.prefix), + self.define('STATIC_CAPSTONE', 'OFF'), + self.define('BUILD_EXAMPLES', 'OFF'), + self.define('BUILD_TESTS', 'OFF'), + ] + return args + @property def root_cmakelists_dir(self): -- GitLab From 8cd81e6972d0385086d282d444c872d7f7b5f351 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 11:48:31 +0200 Subject: [PATCH 2/5] updated spack 0.9.5 --- CHANGELOG.md | 1 + scripts/spack/packages/agios/package.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7aee7a8..00d47d4fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Updated Docker images for 0.9.5 ([!238](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/238)) - Logging uses a higher FD limit ([!259](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/259)) - Avoid throwing runtime error if the fd is invalid (may happen on destruction) + - Updated spack for 0.9.4 ([!260](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/260)) ### Fixed - Dup3 is supported if O_CLOEXEC is not used (i.e. hexdump) ([!228](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/228)) diff --git a/scripts/spack/packages/agios/package.py b/scripts/spack/packages/agios/package.py index c15787fa8..ab0687f84 100644 --- a/scripts/spack/packages/agios/package.py +++ b/scripts/spack/packages/agios/package.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * +from spack.package import * class Agios(CMakePackage): -- GitLab From 258842b1250ed8e7ad58db851d5031aab38a6a61 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 13:31:39 +0200 Subject: [PATCH 3/5] Fixed Argobots --- scripts/spack/packages/gekkofs/package.py | 20 +++++++++++++++---- .../packages/syscall-intercept/package.py | 4 +--- src/client/CMakeLists.txt | 1 + 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/spack/packages/gekkofs/package.py b/scripts/spack/packages/gekkofs/package.py index 69cf5e9c3..b4fea92c3 100755 --- a/scripts/spack/packages/gekkofs/package.py +++ b/scripts/spack/packages/gekkofs/package.py @@ -50,9 +50,9 @@ class Gekkofs(CMakePackage): variant('compile', default='x86', multi=False, values=('x86', 'powerpc', 'arm', 'riscv'), description='Architecture to compile syscall intercept.') # general dependencies - depends_on('cmake@4.0.2: -doc -ncurses -qtgui', type='build') + depends_on('cmake@3.15: -doc -ncurses -qtgui', type='build') depends_on('lz4') - depends_on('argobots') + depends_on('argobots@1.1:', type='link') depends_on('syscall-intercept@arm', when='compile=arm') depends_on('syscall-intercept@powerpc', when='compile=powerpc') depends_on('syscall-intercept@x86', when='compile=x86') @@ -71,10 +71,10 @@ class Gekkofs(CMakePackage): # 0.9.3 specific depends_on('libfabric@1.20.1 fabrics=sockets,tcp,udp', when='@0.9.3,0.9.4,latest') depends_on('mercury@2.3.1 -debug +ofi -mpi -bmi +sm +shared +boostsys -checksum', when='@0.9.3') - depends_on('mochi-margo@0.15.0', when='@0.9.3,latest') + depends_on('mochi-margo@0.15.0', when='@0.9.3') #0.9.4 specific depends_on('mercury@2.4.0 -debug +ofi -mpi -bmi +sm +shared +boostsys -checksum', when='@0.9.4,latest') - depends_on('mochi-margo@0.18.3', when='@0.9.4,latest') + depends_on('mochi-margo@0.18.3', when='@0.9.4,latest', type='link') depends_on('gcc@14.3.1', when='@0.9.4,latest') depends_on('rocksdb@9.4.0 -shared +static +lz4 -snappy -zlib -zstd -bz2 +rtti', when='@0.9.4,latest') @@ -93,6 +93,8 @@ class Gekkofs(CMakePackage): """Set up GekkoFS CMake arguments""" args = [ self.define('CMAKE_INSTALL_LIBDIR', self.prefix.lib), + self.define('CMAKE_PREFIX_PATH', self.prefix), + self.define('CMAKE_INSTALL_PREFIX', self.prefix), self.define('GKFS_BUILD_TESTS', self.run_tests), self.define_from_variant('GKFS_ENABLE_FORWARDING', 'forwarding'), self.define_from_variant('GKFS_ENABLE_AGIOS', 'agios'), @@ -104,6 +106,16 @@ class Gekkofs(CMakePackage): ] return args + + def patch(self): + # For gekkofs, add the missing Argobots::Argobots. + if self.spec.version == Version("0.9.4"): + filter_file( + r"PUBLIC dl", + "PUBLIC dl Argobots::Argobots", + "src/client/CMakeLists.txt" + ) + def check(self): """Run tests""" diff --git a/scripts/spack/packages/syscall-intercept/package.py b/scripts/spack/packages/syscall-intercept/package.py index dad87b532..378a67723 100644 --- a/scripts/spack/packages/syscall-intercept/package.py +++ b/scripts/spack/packages/syscall-intercept/package.py @@ -19,13 +19,11 @@ class SyscallIntercept(CMakePackage): version('arm', branch='arm') version('riscv', branch='riscv') - depends_on('capstone@6.0.0-Alpha1') + depends_on('capstone@6.0.0-Alpha1', type="link") def cmake_args(self): """Set up syscall intercept CMake arguments""" args = [ - self.define('CMAKE_PREFIX_PATH', self.prefix), - self.define('CMAKE_INSTALL_PREFIX', self.prefix), self.define('STATIC_CAPSTONE', 'OFF'), self.define('BUILD_EXAMPLES', 'OFF'), self.define('BUILD_TESTS', 'OFF'), diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index a48b0bfb0..1a920ae1f 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -58,6 +58,7 @@ target_link_libraries( hermes fmt::fmt Threads::Threads + Argobots::Argobots Microsoft.GSL::GSL ) -- GitLab From 8f25cb8624667edae18af6ec57d74db398593514 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 13:55:52 +0200 Subject: [PATCH 4/5] cmake issues with .h --- scripts/spack/packages/gekkofs/package.py | 10 +++++----- src/client/CMakeLists.txt | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/spack/packages/gekkofs/package.py b/scripts/spack/packages/gekkofs/package.py index b4fea92c3..0d628f1b8 100755 --- a/scripts/spack/packages/gekkofs/package.py +++ b/scripts/spack/packages/gekkofs/package.py @@ -24,7 +24,7 @@ class Gekkofs(CMakePackage): maintainers = ['marc_vef', 'ramon_nou'] # set various versions - version('latest', branch='master', submodules=True) + version('latest', branch='rnou/365-spack-update-0-9-5', submodules=True) version('0.9.0', sha256='f6f7ec9735417d71d68553b6a4832e2c23f3e406d8d14ffb293855b8aeec4c3a', deprecated=True) version('0.9.1', sha256='1772b8a9d4777eca895f88cea6a1b4db2fda62e382ec9f73508e38e9d205d5f7', deprecated=True) version('0.9.2', sha256='30e0fb225e890b89eaddd930a10845d549c8f5be7aa4670e2cb97d4aaa3eb459', deprecated=True) @@ -53,10 +53,10 @@ class Gekkofs(CMakePackage): depends_on('cmake@3.15: -doc -ncurses -qtgui', type='build') depends_on('lz4') depends_on('argobots@1.1:', type='link') - depends_on('syscall-intercept@arm', when='compile=arm') - depends_on('syscall-intercept@powerpc', when='compile=powerpc') - depends_on('syscall-intercept@x86', when='compile=x86') - depends_on('syscall-intercept@riscv', when='compile=riscv') + depends_on('syscall-intercept@arm', when='compile=arm', type='link') + depends_on('syscall-intercept@powerpc', when='compile=powerpc', type='link') + depends_on('syscall-intercept@x86', when='compile=x86', type='link') + depends_on('syscall-intercept@riscv', when='compile=riscv', type='link') depends_on('opa-psm2@11.2.185', when='+dedicated_psm2') # 0.9.0 specific depends_on('date cxxstd=14 +shared +tz tzdb=system', when='@0.9.0,0.9.1,0.9.2') diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 1a920ae1f..04eaf1d24 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -36,7 +36,6 @@ target_sources(gkfs_common PUBLIC path.cpp PRIVATE - logging.cpp open_file_map.cpp open_dir.cpp rpc/rpc_types.cpp @@ -74,6 +73,7 @@ endif() target_sources(gkfs_intercept PRIVATE gkfs_functions.cpp + logging.cpp intercept.cpp hooks.cpp preload.cpp @@ -85,6 +85,7 @@ if (GKFS_BUILD_USER_LIB) target_sources( gkfs_user_lib PRIVATE gkfs_functions.cpp + logging.cpp intercept.cpp hooks.cpp preload.cpp @@ -98,6 +99,7 @@ if (GKFS_BUILD_LIBC_INTERCEPTION) target_sources( gkfs_libc_intercept PRIVATE gkfs_functions.cpp + logging.cpp gkfs_libc.cpp intercept.cpp hooks.cpp -- GitLab From a3c098b1502138a4ecdbc96153decb34a91e4afe Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 12 Jun 2025 14:37:48 +0200 Subject: [PATCH 5/5] interface Syscall and argobots --- src/client/CMakeLists.txt | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 04eaf1d24..2ebed1fd1 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -31,11 +31,27 @@ # This builds the `libgkfs_intercept.so` library: the primary GekkoFS client # based on syscall interception. # ############################################################################## + +add_library(MySyscallHeaders INTERFACE) +add_library(MyArgobotsHeaders INTERFACE) + +target_link_libraries(MySyscallHeaders + INTERFACE + Syscall_intercept::Syscall_intercept +) + +target_link_libraries(MyArgobotsHeaders + INTERFACE + Argobots::Argobots +) + + add_library(gkfs_common SHARED) target_sources(gkfs_common PUBLIC path.cpp PRIVATE + logging.cpp open_file_map.cpp open_dir.cpp rpc/rpc_types.cpp @@ -57,8 +73,9 @@ target_link_libraries( hermes fmt::fmt Threads::Threads - Argobots::Argobots Microsoft.GSL::GSL + MySyscallHeaders + MyArgobotsHeaders ) add_library(gkfs_intercept SHARED) @@ -73,7 +90,6 @@ endif() target_sources(gkfs_intercept PRIVATE gkfs_functions.cpp - logging.cpp intercept.cpp hooks.cpp preload.cpp @@ -85,7 +101,6 @@ if (GKFS_BUILD_USER_LIB) target_sources( gkfs_user_lib PRIVATE gkfs_functions.cpp - logging.cpp intercept.cpp hooks.cpp preload.cpp @@ -99,7 +114,6 @@ if (GKFS_BUILD_LIBC_INTERCEPTION) target_sources( gkfs_libc_intercept PRIVATE gkfs_functions.cpp - logging.cpp gkfs_libc.cpp intercept.cpp hooks.cpp -- GitLab