Commit 3b943e65 authored by Alberto Miranda's avatar Alberto Miranda ♨️
Browse files

Merge branch 'amiranda/109-use-mochi-thallium-instead-of-our-custom-margo-engine' into 'main' 

Resolve "Use mochi-thallium instead of our custom Margo engine"

This MR removes our in-house C++ engine for Margo and replaces it with 
mochi-thallium. This makes code simpler since the daemon implementation can 
be fully in C++, which prevents having to muck with C types and their 
lifetimes.

Closes #109

See merge request !74
parents 59bf642e 4822db5b

CMakeLists.txt

+4 −0
Original line number Diff line number Diff line
@@ -189,6 +189,10 @@ find_package(Argobots 1.1 REQUIRED) 
message(STATUS "[${PROJECT_NAME}] Checking for Margo")

find_package(Margo 0.9.6 REQUIRED)



### Thallium

message(STATUS "[${PROJECT_NAME}] Checking for Thallium")

find_package(Thallium REQUIRED)



### {fmt}: required for sensible output formatting

message(STATUS "[${PROJECT_NAME}] Downloading and building {fmt}")

FetchContent_Declare(

docker/0.2.0-wip-debug/Dockerfile

0 → 100644
+170 −0
Original line number Diff line number Diff line 
FROM debian:testing-slim



LABEL Description="Debian-based environment suitable to build scord"



ENV DEPS_INSTALL_PATH /usr/local



RUN apt-get update && \

    apt-get install -y --no-install-recommends \

        git \

        curl \

        ca-certificates \

        libtool \

        pkg-config \

        make \

        automake \

        gcc \

        g++ \

        g++-11 \

        vim \

        gdb \

        procps \

        # AGIOS dependencies

        libconfig-dev \

        # Mercury dependencies

        libltdl-dev \

        lbzip2 \

        # Margo dependencies \

        libjson-c-dev \

        # GekkoFS dependencies

        libboost-program-options-dev \

        uuid-dev  \

        python3 \

        libyaml-dev libcurl4-openssl-dev procps \

        # genopts dependencies

        python3-venv \

        # redis-plus-plus dependencies \

        libhiredis-dev \

        # tests dependencies \

        python3-pip && \

    ### install cmake 3.23.1 ###################################################

    curl -OL https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-x86_64.sh && \

    chmod u+x ./cmake-3.23.1-Linux-x86_64.sh && \

    ./cmake-3.23.1-Linux-x86_64.sh --skip-license --prefix=${DEPS_INSTALL_PATH} && \

    rm ./cmake-3.23.1-Linux-x86_64.sh && \

    ###########################################################################

    ###                           DEPENDENCIES

    ########################################################################### \

    cd && \

    mkdir deps && cd deps && \

    git clone https://github.com/jbeder/yaml-cpp --recurse-submodules && \

    git clone https://github.com/json-c/json-c --recurse-submodules && \

    git clone https://github.com/ofiwg/libfabric --recurse-submodules && \

    git clone https://github.com/pmodels/argobots --recurse-submodules && \

    git clone https://github.com/mercury-hpc/mercury --recurse-submodules && \

    git clone https://github.com/mochi-hpc/mochi-margo --recurse-submodules && \

#    cd mochi-margo && git reset --hard v0.9.9 && cd .. && \

    git clone https://github.com/sewenew/redis-plus-plus --recurse-submodules && \

    git clone https://github.com/francielizanon/agios --recurse-submodules && \

    cd agios && git checkout development && cd .. && \

    git clone https://github.com/USCiLab/cereal --recurse-submodules && \

    git clone https://github.com/mochi-hpc/mochi-thallium --recurse-submodules && \

    cd mochi-thallium && \

    export LD_LIBRARY_PATH=${DEPS_INSTALL_PATH}/lib:${DEPS_INSTALL_PATH}/lib64 && \

    export PKG_CONFIG_PATH=${DEPS_INSTALL_PATH}/lib/pkgconfig:${DEPS_INSTALL_PATH}/lib64/pkgconfig && \

    cd && \

    \

    ### yaml-cpp

    cd deps/yaml-cpp && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DYAML_CPP_BUILD_TESTS=OFF \

          .. && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### argobots

    cd deps/argobots && \

    ./autogen.sh && \

    mkdir build && cd build && \

    CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    cd deps/libfabric && \

    \

    ### libfabric

    ./autogen.sh && \

    mkdir build && cd build && \

    CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    \

    ### mercury

    cd deps/mercury && \

    mkdir build && cd build && \

    cmake -DMERCURY_USE_SELF_FORWARD:BOOL=ON \

          -DBUILD_TESTING:BOOL=ON \

          -DMERCURY_USE_BOOST_PP:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DBUILD_SHARED_LIBS:BOOL=ON \

          -DNA_USE_OFI:BOOL=ON  \

          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \

          -DCMAKE_BUILD_TYPE:STRING=Debug  \

          .. && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    \

    ### json-c

    cd deps/json-c && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          .. && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    \

    ### mochi-margo

    cd deps/mochi-margo && \

    ./prepare.sh && \

    mkdir build && cd build && \

    CFLAGS="-ggdb3 -O0" ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### redis-plus-plus

    cd deps/redis-plus-plus && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DCMAKE_BUILD_TYPE:STRING=Debug \

          .. && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    \

    ### agios

    cd deps/agios && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DCMAKE_BUILD_TYPE:STRING=Debug \

          .. && \

    make install -j && \

    cd .. && rm -rf build && cd && \

    \

    ### cereal

    cd deps/cereal && \

    mkdir build && cd build && \

    cmake -DCMAKE_BUILD_TYPE:STRING=Debug  \

          -DBUILD_DOC:BOOL=OFF \

          -DBUILD_SANDBOX:BOOL=OFF  \

          -DBUILD_TESTS:BOOL=OFF \

          -DSKIP_PERFORMANCE_COMPARISON:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          .. && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### mochi-thallium

    cd deps/mochi-thallium && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DCMAKE_BUILD_TYPE:STRING=Debug  \

          .. && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### python packages for testing scripts\

    pip install lark loguru && \

    \

    ### Cleanup

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages

    apt-get clean && apt-get autoclean

docker/0.2.0-wip-debug/Makefile

0 → 100644
+4 −0
Original line number Diff line number Diff line 
.PHONY: all



all:

	docker build -t bscstorage/scord:0.2.0-wip-debug .

docker/0.2.0-wip-debug/patches/mochi-thallium.patch

0 → 100644
+85 −0
Original line number Diff line number Diff line 
diff --git a/include/thallium/packed_data.hpp b/include/thallium/packed_data.hpp

index 9e6e76e..37e64d3 100644

--- a/include/thallium/packed_data.hpp

+++ b/include/thallium/packed_data.hpp

@@ -14,13 +14,13 @@



 namespace thallium {



-template<typename ... CtxArg> class callable_remote_procedure_with_context;

+template <typename... CtxArg> class callable_remote_procedure_with_context;

 class async_response;

-template<typename ... CtxArg> class request_with_context;

+template <typename... CtxArg> class request_with_context;

 using request = request_with_context<>;



 namespace detail {

-    struct engine_impl;

+struct engine_impl;

 }



 /**

@@ -36,9 +36,9 @@ class packed_data {



   private:

     std::weak_ptr<detail::engine_impl> m_engine_impl;

-    hg_handle_t m_handle = HG_HANDLE_NULL;

-    hg_return_t (*m_unpack_fn)(hg_handle_t,void*) = nullptr;

-    hg_return_t (*m_free_fn)(hg_handle_t,void*) = nullptr;

+    hg_handle_t                        m_handle    = HG_HANDLE_NULL;

+    hg_return_t (*m_unpack_fn)(hg_handle_t, void*) = nullptr;

+    hg_return_t (*m_free_fn)(hg_handle_t, void*)   = nullptr;

     mutable std::tuple<CtxArg...> m_context;



     /**

@@ -62,6 +62,41 @@ class packed_data {

         MARGO_ASSERT(ret, margo_ref_incr);

     }



+    packed_data(const packed_data&)            = delete;

+    packed_data& operator=(const packed_data&) = delete;

+

+    packed_data(packed_data&& rhs)

+    : m_engine_impl(std::move(rhs.m_engine_impl),

+                    m_context(std::move(rhs.m_context))) {

+        m_handle        = rhs.m_handle;

+        rhs.m_handle    = HG_HANDLE_NULL;

+        m_unpack_fn     = rhs.m_unpack_fn;

+        rhs.m_unpack_fn = nullptr;

+        m_free_fn       = rhs.m_free_fn;

+        rhs.m_free_fn   = nullptr;

+    }

+

+    packed_data& operator=(packed_data&& rhs) {

+

+        if(&rhs == this) {

+            return *this;

+        }

+

+        // the original members m_handle, m_context, and m_handle are being

+        // replaced here by the ones from rhs. It may be necessary to release

+        // their resources if `packed_data` has claimed ownership over them,

+        // otherwise we would be leaking

+        m_engine_impl = std::move(rhs.m_engine_impl);

+        m_context = std::move(rhs.m_context);

+

+        m_handle        = rhs.m_handle;

+        rhs.m_handle    = HG_HANDLE_NULL;

+        m_unpack_fn     = rhs.m_unpack_fn;

+        rhs.m_unpack_fn = nullptr;

+        m_free_fn       = rhs.m_free_fn;

+        rhs.m_free_fn   = nullptr;

+    }

+

     packed_data() = default;



   public:

@@ -78,7 +113,7 @@ class packed_data {

      * @tparam NewCtxArg Types of the serialization context.

      * @param args Context.

      */

-    template<typename ... NewCtxArg>

+    template <typename... NewCtxArg>

     auto with_serialization_context(NewCtxArg&&... args) {

         return packed_data<unwrap_decay_t<NewCtxArg>...>(

             m_unpack_fn, m_free_fn, m_handle, m_engine_impl,

docker/0.2.0-wip/Dockerfile

+106 −57
Original line number Diff line number Diff line
@@ -2,8 +2,7 @@ FROM debian:bullseye-slim 


LABEL Description="Debian-based environment suitable to build scord"



ENV INSTALL_PATH /usr/local



ENV DEPS_INSTALL_PATH /usr/local



RUN apt-get update && \

    apt-get install -y --no-install-recommends \
@@ -30,88 +29,138 @@ RUN apt-get update && \ 
        python3 \

        libyaml-dev libcurl4-openssl-dev procps \

        # genopts dependencies

        python3-venv libhiredis-dev && \

    # install cmake 3.14 since it's needed for some dependencies

        python3-venv \

        # redis-plus-plus dependencies \

        libhiredis-dev \

        # tests dependencies \

        python3-pip && \

    ### install cmake 3.23.1 ###################################################

    curl -OL https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1-Linux-x86_64.sh && \

    chmod u+x ./cmake-3.23.1-Linux-x86_64.sh && \

    ./cmake-3.23.1-Linux-x86_64.sh --skip-license --prefix=${INSTALL_PATH} && \

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages

    apt-get clean && apt-get autoclean && \

    ./cmake-3.23.1-Linux-x86_64.sh --skip-license --prefix=${DEPS_INSTALL_PATH} && \

    rm ./cmake-3.23.1-Linux-x86_64.sh && \

    cmake --version && \

    curl -OL https://github.com/jbeder/yaml-cpp/archive/refs/tags/yaml-cpp-0.6.2.tar.gz && \

    tar -xzf yaml-cpp-0.6.2.tar.gz && \

    cd yaml-cpp-yaml-cpp-0.6.2 && \

    mkdir build && \

    cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} .. && \

    make -j install && \

    ###########################################################################

    ###                           DEPENDENCIES

    ########################################################################### \

    cd && \

    rm -rf yaml-cpp-yaml-cpp-0.6.2 && \

    mkdir deps && \

    cd deps && \

    mkdir deps && cd deps && \

    git clone https://github.com/jbeder/yaml-cpp --recurse-submodules && \

    git clone https://github.com/json-c/json-c --recurse-submodules && \

    git clone https://github.com/ofiwg/libfabric --recurse-submodules && \

    git clone https://github.com/pmodels/argobots --recurse-submodules && \

    git clone https://github.com/mercury-hpc/mercury --recurse-submodules && \

    git clone https://github.com/mochi-hpc/mochi-margo --recurse-submodules && \

#    cd mochi-margo && git reset --hard v0.9.9 && cd .. && \

    git clone https://github.com/sewenew/redis-plus-plus --recurse-submodules && \

    git clone https://github.com/francielizanon/agios --recurse-submodules && \

    export LD_LIBRARY_PATH=${INSTALL_PATH}/lib:${INSTALL_PATH}/lib64 && \

    export PKG_CONFIG_PATH=${INSTALL_PATH}/lib/pkgconfig:${INSTALL_PATH}/lib64/pkgconfig && \

    cd agios && git checkout development && cd .. && \

    git clone https://github.com/USCiLab/cereal --recurse-submodules && \

    git clone https://github.com/mochi-hpc/mochi-thallium --recurse-submodules && \

    cd mochi-thallium && \

    export LD_LIBRARY_PATH=${DEPS_INSTALL_PATH}/lib:${DEPS_INSTALL_PATH}/lib64 && \

    export PKG_CONFIG_PATH=${DEPS_INSTALL_PATH}/lib/pkgconfig:${DEPS_INSTALL_PATH}/lib64/pkgconfig && \

    cd && \

    \

    ### yaml-cpp

    cd deps/yaml-cpp && \

    mkdir build && \

    cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} -DYAML_CPP_BUILD_TESTS=OFF .. && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DYAML_CPP_BUILD_TESTS=OFF \

          .. && \

    make -j install && \

    cd ../../ && \

    cd argobots && \

    cd .. && rm -rf build && cd && \

    \

    ### argobots

    cd deps/argobots && \

    ./autogen.sh && \

    ./configure --prefix=${INSTALL_PATH} && \

    mkdir build && cd build && \

    ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make install -j && \

    cd .. && \

    cd libfabric && \

    cd .. && rm -rf build && cd && \

    cd deps/libfabric && \

    \

    ### libfabric

    ./autogen.sh && \

    ./configure --prefix=${INSTALL_PATH} && \

    mkdir build && cd build && \

    ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make install -j && \

    cd .. && \

    cd mercury && \

    mkdir build && \

    cd build && \

    cmake -DMERCURY_USE_SELF_FORWARD:BOOL=ON -DBUILD_TESTING:BOOL=ON -DMERCURY_USE_BOOST_PP:BOOL=ON -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} -DBUILD_SHARED_LIBS:BOOL=ON -DNA_USE_OFI:BOOL=ON -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE:STRING=Release .. && \

    cd .. && rm -rf build && cd && \

    \

    ### mercury

    cd deps/mercury && \

    mkdir build && cd build && \

    cmake -DMERCURY_USE_SELF_FORWARD:BOOL=ON \

          -DBUILD_TESTING:BOOL=ON \

          -DMERCURY_USE_BOOST_PP:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DBUILD_SHARED_LIBS:BOOL=ON \

          -DNA_USE_OFI:BOOL=ON  \

          -DCMAKE_POSITION_INDEPENDENT_CODE=ON \

          -DCMAKE_BUILD_TYPE:STRING=Release  \

          .. && \

    make install -j && \

    cd ../.. && \

    cd json-c && \

    mkdir build && \

    cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} .. && \

    cd .. && rm -rf build && cd && \

    \

    ### json-c

    cd deps/json-c && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          .. && \

    make install -j && \

    cd ../../ && \

    cd mochi-margo && \

    cd .. && rm -rf build && cd && \

    \

    ### mochi-margo

    cd deps/mochi-margo && \

    ./prepare.sh && \

    PKG_CONFIG_PATH=${INSTALL_PATH}/lib/pkgconfig:${INSTALL_PATH}/lib64/pkgconfig ./configure --prefix=${INSTALL_PATH} && \

    mkdir build && cd build && \

    ../configure --prefix=${DEPS_INSTALL_PATH} && \

    make -j install && \

    cd && \

    cd .. && rm -rf build && cd && \

    \

    ### redis-plus-plus

    cd deps/redis-plus-plus && \

    mkdir build && \

    cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} .. && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DCMAKE_BUILD_TYPE:STRING=Release \

          .. && \

    make install -j && \

    cd ../../ && \

    cd && \

    cd .. && rm -rf build && cd && \

    \

    ### agios

    cd deps/agios && \

    git checkout development && \

    mkdir build && \

    cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_PATH} .. && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          -DCMAKE_BUILD_TYPE:STRING=Release \

          .. && \

    make install -j && \

    cd ../../ && \

    cd && \

    rm -rf deps && \

    cd .. && rm -rf build && cd && \

    \

    ### cereal

    cd deps/cereal && \

    mkdir build && cd build && \

    cmake -DCMAKE_BUILD_TYPE:STRING=Release  \

          -DBUILD_DOC:BOOL=OFF \

          -DBUILD_SANDBOX:BOOL=OFF  \

          -DBUILD_TESTS:BOOL=OFF \

          -DSKIP_PERFORMANCE_COMPARISON:BOOL=ON \

          -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH} \

          .. && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### mochi-thallium

    cd deps/mochi-thallium && \

    mkdir build && cd build && \

    cmake -DCMAKE_INSTALL_PREFIX=${DEPS_INSTALL_PATH}  \

          -DCMAKE_BUILD_TYPE:STRING=Release  \

          .. && \

    make -j install && \

    cd .. && rm -rf build && cd && \

    \

    ### python packages for testing scripts\

    pip install lark loguru && \

    \

    ### Cleanup

    # Clean apt cache to reduce image layer size

    rm -rf /var/lib/apt/lists/* && \

    # Clean apt caches of packages