From a2d8277d69d81bfab487aa457c73c095388d2f92 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 19 Jun 2019 11:59:45 +0200 Subject: [PATCH 01/35] Use norns-build-deps image for GCC 4.8 --- .gitlab-ci.yml | 735 +++++++++++++++++++++++++------------------------ 1 file changed, 377 insertions(+), 358 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e159b4..b6fa7a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,368 +8,387 @@ variables: stages: - build - - test - - deploy + # - test + # - deploy # Install dependencies for GCC builds -before_script: - - apt-get update && - apt-get install -y - libboost-system-dev - libboost-filesystem-dev - libboost-program-options-dev - libboost-thread-dev - libboost-regex-dev - libprotobuf-dev - protobuf-compiler - libprotobuf-c0-dev - protobuf-c-compiler - libyaml-cpp-dev - libyaml-dev - libarchive-dev - cmake +# before_script: +# - apt-get update && +# apt-get install -y +# libboost-system-dev +# libboost-filesystem-dev +# libboost-program-options-dev +# libboost-thread-dev +# libboost-regex-dev +# libprotobuf-dev +# protobuf-compiler +# libprotobuf-c0-dev +# protobuf-c-compiler +# libyaml-cpp-dev +# libyaml-dev +# libarchive-dev +# cmake +# +# - pushd . && +# git clone https://github.com/ofiwg/libfabric.git && +# cd libfabric && +# ./autogen.sh && +# mkdir build && +# cd build && +# ../configure && +# make -j $(nproc) && +# make install && +# popd +# +# - pushd . && +# git clone https://github.com/mercury-hpc/mercury.git && +# cd mercury && +# mkdir build && +# cd build && +# cmake +# -DCMAKE_BUILD_TYPE:STRING=Debug +# -DBUILD_TESTING:BOOL=OFF +# -DMERCURY_USE_SM_ROUTING:BOOL=OFF +# -DMERCURY_USE_SELF_FORWARD:BOOL=OFF +# -DMERCURY_USE_CHECKSUMS:BOOL=OFF +# -DMERCURY_USE_BOOST_PP:BOOL=ON +# -DMERCURY_USE_EAGER_BULK:BOOL=ON +# -DBUILD_SHARED_LIBS:BOOL=ON +# -DNA_USE_OFI:BOOL=ON +# .. && +# make -j $(nproc) && +# make install && +# popd - - pushd . && - git clone https://github.com/ofiwg/libfabric.git && - cd libfabric && - ./autogen.sh && - mkdir build && - cd build && - ../configure && - make -j $(nproc) && - make install && - popd - - - pushd . && - git clone https://github.com/mercury-hpc/mercury.git && - cd mercury && - mkdir build && - cd build && - cmake - -DCMAKE_BUILD_TYPE:STRING=Debug - -DBUILD_TESTING:BOOL=OFF - -DMERCURY_USE_SM_ROUTING:BOOL=OFF - -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - -DMERCURY_USE_CHECKSUMS:BOOL=OFF - -DMERCURY_USE_BOOST_PP:BOOL=ON - -DMERCURY_USE_EAGER_BULK:BOOL=ON - -DBUILD_SHARED_LIBS:BOOL=ON - -DNA_USE_OFI:BOOL=ON - .. && - make -j $(nproc) && - make install && - popd - - -### GCC 5 -build:gcc:5: - image: gcc:5 - stage: build - - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - - cd tests - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api - - -### GCC 6 -build:gcc:6: - image: gcc:6 +### GCC 4.8 +build:gcc:4.8: + image: bscstorage/norns-build-deps:gcc4.8-libfabric1.7.1-mercury1.0.1 stage: build - - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - - cd tests - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api - - -### GCC 7 -build:gcc:7: - image: gcc:7 - stage: build - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - - make -j$(nproc) - - cd tests - - make -j$(nproc) - - make -j$(nproc) + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + cd tests + make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core + make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api + make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api_interactive + ) -### GCC 8 -build:gcc:8: - image: gcc:8 - stage: build - - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - - make -j$(nproc) - - cd tests - - make -j$(nproc) - - make -j$(nproc) - - -################################################################################ -# test scripts -################################################################################ -test:coverage: - image: ubuntu:latest - stage: test - - # Install dependencies - before_script: - - apt-get update && - apt-get upgrade -y && - apt-get install -y - git - build-essential - autotools-dev - automake - autoconf - libtool - pkg-config - libboost-system-dev - libboost-filesystem-dev - libboost-program-options-dev - libboost-thread-dev - libboost-regex-dev - libprotobuf-dev - protobuf-compiler - libprotobuf-c-dev - protobuf-c-compiler - libyaml-cpp-dev - libyaml-dev - libarchive-dev - libcap2-bin - valgrind - cmake - lcov - - - pushd . && - git clone https://github.com/ofiwg/libfabric.git && - cd libfabric && - ./autogen.sh && - mkdir build && - cd build && - ../configure && - make -j $(nproc) && - make install && - popd && - ldconfig - - - pushd . && - git clone https://github.com/mercury-hpc/mercury.git && - cd mercury && - mkdir build && - cd build && - cmake - -DCMAKE_BUILD_TYPE:STRING=Debug - -DBUILD_TESTING:BOOL=OFF - -DMERCURY_USE_SM_ROUTING:BOOL=OFF - -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - -DMERCURY_USE_CHECKSUMS:BOOL=OFF - -DMERCURY_USE_BOOST_PP:BOOL=ON - -DMERCURY_USE_EAGER_BULK:BOOL=ON - -DBUILD_SHARED_LIBS:BOOL=ON - -DNA_USE_OFI:BOOL=ON - .. && - make -j $(nproc) && - make install && - popd && - ldconfig - - # Build and test - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - CFLAGS="-O0 --coverage" - CXXFLAGS="-O0 --coverage" - LDFLAGS="--coverage" - - make -j$(nproc) - - cd tests - - make -j$(nproc) core - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - - make -j$(nproc) api - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - - after_script: - - (cd build && ../gencov.sh) - - genhtml -o build/html/coverage build/norns.info - - if [[ -e build/tests.log ]]; - then - cat $(tail -1 build/tests.log)/config/urd.log; - fi - - artifacts: - paths: - - build/html/coverage/ - -test:optimized: - image: ubuntu:latest - stage: test - - # Install dependencies - before_script: - - apt-get update && - apt-get upgrade -y && - apt-get install -y - git - build-essential - autotools-dev - automake - autoconf - libtool - pkg-config - libboost-system-dev - libboost-filesystem-dev - libboost-program-options-dev - libboost-thread-dev - libboost-regex-dev - libprotobuf-dev - protobuf-compiler - libprotobuf-c-dev - protobuf-c-compiler - libyaml-cpp-dev - libyaml-dev - libarchive-dev - libcap2-bin - valgrind - cmake - - - pushd . && - git clone https://github.com/ofiwg/libfabric.git && - cd libfabric && - ./autogen.sh && - mkdir build && - cd build && - ../configure && - make -j $(nproc) && - make install && - popd && - ldconfig - - - pushd . && - git clone https://github.com/mercury-hpc/mercury.git && - cd mercury && - mkdir build && - cd build && - cmake - -DCMAKE_BUILD_TYPE:STRING=Debug - -DBUILD_TESTING:BOOL=OFF - -DMERCURY_USE_SM_ROUTING:BOOL=OFF - -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - -DMERCURY_USE_CHECKSUMS:BOOL=OFF - -DMERCURY_USE_BOOST_PP:BOOL=ON - -DMERCURY_USE_EAGER_BULK:BOOL=ON - -DBUILD_SHARED_LIBS:BOOL=ON - -DNA_USE_OFI:BOOL=ON - .. && - make -j $(nproc) && - make install && - popd && - ldconfig - - # Build and test - script: - - ./bootstrap.sh - - mkdir build && cd build - - ../configure - --enable-tests - - make -j$(nproc) - - cd tests - - make -j$(nproc) core - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - - make -j$(nproc) api - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - - after_script: - - if [[ -e build/tests.log ]]; - then - cat $(tail -1 build/tests.log)/config/urd.log; - fi - -pages: - image: alpine:latest - stage: deploy - before_script: - - '' - dependencies: - - test:coverage - script: - - mv build/html/coverage/ public/ - artifacts: - paths: - - public - expire_in: 30 days - only: - - master + ### ### GCC 5 + ### build:gcc:5: + ### image: gcc:5 + ### stage: build + ### + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + ### - cd tests + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api + ### + ### + ### ### GCC 6 + ### build:gcc:6: + ### image: gcc:6 + ### stage: build + ### + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + ### - cd tests + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core + ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api + ### + ### + ### ### GCC 7 + ### build:gcc:7: + ### image: gcc:7 + ### stage: build + ### + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### - make -j$(nproc) + ### - cd tests + ### - make -j$(nproc) + ### - make -j$(nproc) + ### + ### + ### ### GCC 8 + ### build:gcc:8: + ### image: gcc:8 + ### stage: build + ### + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### - make -j$(nproc) + ### - cd tests + ### - make -j$(nproc) + ### - make -j$(nproc) + ### + ### + ### ################################################################################ + ### # test scripts + ### ################################################################################ + ### test:coverage: + ### image: ubuntu:latest + ### stage: test + ### + ### # Install dependencies + ### before_script: + ### - apt-get update && + ### apt-get upgrade -y && + ### apt-get install -y + ### git + ### build-essential + ### autotools-dev + ### automake + ### autoconf + ### libtool + ### pkg-config + ### libboost-system-dev + ### libboost-filesystem-dev + ### libboost-program-options-dev + ### libboost-thread-dev + ### libboost-regex-dev + ### libprotobuf-dev + ### protobuf-compiler + ### libprotobuf-c-dev + ### protobuf-c-compiler + ### libyaml-cpp-dev + ### libyaml-dev + ### libarchive-dev + ### libcap2-bin + ### valgrind + ### cmake + ### lcov + ### + ### - pushd . && + ### git clone https://github.com/ofiwg/libfabric.git && + ### cd libfabric && + ### ./autogen.sh && + ### mkdir build && + ### cd build && + ### ../configure && + ### make -j $(nproc) && + ### make install && + ### popd && + ### ldconfig + ### + ### - pushd . && + ### git clone https://github.com/mercury-hpc/mercury.git && + ### cd mercury && + ### mkdir build && + ### cd build && + ### cmake + ### -DCMAKE_BUILD_TYPE:STRING=Debug + ### -DBUILD_TESTING:BOOL=OFF + ### -DMERCURY_USE_SM_ROUTING:BOOL=OFF + ### -DMERCURY_USE_SELF_FORWARD:BOOL=OFF + ### -DMERCURY_USE_CHECKSUMS:BOOL=OFF + ### -DMERCURY_USE_BOOST_PP:BOOL=ON + ### -DMERCURY_USE_EAGER_BULK:BOOL=ON + ### -DBUILD_SHARED_LIBS:BOOL=ON + ### -DNA_USE_OFI:BOOL=ON + ### .. && + ### make -j $(nproc) && + ### make install && + ### popd && + ### ldconfig + ### + ### # Build and test + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### CFLAGS="-O0 --coverage" + ### CXXFLAGS="-O0 --coverage" + ### LDFLAGS="--coverage" + ### - make -j$(nproc) + ### - cd tests + ### - make -j$(nproc) core + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as + ### - make -j$(nproc) api + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" + ### + ### after_script: + ### - (cd build && ../gencov.sh) + ### - genhtml -o build/html/coverage build/norns.info + ### - if [[ -e build/tests.log ]]; + ### then + ### cat $(tail -1 build/tests.log)/config/urd.log; + ### fi + ### + ### artifacts: + ### paths: + ### - build/html/coverage/ + ### + ### test:optimized: + ### image: ubuntu:latest + ### stage: test + ### + ### # Install dependencies + ### before_script: + ### - apt-get update && + ### apt-get upgrade -y && + ### apt-get install -y + ### git + ### build-essential + ### autotools-dev + ### automake + ### autoconf + ### libtool + ### pkg-config + ### libboost-system-dev + ### libboost-filesystem-dev + ### libboost-program-options-dev + ### libboost-thread-dev + ### libboost-regex-dev + ### libprotobuf-dev + ### protobuf-compiler + ### libprotobuf-c-dev + ### protobuf-c-compiler + ### libyaml-cpp-dev + ### libyaml-dev + ### libarchive-dev + ### libcap2-bin + ### valgrind + ### cmake + ### + ### - pushd . && + ### git clone https://github.com/ofiwg/libfabric.git && + ### cd libfabric && + ### ./autogen.sh && + ### mkdir build && + ### cd build && + ### ../configure && + ### make -j $(nproc) && + ### make install && + ### popd && + ### ldconfig + ### + ### - pushd . && + ### git clone https://github.com/mercury-hpc/mercury.git && + ### cd mercury && + ### mkdir build && + ### cd build && + ### cmake + ### -DCMAKE_BUILD_TYPE:STRING=Debug + ### -DBUILD_TESTING:BOOL=OFF + ### -DMERCURY_USE_SM_ROUTING:BOOL=OFF + ### -DMERCURY_USE_SELF_FORWARD:BOOL=OFF + ### -DMERCURY_USE_CHECKSUMS:BOOL=OFF + ### -DMERCURY_USE_BOOST_PP:BOOL=ON + ### -DMERCURY_USE_EAGER_BULK:BOOL=ON + ### -DBUILD_SHARED_LIBS:BOOL=ON + ### -DNA_USE_OFI:BOOL=ON + ### .. && + ### make -j $(nproc) && + ### make install && + ### popd && + ### ldconfig + ### + ### # Build and test + ### script: + ### - ./bootstrap.sh + ### - mkdir build && cd build + ### - ../configure + ### --enable-tests + ### - make -j$(nproc) + ### - cd tests + ### - make -j$(nproc) core + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as + ### - make -j$(nproc) api + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" + ### + ### after_script: + ### - if [[ -e build/tests.log ]]; + ### then + ### cat $(tail -1 build/tests.log)/config/urd.log; + ### fi + ### + ### pages: + ### image: alpine:latest + ### stage: deploy + ### before_script: + ### - '' + ### dependencies: + ### - test:coverage + ### script: + ### - mv build/html/coverage/ public/ + ### artifacts: + ### paths: + ### - public + ### expire_in: 30 days + ### only: + ### - master -- GitLab From 43dff9e821ccf254b46dd5cc46e2cbddf75e47aa Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 19 Jun 2019 12:20:01 +0200 Subject: [PATCH 02/35] Fix #38 --- src/rpcs.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcs.hpp b/src/rpcs.hpp index f13cef8..7bb0efd 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -51,7 +51,7 @@ #ifndef HG_GEN_PROC_NAME #define HG_GEN_PROC_NAME(struct_type_name) \ - hermes::detail::hg_proc_ ## struct_type_name + &hermes::detail::hg_proc_ ## struct_type_name #endif // forward declarations -- GitLab From c4f1f8af70926abfa397ab0447e8c14f9e0c22a2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 19 Jun 2019 12:23:34 +0200 Subject: [PATCH 03/35] Remove GLIBCXX_USE_CXX11_ABI --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6fa7a6..911ab9b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -72,11 +72,11 @@ build:gcc:4.8: ./bootstrap.sh mkdir build && cd build ../configure --enable-tests - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + make -j$(nproc) cd tests - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api_interactive + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive ) -- GitLab From 01cd38f6847a8602ef8627e2a570a3c83c545a74 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 19 Jun 2019 17:20:18 +0200 Subject: [PATCH 04/35] Use norns-build-deps image for GCC 4.9 --- .gitlab-ci.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 911ab9b..27e7d9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -79,6 +79,24 @@ build:gcc:4.8: make -j$(nproc) api_interactive ) +### GCC 4.9 +build:gcc:4.9: + image: bscstorage/norns-build-deps:gcc4.9-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) ### ### GCC 5 ### build:gcc:5: -- GitLab From b9d1ae5f086a485d787457109e2db8390328d074 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 20 Jun 2019 16:45:39 +0200 Subject: [PATCH 05/35] Add build jobs for several GCC versions --- .gitlab-ci.yml | 116 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27e7d9a..3612545 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,7 +60,7 @@ stages: # make install && # popd -### GCC 4.8 +### GCC 4.8 with libfabric1.7.1, mercury 1.0.1 build:gcc:4.8: image: bscstorage/norns-build-deps:gcc4.8-libfabric1.7.1-mercury1.0.1 stage: build @@ -78,8 +78,12 @@ build:gcc:4.8: make -j$(nproc) api make -j$(nproc) api_interactive ) +# only: +# - tags +# - triggers +# - schedules -### GCC 4.9 +### GCC 4.9 with libfabric1.7.1, mercury 1.0.1 build:gcc:4.9: image: bscstorage/norns-build-deps:gcc4.9-libfabric1.7.1-mercury1.0.1 stage: build @@ -97,6 +101,114 @@ build:gcc:4.9: make -j$(nproc) api make -j$(nproc) api_interactive ) +# only: +# - schedules + +### GCC 5.4 with libfabric1.7.1, mercury 1.0.1 +build:gcc:5.4: + image: bscstorage/norns-build-deps:gcc5.4-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - schedules + +### GCC 6.5 with libfabric1.7.1, mercury 1.0.1 +build:gcc:6.5: + image: bscstorage/norns-build-deps:gcc6.5-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - schedules + +### GCC 7.4 with libfabric1.7.1, mercury 1.0.1 +build:gcc:7.4: + image: bscstorage/norns-build-deps:gcc7.4-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - schedules + +### GCC 8.1 with libfabric1.7.1, mercury 1.0.1 +build:gcc:8.1: + image: bscstorage/norns-build-deps:gcc8.1-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - schedules + +### GCC 9.1 with libfabric1.7.1, mercury 1.0.1 +build:gcc:9.1: + image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - schedules + ### ### GCC 5 ### build:gcc:5: -- GitLab From 14452bf491e6e9fdbe77987e6c39942247685b96 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 17:11:07 +0200 Subject: [PATCH 06/35] Don't use boost::regex if std::regex is available See #39. --- configure.ac | 14 ++++++- m4/ax_gcc_working_std_regex.m4 | 67 ++++++++++++++++++++++++++++++++++ tests/Makefile.am | 10 ++++- tests/test-env.cpp | 25 +++++++++++++ 4 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 m4/ax_gcc_working_std_regex.m4 diff --git a/configure.ac b/configure.ac index 320d1e8..2cc5a48 100644 --- a/configure.ac +++ b/configure.ac @@ -125,9 +125,19 @@ AX_BOOST_ASIO AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD -# only check for boost_regex if we are building tests +# only check for std::regex/boost::regex if we are building tests AS_IF([test "x$is_enabled_build_tests" = "xyes"], - [ AX_BOOST_REGEX + [ + # check whether the compiler has a functional std::regex + AX_GCC_WORKING_STD_REGEX + AS_IF([test "x$gcc_has_working_std_regex" = xno], + [ + # fallback to boost::regex + AC_MSG_WARN([falling back to boost::regex because std::regex is not functional]) + AX_BOOST_REGEX + ], []) + AM_CONDITIONAL([HAVE_WORKING_STD_REGEX], + test x$GCC_HAS_WORKING_STD_REGEX = xyes) AC_CONFIG_FILES(tests/Makefile) ], []) diff --git a/m4/ax_gcc_working_std_regex.m4 b/m4/ax_gcc_working_std_regex.m4 new file mode 100644 index 0000000..b0cf1a2 --- /dev/null +++ b/m4/ax_gcc_working_std_regex.m4 @@ -0,0 +1,67 @@ +# SYNOPSIS +# +# AX_GCC_WORKING_STD_REGEX +# +# DESCRIPTION +# +# This macro checks if GCC has a working implementation of std::regex, since +# GCC 4.7/4.8 provided only valid headers and function stubs. Further +# information is available: +# . +# +# If the std::regex implementation is functional, the macro will set the +# variable 'gcc_has_working_std_regex' to 'yes' and AC_DEFINE the C preprocessor +# variable HAVE_WORKING_STD_REGEX. +# +# An automake conditional can be subsequently defined as +# AM_CONDITIONAL([HAVE_WORKING_STD_REGEX], +# [test x$gcc_has_working_std_regex = xyes]) +# +# LICENSE +# +# Copyright (c) 2019 Alberto Miranda +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 10 + +AC_DEFUN([AX_GCC_WORKING_STD_REGEX], +[ + gcc_has_working_std_regex=no + AC_CACHE_CHECK( + [whether GCC has a working std::regex implementation], + ax_cv_gcc_working_regex, + [ + AC_LANG_PUSH([C++]) + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM( + [[ @%:@include ]], + [[ + @%:@if __cplusplus >= 201103L && \ + (!defined(__GLIBCXX__) || (__cplusplus >= 201402L) || \ + (defined(_GLIBCXX_REGEX_DFS_QUANTIFIERS_LIMIT) || \ + defined(_GLIBCXX_REGEX_STATE_LIMIT) || \ + (defined(_GLIBCXX_RELEASE) && \ + _GLIBCXX_RELEASE > 4))) + @%:@define HAVE_WORKING_REGEX 1 + @%:@else + @%:@define HAVE_WORKING_REGEX 0 + @%:@error std::regex does not work + @%:@endif + ]] + )], + ax_cv_gcc_working_regex=yes, + ax_cv_gcc_working_regex=no) + AC_LANG_POP([C++]) + ] + ) + + if test "x$ax_cv_gcc_working_regex" = "xyes"; then + gcc_has_working_std_regex=yes + AC_DEFINE([HAVE_WORKING_STD_REGEX], [1], + [define if the std::regex implementation works as expected]) + fi +]) diff --git a/tests/Makefile.am b/tests/Makefile.am index 483232f..bb24660 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -100,7 +100,6 @@ api_LDFLAGS = \ @BOOST_PROGRAM_OPTIONS_LIB@ \ @BOOST_SYSTEM_LIB@ \ @BOOST_THREAD_LIB@ \ - @BOOST_REGEX_LIB@ \ @PROTOBUF_LIBS@ \ -no-install \ -Wl,-rpath,$(top_builddir)/lib/.libs \ @@ -109,6 +108,10 @@ api_LDFLAGS = \ $(top_builddir)/lib/libnornsctl_debug.la \ $(END) +if !HAVE_WORKING_STD_REGEX +api_LDFLAGS += @BOOST_REGEX_LIB@ +endif + EXTRA_api_DEPENDENCIES = \ $(top_builddir)/src/liburd_aux.la \ $(top_builddir)/lib/libnorns_debug.la \ @@ -178,13 +181,16 @@ core_LDFLAGS = \ @BOOST_PROGRAM_OPTIONS_LIB@ \ @BOOST_SYSTEM_LIB@ \ @BOOST_THREAD_LIB@ \ - @BOOST_REGEX_LIB@ \ @PROTOBUF_LIBS@ \ $(top_builddir)/src/liburd_aux.la \ $(top_builddir)/lib/libnorns_debug.la \ $(top_builddir)/lib/libnornsctl_debug.la \ $(END) +if !HAVE_WORKING_STD_REGEX +core_LDFLAGS += @BOOST_REGEX_LIB@ +endif + EXTRA_core_DEPENDENCIES = \ $(top_builddir)/src/liburd_aux.la \ $(top_builddir)/lib/libnorns_debug.la \ diff --git a/tests/test-env.cpp b/tests/test-env.cpp index f12fdc0..134956b 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -35,12 +35,19 @@ * SOFTWARE. * *************************************************************************/ + #include #include #include #include #include + +#include +#ifdef HAVE_WORKING_STD_REGEX +#include +#else #include +#endif #include #include "catch.hpp" @@ -93,6 +100,22 @@ create_config_file(const bfs::path& basedir, const bfs::path config_file = cfgdir / name; +#ifdef HAVE_WORKING_STD_REGEX + + auto outstr = std::regex_replace(config_file::cftemplate, + std::regex("@localstatedir@"), + cfgdir.string()); + + outstr = std::regex_replace(outstr, + std::regex("(staging_directory:)\\s*?\".*?\"(,?)$"), + "\\1 \"" + stdir.string() + "\"\\2"); + + for(const auto& r : reps) { + outstr = std::regex_replace(outstr, std::regex(r.first), r.second); + } + +#else + auto outstr = boost::regex_replace(config_file::cftemplate, boost::regex("@localstatedir@"), cfgdir.string()); @@ -105,6 +128,8 @@ create_config_file(const bfs::path& basedir, outstr = boost::regex_replace(outstr, boost::regex(r.first), r.second); } +#endif + bfs::ofstream outf(config_file); outf << outstr; outf.close(); -- GitLab From 1580b078a29ebca10fdc593f4633124702f49828 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 18:44:51 +0200 Subject: [PATCH 07/35] Add test job for GCC 9.1 --- .gitlab-ci.yml | 127 ++++++++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 64 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3612545..a0e6dd8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,57 +11,13 @@ stages: # - test # - deploy -# Install dependencies for GCC builds -# before_script: -# - apt-get update && -# apt-get install -y -# libboost-system-dev -# libboost-filesystem-dev -# libboost-program-options-dev -# libboost-thread-dev -# libboost-regex-dev -# libprotobuf-dev -# protobuf-compiler -# libprotobuf-c0-dev -# protobuf-c-compiler -# libyaml-cpp-dev -# libyaml-dev -# libarchive-dev -# cmake -# -# - pushd . && -# git clone https://github.com/ofiwg/libfabric.git && -# cd libfabric && -# ./autogen.sh && -# mkdir build && -# cd build && -# ../configure && -# make -j $(nproc) && -# make install && -# popd -# -# - pushd . && -# git clone https://github.com/mercury-hpc/mercury.git && -# cd mercury && -# mkdir build && -# cd build && -# cmake -# -DCMAKE_BUILD_TYPE:STRING=Debug -# -DBUILD_TESTING:BOOL=OFF -# -DMERCURY_USE_SM_ROUTING:BOOL=OFF -# -DMERCURY_USE_SELF_FORWARD:BOOL=OFF -# -DMERCURY_USE_CHECKSUMS:BOOL=OFF -# -DMERCURY_USE_BOOST_PP:BOOL=ON -# -DMERCURY_USE_EAGER_BULK:BOOL=ON -# -DBUILD_SHARED_LIBS:BOOL=ON -# -DNA_USE_OFI:BOOL=ON -# .. && -# make -j $(nproc) && -# make install && -# popd + +################################################################################ +# Build jobs +################################################################################ ### GCC 4.8 with libfabric1.7.1, mercury 1.0.1 -build:gcc:4.8: +build:gcc_4.8:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc4.8-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -78,10 +34,14 @@ build:gcc:4.8: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules + artifacts: + paths: + - build + ### GCC 4.9 with libfabric1.7.1, mercury 1.0.1 build:gcc:4.9: @@ -101,8 +61,11 @@ build:gcc:4.9: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - schedules + only: + - tags + - triggers + - schedules + ### GCC 5.4 with libfabric1.7.1, mercury 1.0.1 build:gcc:5.4: @@ -122,8 +85,11 @@ build:gcc:5.4: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - schedules + only: + - tags + - triggers + - schedules + ### GCC 6.5 with libfabric1.7.1, mercury 1.0.1 build:gcc:6.5: @@ -143,8 +109,11 @@ build:gcc:6.5: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - schedules + only: + - tags + - triggers + - schedules + ### GCC 7.4 with libfabric1.7.1, mercury 1.0.1 build:gcc:7.4: @@ -164,8 +133,11 @@ build:gcc:7.4: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - schedules + only: + - tags + - triggers + - schedules + ### GCC 8.1 with libfabric1.7.1, mercury 1.0.1 build:gcc:8.1: @@ -185,11 +157,14 @@ build:gcc:8.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - schedules + only: + - tags + - triggers + - schedules + ### GCC 9.1 with libfabric1.7.1, mercury 1.0.1 -build:gcc:9.1: +build:gcc_9.1: image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -206,10 +181,34 @@ build:gcc:9.1: make -j$(nproc) api make -j$(nproc) api_interactive ) +# This one is always run # only: +# - tags +# - triggers # - schedules +################################################################################ +# Test jobs +################################################################################ + +### GCC 9.1 with libfabric1.7.1, mercury 1.0.1 +test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + stage: test + dependencies: + - build:gcc_9.1 + variables: + NORNS_DEBUG_CONFIG_FILE_OVERRIDE: + script: + - | + ( + set -Eeuo pipefail + set -x + cd build/tests + ./core -as + ) + ### ### GCC 5 ### build:gcc:5: ### image: gcc:5 -- GitLab From fdf8cdb46bba047459794ed3763b18f59d4222da Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 18:48:06 +0200 Subject: [PATCH 08/35] Fix syntax in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a0e6dd8..c815efd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -199,7 +199,7 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: dependencies: - build:gcc_9.1 variables: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE: + - $NORNS_DEBUG_CONFIG_FILE_OVERRIDE == 1 script: - | ( -- GitLab From 00ba41af98efe7ab0c7d38a7a7c59c65acf60f2c Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 18:52:04 +0200 Subject: [PATCH 09/35] Fix missing stage in .gitlab-ci.yml --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c815efd..46bb9e2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: stages: - build - # - test + - test # - deploy @@ -199,7 +199,7 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: dependencies: - build:gcc_9.1 variables: - - $NORNS_DEBUG_CONFIG_FILE_OVERRIDE == 1 + NORNS_DEBUG_CONFIG_FILE_OVERRIDE: 1 script: - | ( -- GitLab From 786cc51a0a3e0ff6bb6c10d12827f39aa841f733 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 19:03:19 +0200 Subject: [PATCH 10/35] Add missing artifact to GCC 9.1 build job --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 46bb9e2..082a3f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,6 +181,9 @@ build:gcc_9.1: make -j$(nproc) api make -j$(nproc) api_interactive ) + artifacts: + paths: + - build # This one is always run # only: # - tags -- GitLab From 40817e5457d2f099a69afa14ce3e103f647ecf98 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 21 Jun 2019 20:52:03 +0200 Subject: [PATCH 11/35] Check build size --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 082a3f8..441a971 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,6 +181,7 @@ build:gcc_9.1: make -j$(nproc) api make -j$(nproc) api_interactive ) + - du -h . artifacts: paths: - build -- GitLab From f73c6fe82e1e9857adab2e0f61a26b23d1e18bfe Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 22 Jun 2019 00:11:48 +0200 Subject: [PATCH 12/35] Use CI_PROJECT_DIR in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 441a971..5e6ff9e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,7 +184,7 @@ build:gcc_9.1: - du -h . artifacts: paths: - - build + - "${CI_PROJECT_DIR}/build" # This one is always run # only: # - tags -- GitLab From 3cce0b5c2e75428a8e8a1c4b2b4b5efd81fed2cf Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 22 Jun 2019 10:27:29 +0200 Subject: [PATCH 13/35] Check build size using CI_PROJECT_DIR --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e6ff9e..c2e5123 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,7 +181,7 @@ build:gcc_9.1: make -j$(nproc) api make -j$(nproc) api_interactive ) - - du -h . + - du -h "${CI_PROJECT_DIR}/build" artifacts: paths: - "${CI_PROJECT_DIR}/build" -- GitLab From 35dd55cdd5da106e454477ffd75162e92e90ed64 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 15:48:04 +0200 Subject: [PATCH 14/35] Fix issues due to std::regex vs boost::regex --- tests/test-env.cpp | 44 +++++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 27 deletions(-) diff --git a/tests/test-env.cpp b/tests/test-env.cpp index 134956b..d633c51 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -43,11 +43,19 @@ #include #include + #ifdef HAVE_WORKING_STD_REGEX + #include -#else +namespace RE = std; + +#else // !HAVE_WORKING_STD_REGEX + #include -#endif +namespace RE = boost; + +#endif // HAVE_WORKING_STD_REGEX + #include #include "catch.hpp" @@ -100,36 +108,18 @@ create_config_file(const bfs::path& basedir, const bfs::path config_file = cfgdir / name; -#ifdef HAVE_WORKING_STD_REGEX - - auto outstr = std::regex_replace(config_file::cftemplate, - std::regex("@localstatedir@"), - cfgdir.string()); + auto outstr = RE::regex_replace(config_file::cftemplate, + std::regex("@localstatedir@"), + cfgdir.string()); - outstr = std::regex_replace(outstr, - std::regex("(staging_directory:)\\s*?\".*?\"(,?)$"), - "\\1 \"" + stdir.string() + "\"\\2"); + outstr = RE::regex_replace(outstr, + std::regex(R"((staging_directory:)\s*?".*?"(,?)(?=\n|$))"), + R"($1 ")" + stdir.string() + R"("$2)"); for(const auto& r : reps) { - outstr = std::regex_replace(outstr, std::regex(r.first), r.second); + outstr = RE::regex_replace(outstr, std::regex(r.first), r.second); } -#else - - auto outstr = boost::regex_replace(config_file::cftemplate, - boost::regex("@localstatedir@"), - cfgdir.string()); - - outstr = boost::regex_replace(outstr, - boost::regex("(staging_directory:)\\s*?\".*?\"(,?)$"), - "\\1 \"" + stdir.string() + "\"\\2"); - - for(const auto& r : reps) { - outstr = boost::regex_replace(outstr, boost::regex(r.first), r.second); - } - -#endif - bfs::ofstream outf(config_file); outf << outstr; outf.close(); -- GitLab From 80ddb7de0789591d59b8338063821bb21502c132 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 16:32:55 +0200 Subject: [PATCH 15/35] Reduce artifact size and set expiration time --- .gitlab-ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2e5123..3926572 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -164,7 +164,7 @@ build:gcc:8.1: ### GCC 9.1 with libfabric1.7.1, mercury 1.0.1 -build:gcc_9.1: +build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -181,10 +181,15 @@ build:gcc_9.1: make -j$(nproc) api make -j$(nproc) api_interactive ) + - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" | xargs du -ch + - du -h "${CI_PROJECT_DIR}/build" + - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" -delete - du -h "${CI_PROJECT_DIR}/build" artifacts: paths: - "${CI_PROJECT_DIR}/build" + expire_in: + - 1 day # This one is always run # only: # - tags -- GitLab From fa2aa75c7686b782f1a7f50f192fcb8a01933ead Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 16:40:02 +0200 Subject: [PATCH 16/35] Fix syntax in .gitlab-ci.yml --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3926572..4b853a0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -188,8 +188,7 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: artifacts: paths: - "${CI_PROJECT_DIR}/build" - expire_in: - - 1 day + expire_in: 1 day # This one is always run # only: # - tags @@ -206,7 +205,7 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 stage: test dependencies: - - build:gcc_9.1 + - build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1 variables: NORNS_DEBUG_CONFIG_FILE_OVERRIDE: 1 script: -- GitLab From 33842c6f304a57df7b12443b2556774472bad5de Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 17:00:21 +0200 Subject: [PATCH 17/35] Fix artifact cleanup command in .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4b853a0..7c0d7cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -183,7 +183,7 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: ) - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" | xargs du -ch - du -h "${CI_PROJECT_DIR}/build" - - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" -delete + - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" -exec rm -v {} \; - du -h "${CI_PROJECT_DIR}/build" artifacts: paths: -- GitLab From 619ac9cd5424330df0df544a8a7aaef31947dee4 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 19:08:48 +0200 Subject: [PATCH 18/35] Fix find command in .gitlab-ci.yml --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7c0d7cb..f499d28 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,9 +181,7 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) - - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" | xargs du -ch - - du -h "${CI_PROJECT_DIR}/build" - - find "${CI_PROJECT_DIR}/build" -name "*.o" -or -name "*.lo" -exec rm -v {} \; + - find "${CI_PROJECT_DIR}/build" \( -name "*.o" -or -name "*.lo" \) -delete - du -h "${CI_PROJECT_DIR}/build" artifacts: paths: -- GitLab From 0843005f778d756b55dfa5e94bd8cb516619e39e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 19:21:27 +0200 Subject: [PATCH 19/35] Re-enable API tests in .gitlab-ci.yml --- .gitlab-ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f499d28..f73d366 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -213,7 +213,40 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: set -x cd build/tests ./core -as + ./api -as "[api::NORNS_TASK]" + ./api -as "[api::norns_error]" + ./api -as "[api::norns_iotask_init]" + ./api -as "[api::norns_resource_init]" + ./api -as "[api::norns_submit]" + ./api -as "[api::norns_submit_copy_buffer_to_file]" + ./api -as "[api::norns_submit_copy_local_posix_files]" + ./api -as "[api::norns_submit_pull_errors]" + ./api -as "[api::norns_submit_pull_links]" + ./api -as "[api::norns_submit_pull_to_posix_file]" + ./api -as "[api::norns_submit_pull_to_posix_subdir]" + ./api -as "[api::norns_submit_push_errors]" + ./api -as "[api::norns_submit_push_links]" + ./api -as "[api::norns_submit_push_memory_to_posix_file]" + ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + ./api -as "[api::norns_submit_push_to_posix_file]" + ./api -as "[api::norns_submit_push_to_posix_subdir]" + ./api -as "[api::norns_submit_remove_local_posix_files]" + ./api -as "[api::nornsctl_add_process]" + ./api -as "[api::nornsctl_register_job]" + ./api -as "[api::nornsctl_register_namespace]" + ./api -as "[api::nornsctl_remove_process]" + ./api -as "[api::nornsctl_send_command]" + ./api -as "[api::nornsctl_status]" + ./api -as "[api::nornsctl_unregister_job]" + ./api -as "[api::nornsctl_unregister_namespace]" + ./api -as "[api::nornsctl_update_job]" ) + artifacts: + paths: + - "${CI_PROJECT_DIR}/build/tests" + when: on_failure + expire_in: 1 week + ### ### GCC 5 ### build:gcc:5: -- GitLab From a84ba75198ef619d720a1e1fc7264d8682a7ff6f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 19:53:05 +0200 Subject: [PATCH 20/35] Add coverage jobs --- .gitlab-ci.yml | 140 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 133 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f73d366..016eae0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,15 +1,10 @@ -# This file is a template, and might need editing before it works on your project. -# use the official gcc image, based on debian -# can use verions as well, like gcc:5.2 -# see https://hub.docker.com/_/gcc/ - variables: GIT_SUBMODULE_STRATEGY: normal stages: - build - test - # - deploy + - deploy ################################################################################ @@ -187,7 +182,7 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: paths: - "${CI_PROJECT_DIR}/build" expire_in: 1 day -# This one is always run +# This one should always run # only: # - tags # - triggers @@ -248,6 +243,137 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: expire_in: 1 week +################################################################################ +# Coverage jobs +################################################################################ + +### +build:coverage: + image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + stage: build + variables: + CFLAGS: "-O0 --coverage" + CXXFLAGS: "-O0 --coverage" + LDFLAGS: "--coverage" + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + ) + - find "${CI_PROJECT_DIR}/build" \( -name "*.o" -or -name "*.lo" \) -delete + - du -h "${CI_PROJECT_DIR}/build" + artifacts: + paths: + - "${CI_PROJECT_DIR}/build" + expire_in: 1 day + +### +test:coverage: + image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + stage: test + dependencies: + - build:coverage + variables: + NORNS_DEBUG_CONFIG_FILE_OVERRIDE: 1 + script: + - | + ( + set -Eeuo pipefail + set -x + cd build/tests + ./core -as + ./api -as "[api::NORNS_TASK]" + ./api -as "[api::norns_error]" + ./api -as "[api::norns_iotask_init]" + ./api -as "[api::norns_resource_init]" + ./api -as "[api::norns_submit]" + ./api -as "[api::norns_submit_copy_buffer_to_file]" + ./api -as "[api::norns_submit_copy_local_posix_files]" + ./api -as "[api::norns_submit_pull_errors]" + ./api -as "[api::norns_submit_pull_links]" + ./api -as "[api::norns_submit_pull_to_posix_file]" + ./api -as "[api::norns_submit_pull_to_posix_subdir]" + ./api -as "[api::norns_submit_push_errors]" + ./api -as "[api::norns_submit_push_links]" + ./api -as "[api::norns_submit_push_memory_to_posix_file]" + ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + ./api -as "[api::norns_submit_push_to_posix_file]" + ./api -as "[api::norns_submit_push_to_posix_subdir]" + ./api -as "[api::norns_submit_remove_local_posix_files]" + ./api -as "[api::nornsctl_add_process]" + ./api -as "[api::nornsctl_register_job]" + ./api -as "[api::nornsctl_register_namespace]" + ./api -as "[api::nornsctl_remove_process]" + ./api -as "[api::nornsctl_send_command]" + ./api -as "[api::nornsctl_status]" + ./api -as "[api::nornsctl_unregister_job]" + ./api -as "[api::nornsctl_unregister_namespace]" + ./api -as "[api::nornsctl_update_job]" + ) + artifacts: + paths: + - "${CI_PROJECT_DIR}/build/tests" + when: on_failure + expire_in: 1 week + + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as + ### - make -j$(nproc) api + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" + ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" + + after_script: + - (cd build && ${CI_PROJECT_DIR}/gencov.sh) + - genhtml -o build/html/coverage build/norns.info + +pages: + image: alpine:latest + stage: deploy + before_script: + - '' + dependencies: + - test:coverage + script: + - mv ${CI_PROJECT_DIR}/build/html/coverage/ ${CI_PROJECT_DIR}/public/ + artifacts: + paths: + - ${CI_PROJECT_DIR}/public + expire_in: 30 days +# only: +# - master + ### ### GCC 5 ### build:gcc:5: ### image: gcc:5 -- GitLab From f82c094b59be3085b8c62d7b1b3fd643a3697b07 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 25 Jun 2019 20:12:47 +0200 Subject: [PATCH 21/35] Propagate coverage output to deploy job --- .gitlab-ci.yml | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 016eae0..491379c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -320,40 +320,9 @@ test:coverage: ) artifacts: paths: - - "${CI_PROJECT_DIR}/build/tests" - when: on_failure + - ${CI_PROJECT_DIR}/build/html/coverage/ expire_in: 1 week - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - ### - make -j$(nproc) api - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - after_script: - (cd build && ${CI_PROJECT_DIR}/gencov.sh) - genhtml -o build/html/coverage build/norns.info -- GitLab From 46a67fbecbf1722a7f57e14dfc7aebc2dbc428df Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 00:09:34 +0200 Subject: [PATCH 22/35] Fix coverage paths --- .gitlab-ci.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 491379c..a983667 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -318,14 +318,15 @@ test:coverage: ./api -as "[api::nornsctl_unregister_namespace]" ./api -as "[api::nornsctl_update_job]" ) + after_script: + - (cd build && ${CI_PROJECT_DIR}/gencov.sh) + - genhtml -o build/html/coverage build/norns.info + artifacts: paths: - - ${CI_PROJECT_DIR}/build/html/coverage/ + - build/html/coverage/ expire_in: 1 week - after_script: - - (cd build && ${CI_PROJECT_DIR}/gencov.sh) - - genhtml -o build/html/coverage build/norns.info pages: image: alpine:latest @@ -335,10 +336,10 @@ pages: dependencies: - test:coverage script: - - mv ${CI_PROJECT_DIR}/build/html/coverage/ ${CI_PROJECT_DIR}/public/ + - mv build/html/coverage/ public/ artifacts: paths: - - ${CI_PROJECT_DIR}/public + - public expire_in: 30 days # only: # - master -- GitLab From ca54174b86c60e2a505c100ed982787d5b27b99d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 08:06:21 +0200 Subject: [PATCH 23/35] Fix incorrect std::regex() usage --- tests/test-env.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test-env.cpp b/tests/test-env.cpp index d633c51..3aca4d3 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -109,15 +109,15 @@ create_config_file(const bfs::path& basedir, const bfs::path config_file = cfgdir / name; auto outstr = RE::regex_replace(config_file::cftemplate, - std::regex("@localstatedir@"), + RE::regex("@localstatedir@"), cfgdir.string()); outstr = RE::regex_replace(outstr, - std::regex(R"((staging_directory:)\s*?".*?"(,?)(?=\n|$))"), + RE::regex(R"((staging_directory:)\s*?".*?"(,?)(?=\n|$))"), R"($1 ")" + stdir.string() + R"("$2)"); for(const auto& r : reps) { - outstr = RE::regex_replace(outstr, std::regex(r.first), r.second); + outstr = RE::regex_replace(outstr, RE::regex(r.first), r.second); } bfs::ofstream outf(config_file); -- GitLab From 6e8c951bf2ddc4b947590c4b11d64a713f86fb4e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 09:42:22 +0200 Subject: [PATCH 24/35] CI: Remove outdated jobs and rename build jobs --- .gitlab-ci.yml | 321 +------------------------------------------------ 1 file changed, 4 insertions(+), 317 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a983667..832c1fa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -63,7 +63,7 @@ build:gcc:4.9: ### GCC 5.4 with libfabric1.7.1, mercury 1.0.1 -build:gcc:5.4: +build:gcc_5.4:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc5.4-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -87,7 +87,7 @@ build:gcc:5.4: ### GCC 6.5 with libfabric1.7.1, mercury 1.0.1 -build:gcc:6.5: +build:gcc_6.5:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc6.5-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -111,7 +111,7 @@ build:gcc:6.5: ### GCC 7.4 with libfabric1.7.1, mercury 1.0.1 -build:gcc:7.4: +build:gcc_7.4:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc7.4-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -135,7 +135,7 @@ build:gcc:7.4: ### GCC 8.1 with libfabric1.7.1, mercury 1.0.1 -build:gcc:8.1: +build:gcc_8.1:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc8.1-libfabric1.7.1-mercury1.0.1 stage: build script: @@ -343,316 +343,3 @@ pages: expire_in: 30 days # only: # - master - - ### ### GCC 5 - ### build:gcc:5: - ### image: gcc:5 - ### stage: build - ### - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - ### - cd tests - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api - ### - ### - ### ### GCC 6 - ### build:gcc:6: - ### image: gcc:6 - ### stage: build - ### - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - ### - cd tests - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - ### - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api - ### - ### - ### ### GCC 7 - ### build:gcc:7: - ### image: gcc:7 - ### stage: build - ### - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### - make -j$(nproc) - ### - cd tests - ### - make -j$(nproc) - ### - make -j$(nproc) - ### - ### - ### ### GCC 8 - ### build:gcc:8: - ### image: gcc:8 - ### stage: build - ### - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### - make -j$(nproc) - ### - cd tests - ### - make -j$(nproc) - ### - make -j$(nproc) - ### - ### - ### ################################################################################ - ### # test scripts - ### ################################################################################ - ### test:coverage: - ### image: ubuntu:latest - ### stage: test - ### - ### # Install dependencies - ### before_script: - ### - apt-get update && - ### apt-get upgrade -y && - ### apt-get install -y - ### git - ### build-essential - ### autotools-dev - ### automake - ### autoconf - ### libtool - ### pkg-config - ### libboost-system-dev - ### libboost-filesystem-dev - ### libboost-program-options-dev - ### libboost-thread-dev - ### libboost-regex-dev - ### libprotobuf-dev - ### protobuf-compiler - ### libprotobuf-c-dev - ### protobuf-c-compiler - ### libyaml-cpp-dev - ### libyaml-dev - ### libarchive-dev - ### libcap2-bin - ### valgrind - ### cmake - ### lcov - ### - ### - pushd . && - ### git clone https://github.com/ofiwg/libfabric.git && - ### cd libfabric && - ### ./autogen.sh && - ### mkdir build && - ### cd build && - ### ../configure && - ### make -j $(nproc) && - ### make install && - ### popd && - ### ldconfig - ### - ### - pushd . && - ### git clone https://github.com/mercury-hpc/mercury.git && - ### cd mercury && - ### mkdir build && - ### cd build && - ### cmake - ### -DCMAKE_BUILD_TYPE:STRING=Debug - ### -DBUILD_TESTING:BOOL=OFF - ### -DMERCURY_USE_SM_ROUTING:BOOL=OFF - ### -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - ### -DMERCURY_USE_CHECKSUMS:BOOL=OFF - ### -DMERCURY_USE_BOOST_PP:BOOL=ON - ### -DMERCURY_USE_EAGER_BULK:BOOL=ON - ### -DBUILD_SHARED_LIBS:BOOL=ON - ### -DNA_USE_OFI:BOOL=ON - ### .. && - ### make -j $(nproc) && - ### make install && - ### popd && - ### ldconfig - ### - ### # Build and test - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### CFLAGS="-O0 --coverage" - ### CXXFLAGS="-O0 --coverage" - ### LDFLAGS="--coverage" - ### - make -j$(nproc) - ### - cd tests - ### - make -j$(nproc) core - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - ### - make -j$(nproc) api - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - ### - ### after_script: - ### - (cd build && ../gencov.sh) - ### - genhtml -o build/html/coverage build/norns.info - ### - if [[ -e build/tests.log ]]; - ### then - ### cat $(tail -1 build/tests.log)/config/urd.log; - ### fi - ### - ### artifacts: - ### paths: - ### - build/html/coverage/ - ### - ### test:optimized: - ### image: ubuntu:latest - ### stage: test - ### - ### # Install dependencies - ### before_script: - ### - apt-get update && - ### apt-get upgrade -y && - ### apt-get install -y - ### git - ### build-essential - ### autotools-dev - ### automake - ### autoconf - ### libtool - ### pkg-config - ### libboost-system-dev - ### libboost-filesystem-dev - ### libboost-program-options-dev - ### libboost-thread-dev - ### libboost-regex-dev - ### libprotobuf-dev - ### protobuf-compiler - ### libprotobuf-c-dev - ### protobuf-c-compiler - ### libyaml-cpp-dev - ### libyaml-dev - ### libarchive-dev - ### libcap2-bin - ### valgrind - ### cmake - ### - ### - pushd . && - ### git clone https://github.com/ofiwg/libfabric.git && - ### cd libfabric && - ### ./autogen.sh && - ### mkdir build && - ### cd build && - ### ../configure && - ### make -j $(nproc) && - ### make install && - ### popd && - ### ldconfig - ### - ### - pushd . && - ### git clone https://github.com/mercury-hpc/mercury.git && - ### cd mercury && - ### mkdir build && - ### cd build && - ### cmake - ### -DCMAKE_BUILD_TYPE:STRING=Debug - ### -DBUILD_TESTING:BOOL=OFF - ### -DMERCURY_USE_SM_ROUTING:BOOL=OFF - ### -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - ### -DMERCURY_USE_CHECKSUMS:BOOL=OFF - ### -DMERCURY_USE_BOOST_PP:BOOL=ON - ### -DMERCURY_USE_EAGER_BULK:BOOL=ON - ### -DBUILD_SHARED_LIBS:BOOL=ON - ### -DNA_USE_OFI:BOOL=ON - ### .. && - ### make -j $(nproc) && - ### make install && - ### popd && - ### ldconfig - ### - ### # Build and test - ### script: - ### - ./bootstrap.sh - ### - mkdir build && cd build - ### - ../configure - ### --enable-tests - ### - make -j$(nproc) - ### - cd tests - ### - make -j$(nproc) core - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - ### - make -j$(nproc) api - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_error]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - ### - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - ### - ### after_script: - ### - if [[ -e build/tests.log ]]; - ### then - ### cat $(tail -1 build/tests.log)/config/urd.log; - ### fi - ### - ### pages: - ### image: alpine:latest - ### stage: deploy - ### before_script: - ### - '' - ### dependencies: - ### - test:coverage - ### script: - ### - mv build/html/coverage/ public/ - ### artifacts: - ### paths: - ### - public - ### expire_in: 30 days - ### only: - ### - master -- GitLab From cfbe9132e7faf3553aafabd11d96f351f4cff072 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 09:44:14 +0200 Subject: [PATCH 25/35] CI: Improve error reporting in coverage job --- .gitlab-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 832c1fa..efc47af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -319,8 +319,14 @@ test:coverage: ./api -as "[api::nornsctl_update_job]" ) after_script: - - (cd build && ${CI_PROJECT_DIR}/gencov.sh) - - genhtml -o build/html/coverage build/norns.info + - | + ( + set -Eeuo pipefail + set -x + cd build + ${CI_PROJECT_DIR}/gencov.sh + genhtml -o build/html/coverage build/norns.info + ) artifacts: paths: -- GitLab From 0bc67763856b7d6413050e73dc162dad090420ee Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 10:54:24 +0200 Subject: [PATCH 26/35] CI: Fail coverage job if norns.info does not exist --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index efc47af..6d85f0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -325,7 +325,8 @@ test:coverage: set -x cd build ${CI_PROJECT_DIR}/gencov.sh - genhtml -o build/html/coverage build/norns.info + [[ -e norns.info ]] || false + genhtml -o html/coverage norns.info ) artifacts: -- GitLab From a34918d0f4344f72438ea65084103846f1fdf2f6 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 15:09:15 +0200 Subject: [PATCH 27/35] Fix AM_CONDITIONAL in configure.ac --- configure.ac | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 2cc5a48..24a93e7 100644 --- a/configure.ac +++ b/configure.ac @@ -125,21 +125,20 @@ AX_BOOST_ASIO AX_BOOST_PROGRAM_OPTIONS AX_BOOST_THREAD +# check whether the compiler has a functional std::regex +AX_GCC_WORKING_STD_REGEX +AS_IF([test "x$gcc_has_working_std_regex" = xno], + [ + # fallback to boost::regex + AC_MSG_WARN([falling back to boost::regex because std::regex is not functional]) + AX_BOOST_REGEX + ], []) +AM_CONDITIONAL([HAVE_WORKING_STD_REGEX], + test x$GCC_HAS_WORKING_STD_REGEX = xyes) + # only check for std::regex/boost::regex if we are building tests AS_IF([test "x$is_enabled_build_tests" = "xyes"], - [ - # check whether the compiler has a functional std::regex - AX_GCC_WORKING_STD_REGEX - AS_IF([test "x$gcc_has_working_std_regex" = xno], - [ - # fallback to boost::regex - AC_MSG_WARN([falling back to boost::regex because std::regex is not functional]) - AX_BOOST_REGEX - ], []) - AM_CONDITIONAL([HAVE_WORKING_STD_REGEX], - test x$GCC_HAS_WORKING_STD_REGEX = xyes) - AC_CONFIG_FILES(tests/Makefile) - ], []) + [ AC_CONFIG_FILES(tests/Makefile) ], []) # check for mercury PKG_CHECK_MODULES([MERCURY], [mercury >= 0.26]) -- GitLab From 09912545725ac75640dfdcb22eb479b7c27b1623 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 19:00:36 +0200 Subject: [PATCH 28/35] CI: Add build jobs for Clang 3.6/3.7 --- .gitlab-ci.yml | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6d85f0e..040c426 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,10 @@ stages: # Build jobs ################################################################################ +################################################################################ +######################### GCC ######################################### +################################################################################ + ### GCC 4.8 with libfabric1.7.1, mercury 1.0.1 build:gcc_4.8:libfabric_1.7.1:mercury_1.0.1: image: bscstorage/norns-build-deps:gcc4.8-libfabric1.7.1-mercury1.0.1 @@ -189,6 +193,63 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: # - schedules +################################################################################ +######################### LLVM/Clang ######################################### +################################################################################ + +### Clang 3.6 with libfabric1.7.1, mercury 1.0.1 +build:clang_3.6:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang3.6-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) + only: + - tags + - triggers + - schedules + artifacts: + paths: + - build + + +### Clang 3.7 with libfabric1.7.1, mercury 1.0.1 +build:clang_3.7:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang3.7-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) + only: + - tags + - triggers + - schedules + artifacts: + paths: + - build + ################################################################################ # Test jobs ################################################################################ -- GitLab From 18c6ebf6583230f530710d4dd3d4131fb0f3530d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 19:04:16 +0200 Subject: [PATCH 29/35] CI: Enable Clang build jobs --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 040c426..8160a87 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -215,10 +215,10 @@ build:clang_3.6:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) - only: - - tags - - triggers - - schedules +# only: +# - tags +# - triggers +# - schedules artifacts: paths: - build @@ -242,10 +242,10 @@ build:clang_3.7:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) - only: - - tags - - triggers - - schedules +# only: +# - tags +# - triggers +# - schedules artifacts: paths: - build -- GitLab From 12f8a046064bb36f89b4e08bba984459add87e41 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 20:10:18 +0200 Subject: [PATCH 30/35] CI: Remove job for Clang 3.7, add jobs for 3.8/3.9 --- .gitlab-ci.yml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8160a87..c83cd52 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -185,7 +185,7 @@ build:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: artifacts: paths: - "${CI_PROJECT_DIR}/build" - expire_in: 1 day + expire_in: 12 hours # This one should always run # only: # - tags @@ -219,14 +219,33 @@ build:clang_3.6:libfabric_1.7.1:mercury_1.0.1: # - tags # - triggers # - schedules - artifacts: - paths: - - build +### Clang 3.8 with libfabric1.7.1, mercury 1.0.1 +build:clang_3.8:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang3.8-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules -### Clang 3.7 with libfabric1.7.1, mercury 1.0.1 -build:clang_3.7:libfabric_1.7.1:mercury_1.0.1: - image: bscstorage/norns-build-deps:clang3.7-libfabric1.7.1-mercury1.0.1 +### Clang 3.9 with libfabric1.7.1, mercury 1.0.1 +build:clang_3.9:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang3.9-libfabric1.7.1-mercury1.0.1 stage: build script: - | @@ -246,9 +265,6 @@ build:clang_3.7:libfabric_1.7.1:mercury_1.0.1: # - tags # - triggers # - schedules - artifacts: - paths: - - build ################################################################################ # Test jobs -- GitLab From f3e727d9576d421a502162e39743d8a13a7ed1f2 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 20:59:15 +0200 Subject: [PATCH 31/35] CI: Add build job for Clang 4.0 --- .gitlab-ci.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c83cd52..a7cfe05 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -266,6 +266,29 @@ build:clang_3.9:libfabric_1.7.1:mercury_1.0.1: # - triggers # - schedules +### Clang 4.0 with libfabric1.7.1, mercury 1.0.1 +build:clang_4.0:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang4.0-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + ################################################################################ # Test jobs ################################################################################ -- GitLab From 2a52a0a41c45b3f6f94137cc2092817c3922e415 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 26 Jun 2019 23:07:50 +0200 Subject: [PATCH 32/35] Add build jobs for Clang 5, 6, 7, 8, and 9 --- .gitlab-ci.yml | 119 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7cfe05..2712ffd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -289,6 +289,125 @@ build:clang_4.0:libfabric_1.7.1:mercury_1.0.1: # - triggers # - schedules +### Clang 5.0 with libfabric1.7.1, mercury 1.0.1 +build:clang_5.0:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang5.0-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + +### Clang 6.0 with libfabric1.7.1, mercury 1.0.1 +build:clang_6.0:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang6.0-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + +### Clang 7.1 with libfabric1.7.1, mercury 1.0.1 +build:clang_7.1:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang7.1-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + +### Clang 8.0 with libfabric1.7.1, mercury 1.0.1 +build:clang_8.0:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang8.0-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + +### Clang 9.0 with libfabric1.7.1, mercury 1.0.1 +build:clang_9.0:libfabric_1.7.1:mercury_1.0.1: + image: bscstorage/norns-build-deps:clang9.0-libfabric1.7.1-mercury1.0.1 + stage: build + script: + - | + ( + set -Eeuo pipefail + set -x + ./bootstrap.sh + mkdir build && cd build + ../configure --enable-tests + make -j$(nproc) + cd tests + make -j$(nproc) core + make -j$(nproc) api + make -j$(nproc) api_interactive + ) +# only: +# - tags +# - triggers +# - schedules + + + + + ################################################################################ # Test jobs ################################################################################ -- GitLab From 4b3d75290e6b403fc499b5c666f85b7b8fa9e9de Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 27 Jun 2019 08:54:34 +0200 Subject: [PATCH 33/35] Run some clang jobs only on schedules --- .gitlab-ci.yml | 68 ++++++++++++++++++++++++-------------------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2712ffd..ae55e50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -215,10 +215,10 @@ build:clang_3.6:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 3.8 with libfabric1.7.1, mercury 1.0.1 build:clang_3.8:libfabric_1.7.1:mercury_1.0.1: @@ -238,10 +238,10 @@ build:clang_3.8:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 3.9 with libfabric1.7.1, mercury 1.0.1 build:clang_3.9:libfabric_1.7.1:mercury_1.0.1: @@ -261,10 +261,10 @@ build:clang_3.9:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 4.0 with libfabric1.7.1, mercury 1.0.1 build:clang_4.0:libfabric_1.7.1:mercury_1.0.1: @@ -284,10 +284,10 @@ build:clang_4.0:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 5.0 with libfabric1.7.1, mercury 1.0.1 build:clang_5.0:libfabric_1.7.1:mercury_1.0.1: @@ -307,10 +307,10 @@ build:clang_5.0:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 6.0 with libfabric1.7.1, mercury 1.0.1 build:clang_6.0:libfabric_1.7.1:mercury_1.0.1: @@ -330,10 +330,10 @@ build:clang_6.0:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 7.1 with libfabric1.7.1, mercury 1.0.1 build:clang_7.1:libfabric_1.7.1:mercury_1.0.1: @@ -353,10 +353,10 @@ build:clang_7.1:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 8.0 with libfabric1.7.1, mercury 1.0.1 build:clang_8.0:libfabric_1.7.1:mercury_1.0.1: @@ -376,10 +376,10 @@ build:clang_8.0:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) -# only: -# - tags -# - triggers -# - schedules + only: + - tags + - triggers + - schedules ### Clang 9.0 with libfabric1.7.1, mercury 1.0.1 build:clang_9.0:libfabric_1.7.1:mercury_1.0.1: @@ -399,15 +399,13 @@ build:clang_9.0:libfabric_1.7.1:mercury_1.0.1: make -j$(nproc) api make -j$(nproc) api_interactive ) +# This one should always run # only: # - tags # - triggers # - schedules - - - ################################################################################ # Test jobs ################################################################################ -- GitLab From c214e5d99d2f1f62eafa218687d208a95c91ad4e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 27 Jun 2019 10:25:09 +0200 Subject: [PATCH 34/35] Use GCC 8 to generate coverage reports --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ae55e50..b0d856d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -466,7 +466,7 @@ test:gcc_9.1:libfabric_1.7.1:mercury_1.0.1: ### build:coverage: - image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + image: bscstorage/norns-build-deps:gcc8.1-libfabric1.7.1-mercury1.0.1 stage: build variables: CFLAGS: "-O0 --coverage" @@ -494,7 +494,7 @@ build:coverage: ### test:coverage: - image: bscstorage/norns-build-deps:gcc9.1-libfabric1.7.1-mercury1.0.1 + image: bscstorage/norns-build-deps:gcc8.1-libfabric1.7.1-mercury1.0.1 stage: test dependencies: - build:coverage -- GitLab From aa6305fe35d89b734c764f4a42b99b9dbff58305 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 27 Jun 2019 16:29:55 +0200 Subject: [PATCH 35/35] Move lcov report generation to script section --- .gitlab-ci.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0d856d..c67216a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -534,15 +534,8 @@ test:coverage: ./api -as "[api::nornsctl_unregister_job]" ./api -as "[api::nornsctl_unregister_namespace]" ./api -as "[api::nornsctl_update_job]" - ) - after_script: - - | - ( - set -Eeuo pipefail - set -x - cd build + cd ${CI_PROJECT_DIR}/build ${CI_PROJECT_DIR}/gencov.sh - [[ -e norns.info ]] || false genhtml -o html/coverage norns.info ) -- GitLab