From 58226888fe04d2969f1fc8a936fa1114fe3a15ea Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 8 Sep 2022 14:19:46 +0200 Subject: [PATCH 1/4] Add runner.sh script to start/stop daemons as CTest fixtures See https://discourse.cmake.org/t/ctest-able-to-daemonize-fixtures/1012/3 for a rationale. --- scripts/runner.sh | 110 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 scripts/runner.sh diff --git a/scripts/runner.sh b/scripts/runner.sh new file mode 100755 index 00000000..82c766d1 --- /dev/null +++ b/scripts/runner.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +function help() { + echo "Usage:" + echo " $(basename "$0") COMMAND" + echo "" + echo " Where COMMAND is one of:" + echo " start PIDFILE PROGRAM ARGS... Run PROGRAM and record its PID in PIDFILE" + echo " stop SIGNAL PIDFILE Send SIGNAL to the PID contained in PIDFILE" + echo " help Print this message" +} + +function readpid() { + + if [ $# -eq 0 ]; then + echo "FATAL: readpid(): Missing pifile" >&2 + exit 1 + fi + + pidfile="$1" + + read -r pid <"$pidfile" + echo $pid +} + +function run() { + + if [ $# -eq 0 ]; then + echo "FATAL: missing program pidfile" >&2 + elif [ $# -eq 1 ]; then + echo "FATAL: missing program to run" >&2 + help + exit 1 + fi + + pidfile="$1" + shift + + if [ -e "$pidfile" ]; then + pid=$(readpid "$pidfile") + echo "$pid" + + if pgrep --pidfile "$pidfile"; then + exit 1 + fi + fi + + "$@" 2>/dev/null 1>/dev/null 0"$pidfile" + sleep 1 + + if ! kill -0 $pid; then + echo "Process $pid does not seem to exist." >&2 + echo "The program below may not exist or may have crashed while starting:" >&2 + echo " $*" >&2 + exit 1 + fi + + exit 0 +} + +function stop() { + + if [ $# -eq 0 ]; then + echo "FATAL: missing signal" >&2 + exit 1 + elif [ $# -eq 1 ]; then + echo "FATAL: missing pidfile" >&2 + exit 1 + fi + + signal="$1" + pidfile="$2" + + if [ ! -e "$pidfile" ]; then + echo "FATAL: pidfile '$pidfile' does not exist" >&2 + exit 1 + fi + + if pkill "-$signal" --pidfile "$pidfile"; then + rm "$pidfile" + exit 0 + fi + exit 1 +} + +if [ $# -eq 0 ]; then + echo "FATAL: missing arguments" >&2 + help + exit 1 +fi + +case $1 in + +start) + shift + run "$@" + ;; + +stop) + shift + stop "$@" + ;; + +help) + help + exit 0 + ;; +esac -- GitLab From fd3536e75346e33181ebf2dd572a655da516a840 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 9 Sep 2022 10:03:45 +0200 Subject: [PATCH 2/4] Add missing c/ADM_ping example --- examples/c/ADM_ping.c | 49 +++++++++++++++++++++++++++++++++++++++ examples/c/CMakeLists.txt | 2 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 examples/c/ADM_ping.c diff --git a/examples/c/ADM_ping.c b/examples/c/ADM_ping.c new file mode 100644 index 00000000..0140a9d1 --- /dev/null +++ b/examples/c/ADM_ping.c @@ -0,0 +1,49 @@ +/****************************************************************************** + * Copyright 2021-2022, Barcelona Supercomputing Center (BSC), Spain + * + * This software was partially supported by the EuroHPC-funded project ADMIRE + * (Project ID: 956748, https://www.admire-eurohpc.eu). + * + * This file is part of scord. + * + * scord is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * scord is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with scord. If not, see . + * + * SPDX-License-Identifier: GPL-3.0-or-later + *****************************************************************************/ + +#include +#include +#include +#include + +int +main(int argc, char* argv[]) { + + if(argc != 2) { + fprintf(stderr, "ERROR: no location provided\n"); + fprintf(stderr, "Usage: ADM_ping \n"); + exit(EXIT_FAILURE); + } + + ADM_server_t server = ADM_server_create("tcp", argv[1]); + + ADM_return_t ret = ADM_ping(server); + + if(ret != ADM_SUCCESS) { + fprintf(stdout, "ADM_ping() remote procedure not completed " + "successfully\n"); + exit(EXIT_FAILURE); + } + exit(EXIT_SUCCESS); +} diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index dedc9f83..df9e4fd8 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -22,7 +22,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -list(APPEND examples_c ADM_register_job ADM_cancel_transfer ADM_connect_data_operation ADM_define_data_operation +list(APPEND examples_c ADM_ping ADM_register_job ADM_cancel_transfer ADM_connect_data_operation ADM_define_data_operation ADM_deploy_adhoc_storage ADM_finalize_data_operation ADM_get_pending_transfers ADM_get_qos_constraints ADM_get_statistics ADM_get_transfer_priority ADM_link_transfer_to_data_operation ADM_register_adhoc_storage ADM_remove_adhoc_storage ADM_remove_job ADM_set_dataset_information -- GitLab From 5e6df41a5b531784671ac0a58589735d6db45426 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 9 Sep 2022 10:05:02 +0200 Subject: [PATCH 3/4] RPC tests now depend on start|stop_scord_daemon fixtures --- examples/CMakeLists.txt | 17 ++++++++ examples/c/CMakeLists.txt | 85 ++++++++++++------------------------- examples/cxx/CMakeLists.txt | 79 ++++++++++------------------------ 3 files changed, 67 insertions(+), 114 deletions(-) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 91d8ee9b..931da9ce 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -22,5 +22,22 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ +if(SCORD_BUILD_TESTS) + add_test(start_scord_daemon + ${CMAKE_SOURCE_DIR}/scripts/runner.sh start scord.pid + ${CMAKE_BINARY_DIR}/src/scord/scord -C -f + ) + + set_tests_properties(start_scord_daemon PROPERTIES FIXTURES_SETUP + scord_daemon) + + add_test(stop_scord_daemon + ${CMAKE_SOURCE_DIR}/scripts/runner.sh stop TERM scord.pid + ) + + set_tests_properties(stop_scord_daemon PROPERTIES FIXTURES_CLEANUP + scord_daemon) +endif() + add_subdirectory(c) add_subdirectory(cxx) diff --git a/examples/c/CMakeLists.txt b/examples/c/CMakeLists.txt index df9e4fd8..7dd51efd 100644 --- a/examples/c/CMakeLists.txt +++ b/examples/c/CMakeLists.txt @@ -22,21 +22,33 @@ # SPDX-License-Identifier: GPL-3.0-or-later # ################################################################################ -list(APPEND examples_c ADM_ping ADM_register_job ADM_cancel_transfer ADM_connect_data_operation ADM_define_data_operation - ADM_deploy_adhoc_storage ADM_finalize_data_operation ADM_get_pending_transfers ADM_get_qos_constraints - ADM_get_statistics ADM_get_transfer_priority ADM_link_transfer_to_data_operation - ADM_register_adhoc_storage ADM_remove_adhoc_storage ADM_remove_job ADM_set_dataset_information - ADM_set_io_resources ADM_set_qos_constraints ADM_set_transfer_priority ADM_transfer_dataset - ADM_update_adhoc_storage ADM_update_job ADM_register_pfs_storage - ADM_update_pfs_storage ADM_remove_pfs_storage) - -# ADM_in_situ_ops ADM_in_transit_ops not implemented +list(APPEND examples_c + # ping + ADM_ping + # job + ADM_register_job ADM_update_job ADM_remove_job + # adhoc storage + ADM_register_adhoc_storage ADM_update_adhoc_storage ADM_remove_adhoc_storage + ADM_deploy_adhoc_storage + # pfs storage + ADM_register_pfs_storage ADM_update_pfs_storage ADM_remove_pfs_storage + # transfers + ADM_transfer_dataset ADM_get_transfer_priority ADM_set_transfer_priority + ADM_cancel_transfer ADM_get_pending_transfers + # qos + ADM_set_qos_constraints ADM_get_qos_constraints + # data operations + ADM_define_data_operation ADM_connect_data_operation + ADM_finalize_data_operation ADM_link_transfer_to_data_operation + # ADM_in_situ_ops ADM_in_transit_ops + # misc + ADM_get_statistics ADM_set_dataset_information ADM_set_io_resources + ) add_library(c_examples_common STATIC) target_sources(c_examples_common PUBLIC common.h PRIVATE common.c) target_link_libraries(c_examples_common common::api::types) - foreach(example IN LISTS examples_c) add_executable(${example}_c) target_sources(${example}_c PRIVATE ${example}.c) @@ -45,52 +57,9 @@ foreach(example IN LISTS examples_c) endforeach() if(SCORD_BUILD_TESTS) - add_test(ADM_register_job_c_test ADM_register_job ofi+tcp://127.0.0.1:52000) - - add_test(ADM_cancel_transfer_c_test ADM_cancel_transfer ofi+tcp://127.0.0.1:52000) - - add_test(ADM_connect_data_operation_c_test ADM_connect_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_define_data_operation_c_test ADM_define_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_deploy_adhoc_storage_c_test ADM_deploy_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_finalize_data_operation_c_test ADM_finalize_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_pending_transfers_c_test ADM_get_pending_transfers ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_qos_constraints_c_test ADM_get_qos_constraints ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_statistics_c_test ADM_get_statistics ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_transfer_priority_c_test ADM_get_transfer_priority ofi+tcp://127.0.0.1:52000) - - add_test(ADM_link_transfer_to_data_operation_c_test ADM_link_transfer_to_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_register_adhoc_storage_c_test ADM_register_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_register_pfs_storage_c_test ADM_register_pfs_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_remove_adhoc_storage_c_test ADM_remove_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_remove_job_c_test ADM_remove_job ofi+tcp://127.0.0.1:52000) - - # TODO: ADM_remove_pfs_storage test is missing because is not working in cpp. - # Will be created when it works in cpp. - add_test(ADM_set_dataset_information_c_test ADM_set_dataset_information ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_io_resources_c_test ADM_set_io_resources ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_qos_constraints_c_test ADM_set_qos_constraints ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_transfer_priority_c_test ADM_set_transfer_priority ofi+tcp://127.0.0.1:52000) - - add_test(ADM_transfer_dataset_c_test ADM_transfer_dataset ofi+tcp://127.0.0.1:52000) - - add_test(ADM_update_adhoc_storage_c_test ADM_update_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_update_job_c_test ADM_update_job ofi+tcp://127.0.0.1:52000) - - # TODO: ADM_update_pfs_storage test is missing because is not working in cpp. - # Will be created when it works in cpp. + foreach(example IN LISTS examples_c) + add_test(${example}_c_test ${example} ofi+tcp://${SCORD_BIND_ADDRESS}:${SCORD_BIND_PORT}) + set_tests_properties(${example}_c_test + PROPERTIES FIXTURES_REQUIRED scord_daemon) + endforeach() endif() diff --git a/examples/cxx/CMakeLists.txt b/examples/cxx/CMakeLists.txt index 2428d8bd..1263a187 100644 --- a/examples/cxx/CMakeLists.txt +++ b/examples/cxx/CMakeLists.txt @@ -23,19 +23,27 @@ ################################################################################ list(APPEND examples_cxx + # ping ADM_ping + # job ADM_register_job ADM_update_job ADM_remove_job - ADM_register_adhoc_storage ADM_update_adhoc_storage - ADM_remove_adhoc_storage ADM_deploy_adhoc_storage - ADM_register_pfs_storage ADM_update_pfs_storage - ADM_remove_pfs_storage - + # adhoc storage + ADM_register_adhoc_storage ADM_update_adhoc_storage ADM_remove_adhoc_storage + ADM_deploy_adhoc_storage + # pfs storage + ADM_register_pfs_storage ADM_update_pfs_storage ADM_remove_pfs_storage + # transfers + ADM_transfer_dataset ADM_get_transfer_priority ADM_set_transfer_priority + ADM_cancel_transfer ADM_get_pending_transfers + # qos + ADM_set_qos_constraints ADM_get_qos_constraints + # data operations + ADM_define_data_operation ADM_connect_data_operation + ADM_finalize_data_operation ADM_link_transfer_to_data_operation # ADM_in_situ_ops ADM_in_transit_ops - ADM_transfer_dataset - ADM_set_dataset_information ADM_set_io_resources ADM_get_transfer_priority - ADM_set_transfer_priority ADM_cancel_transfer ADM_get_pending_transfers - ADM_set_qos_constraints ADM_get_qos_constraints ADM_define_data_operation ADM_connect_data_operation - ADM_finalize_data_operation ADM_link_transfer_to_data_operation ADM_get_statistics) + #misc + ADM_get_statistics ADM_set_dataset_information ADM_set_io_resources + ) add_library(cxx_examples_common STATIC) target_sources(cxx_examples_common PUBLIC common.hpp PRIVATE common.cpp) @@ -50,50 +58,9 @@ foreach(example IN LISTS examples_cxx) endforeach() if(SCORD_BUILD_TESTS) - add_test(ADM_ping_test ADM_ping ofi+tcp://127.0.0.1:52000) - - add_test(ADM_cancel_transfer_cxx_test ADM_cancel_transfer ofi+tcp://127.0.0.1:52000) - set_tests_properties(ADM_cancel_transfer_cxx_test PROPERTIES LABELS "ADM_cancel_transfer;cxx") - - add_test(ADM_connect_data_operation_cxx_test ADM_connect_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_define_data_operation_cxx_test ADM_define_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_deploy_adhoc_storage_cxx_test ADM_deploy_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_finalize_data_operation_cxx_test ADM_finalize_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_pending_transfers_cxx_test ADM_get_pending_transfers ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_qos_constraints_cxx_test ADM_get_qos_constraints ofi+tcp://127.0.0.1:52000) - - add_test(ADM_get_transfer_priority_cxx_test ADM_get_transfer_priority ofi+tcp://127.0.0.1:52000) - - add_test(ADM_link_transfer_to_data_operation_cxx_test ADM_link_transfer_to_data_operation ofi+tcp://127.0.0.1:52000) - - add_test(ADM_register_job_cxx_test ADM_register_job ofi+tcp://127.0.0.1:52000) - - add_test(ADM_register_pfs_storage_cxx_test ADM_register_pfs_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_remove_adhoc_storage_cxx_test ADM_remove_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_remove_job_cxx_test ADM_remove_job ofi+tcp://127.0.0.1:52000) - - add_test(ADM_remove_pfs_storage_cxx_test ADM_remove_pfs_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_dataset_information_cxx_test ADM_set_dataset_information ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_io_resources_cxx_test ADM_set_io_resources ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_qos_constraints_cxx_test ADM_set_qos_constraints ofi+tcp://127.0.0.1:52000) - - add_test(ADM_set_transfer_priority_cxx_test ADM_set_transfer_priority ofi+tcp://127.0.0.1:52000) - - add_test(ADM_transfer_dataset_cxx_test ADM_transfer_dataset ofi+tcp://127.0.0.1:52000) - - add_test(ADM_update_adhoc_storage_cxx_test ADM_update_adhoc_storage ofi+tcp://127.0.0.1:52000) - - add_test(ADM_update_job_cxx_test ADM_update_job ofi+tcp://127.0.0.1:52000) - - # TODO: add_test(ADM_update_pfs_storage_cxx_test ADM_update_pfs_storage ofi+tcp://127.0.0.1:52000 42) + foreach(example IN LISTS examples_cxx) + add_test(${example}_cxx_test ${example} ofi+tcp://${SCORD_BIND_ADDRESS}:${SCORD_BIND_PORT}) + set_tests_properties(${example}_cxx_test + PROPERTIES FIXTURES_REQUIRED scord_daemon) + endforeach() endif() -- GitLab From 3c875eca3473db0ca6250e417bf8a5f05833de5a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 9 Sep 2022 11:56:02 +0200 Subject: [PATCH 4/4] Update .gitlab-ci.yml --- .gitlab-ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3b2b0eb..3c3aba83 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,6 +32,8 @@ build: - compiled/lib/ - build/examples/ - build/tests/ + - build/src/scord/scord + - build/src/scord-ctl/scord-ctl # depending on your build setup it's most likely a good idea to cache outputs to reduce the build time cache: key: $CI_COMMIT_REF_SLUG @@ -48,11 +50,8 @@ rpc: - export ASAN_OPTIONS=detect_odr_violation=0 - export LSAN_OPTIONS=verbosity=1:log_threads=1:suppressions=${CI_PROJECT_DIR}/tests/LSanSuppress.supp - export LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:${CI_PROJECT_DIR}/compiled/lib - - compiled/bin/scord -f --force-console & - - build/examples/cxx/ADM_ping ofi+tcp://127.0.0.1:52000 - cd build/examples/ - ctest -j$(nproc) --output-on-failure --output-junit rpc-report.xml - - pkill -TERM scord artifacts: expire_in: 1 week paths: -- GitLab