From 8edc4612911f4d81a813f51de6caa2ba7edff9c1 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 26 Mar 2019 17:49:48 +0100 Subject: [PATCH 1/4] Add basic MPI transfer test with multiple nodes --- configure.ac | 75 +++++++-- m4/ax_mpi.m4 | 176 ++++++++++++++++++++ tests/Makefile.am | 12 +- tests/mpi/Makefile.am | 106 ++++++++++++ tests/mpi/mpi-remote-transfers.cpp | 256 +++++++++++++++++++++++++++++ tests/mpi/mpi-tests-main.cpp | 48 ++++++ tests/test-env.cpp | 4 +- tests/test-env.hpp | 2 +- 8 files changed, 663 insertions(+), 16 deletions(-) create mode 100644 m4/ax_mpi.m4 create mode 100644 tests/mpi/Makefile.am create mode 100644 tests/mpi/mpi-remote-transfers.cpp create mode 100644 tests/mpi/mpi-tests-main.cpp diff --git a/configure.ac b/configure.ac index 47ce66d..dd0f74a 100644 --- a/configure.ac +++ b/configure.ac @@ -81,25 +81,68 @@ fi ################################################################################ is_enabled_build_tests="no" +AC_MSG_CHECKING([whether tests should be built]) AC_ARG_ENABLE([tests], AS_HELP_STRING([--enable-tests], [Build norns self tests]), [ - if test x$enableval = xyes -o x$enableval = x; - then - is_enabled_build_tests="yes" - else if test x$enableval = xno -o x$enableval = x; - then - is_enabled_build_tests="no" - else - AC_MSG_ERROR([This option can only be given 'yes' or 'no' values]) - fi - fi + AS_IF( + [ test "x$enableval" = "xyes" -o \ + "x$enableval" = "x" ], + [ + is_enabled_build_tests="yes" + AC_MSG_RESULT([yes]) + ], + [ + AS_IF( + [ test "x$enableval" = "xno" ], + [ + is_enabled_build_tests="no" + AC_MSG_RESULT([no]) + ], + [ AC_MSG_ERROR([This option can only be given 'yes' or 'no' values]) ] + ) + ] + + ) ], - [ is_enabled_build_tests="no" ]) + [ AC_MSG_RESULT([no]) ]) AM_CONDITIONAL([BUILD_TESTS], test x$is_enabled_build_tests = xyes) +################################################################################ +### check whether mpi tests should be built +################################################################################ + +is_enabled_build_mpi_tests="no" +AC_MSG_CHECKING([whether MPI tests should be built]) +AC_ARG_ENABLE([mpi-tests], + AS_HELP_STRING([--enable-mpi-tests], [Build NORNS MPI tests]), + [ + AS_IF( + [ test "x$enableval" = "xyes" -o \ + "x$enableval" = "x" ], + [ + is_enabled_build_mpi_tests="yes" + AC_MSG_RESULT([yes]) + ], + [ + AS_IF( + [ test "x$enableval" = "xno" ], + [ + is_enabled_build_mpi_tests="no" + AC_MSG_RESULT([no]) + ], + [ AC_MSG_ERROR([This option can only be given 'yes' or 'no' values]) ] + ) + ] + ) + ], + [ AC_MSG_RESULT([no]) ]) + +AM_CONDITIONAL([BUILD_MPI_TESTS], test x$is_enabled_build_mpi_tests = xyes) + + ################################################################################ ### check for dependencies ################################################################################ @@ -122,6 +165,16 @@ AS_IF([test "x$is_enabled_build_tests" = "xyes"], AC_CONFIG_FILES(tests/Makefile) ], []) +# only check for MPI if we are building MPI tests +AS_IF( + [ test "x$is_enabled_build_mpi_tests" = "xyes" ], + [ AX_MPI( + [], + [AC_MSG_ERROR([MPI tests require MPI, but MPI was not found in this system])]) + AC_CONFIG_FILES(tests/mpi/Makefile) + ], []) + + # check for mercury PKG_CHECK_MODULES([MERCURY], [mercury >= 0.26]) diff --git a/m4/ax_mpi.m4 b/m4/ax_mpi.m4 new file mode 100644 index 0000000..f6db3d0 --- /dev/null +++ b/m4/ax_mpi.m4 @@ -0,0 +1,176 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_mpi.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_MPI([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +# +# DESCRIPTION +# +# This macro tries to find out how to compile programs that use MPI +# (Message Passing Interface), a standard API for parallel process +# communication (see http://www-unix.mcs.anl.gov/mpi/) +# +# On success, it sets the MPICC, MPICXX, MPIF77, or MPIFC output variable +# to the name of the MPI compiler, depending upon the current language. +# (This may just be $CC/$CXX/$F77/$FC, but is more often something like +# mpicc/mpiCC/mpif77/mpif90.) It also sets MPILIBS to any libraries that +# are needed for linking MPI (e.g. -lmpi or -lfmpi, if a special +# MPICC/MPICXX/MPIF77/MPIFC was not found). +# +# Note that this macro should be used only if you just have a few source +# files that need to be compiled using MPI. In particular, you should +# neither overwrite CC/CXX/F77/FC with the values of +# MPICC/MPICXX/MPIF77/MPIFC, nor assume that you can use the same flags +# etc. as the standard compilers. If you want to compile a whole program +# using the MPI compiler commands, use one of the macros +# AX_PROG_{CC,CXX,FC}_MPI. +# +# ACTION-IF-FOUND is a list of shell commands to run if an MPI library is +# found, and ACTION-IF-NOT-FOUND is a list of commands to run if it is not +# found. If ACTION-IF-FOUND is not specified, the default action will +# define HAVE_MPI. +# +# LICENSE +# +# Copyright (c) 2008 Steven G. Johnson +# Copyright (c) 2008 Julian C. Cummings +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 9 + +AU_ALIAS([ACX_MPI], [AX_MPI]) +AC_DEFUN([AX_MPI], [ +AC_PREREQ(2.50) dnl for AC_LANG_CASE + +AC_LANG_CASE([C], [ + AC_REQUIRE([AC_PROG_CC]) + AC_ARG_VAR(MPICC,[MPI C compiler command]) + AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, $CC) + ax_mpi_save_CC="$CC" + CC="$MPICC" + AC_SUBST(MPICC) +], +[C++], [ + AC_REQUIRE([AC_PROG_CXX]) + AC_ARG_VAR(MPICXX,[MPI C++ compiler command]) + AC_CHECK_PROGS(MPICXX, mpic++ mpicxx mpiCC hcp mpxlC_r mpxlC mpCC cmpic++, $CXX) + ax_mpi_save_CXX="$CXX" + CXX="$MPICXX" + AC_SUBST(MPICXX) +], +[Fortran 77], [ + AC_REQUIRE([AC_PROG_F77]) + AC_ARG_VAR(MPIF77,[MPI Fortran 77 compiler command]) + AC_CHECK_PROGS(MPIF77, mpif77 hf77 mpxlf_r mpxlf mpf77 cmpifc, $F77) + ax_mpi_save_F77="$F77" + F77="$MPIF77" + AC_SUBST(MPIF77) +], +[Fortran], [ + AC_REQUIRE([AC_PROG_FC]) + AC_ARG_VAR(MPIFC,[MPI Fortran compiler command]) + AC_CHECK_PROGS(MPIFC, mpif90 mpxlf95_r mpxlf90_r mpxlf95 mpxlf90 mpf90 cmpif90c, $FC) + ax_mpi_save_FC="$FC" + FC="$MPIFC" + AC_SUBST(MPIFC) +]) + +if test x = x"$MPILIBS"; then + AC_LANG_CASE([C], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [C++], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [Fortran 77], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])], + [Fortran], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])]) +fi +AC_LANG_CASE([Fortran 77], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpich, MPI_Init, [MPILIBS="-lfmpich"]) + fi +], +[Fortran], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpichf90, MPI_Init, [MPILIBS="-lmpichf90"]) + fi +]) +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"]) +fi +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpich, MPI_Init, [MPILIBS="-lmpich"]) +fi + +dnl We have to use AC_TRY_COMPILE and not AC_CHECK_HEADER because the +dnl latter uses $CPP, not $CC (which may be mpicc). +AC_LANG_CASE([C], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[C++], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran 77], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi]) + +AC_LANG_CASE([C], [CC="$ax_mpi_save_CC"], + [C++], [CXX="$ax_mpi_save_CXX"], + [Fortran 77], [F77="$ax_mpi_save_F77"], + [Fortran], [FC="$ax_mpi_save_FC"]) + +AC_SUBST(MPILIBS) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x = x"$MPILIBS"; then + $2 + : +else + ifelse([$1],,[AC_DEFINE(HAVE_MPI,1,[Define if you have the MPI library.])],[$1]) + : +fi +])dnl AX_MPI diff --git a/tests/Makefile.am b/tests/Makefile.am index bf3a90b..d78d95a 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -25,14 +25,19 @@ # . # ########################################################################## +include $(top_srcdir)/config/Make-inc.mk + AM_CPPFLAGS = -I m4 +# XXX if WITH_MPI +if BUILD_MPI_TESTS +SUBDIRS = mpi +endif + TESTS = api core check_PROGRAMS = $(TESTS) api_interactive -END = - COMMON_SOURCES = \ compare-files.cpp \ compare-files.hpp \ @@ -182,5 +187,8 @@ EXTRA_core_DEPENDENCIES = \ $(top_builddir)/lib/libnornsctl_debug.la \ $(END) +mpi_CXXFLAGS = \ + -Wall -Wextra + MOSTLYCLEANFILES = \ config-template.cpp diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am new file mode 100644 index 0000000..7c131bc --- /dev/null +++ b/tests/mpi/Makefile.am @@ -0,0 +1,106 @@ +########################################################################## +# Copyright (C) 2017-2018 Barcelona Supercomputing Center # +# Centro Nacional de Supercomputacion # +# All rights reserved. # +# # +# This file is part of the NORNS Data Scheduler, a service that allows # +# other programs to start, track and manage asynchronous transfers of # +# data resources transfers requests between different storage backends. # +# # +# See AUTHORS file in the top level directory for information # +# regarding developers and contributors. # +# # +# The NORNS Data Scheduler is free software: you can redistribute it # +# and/or modify it under the terms of the GNU Lesser General Public # +# License as published by the Free Software Foundation, either # +# version 3 of the License, or (at your option) any later version. # +# # +# The NORNS Data Scheduler is distributed in the hope that it will be # +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty # +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # +# Lesser General Public License for more details. # +# # +# You should have received a copy of the GNU Lesser General # +# Public License along with the NORNS Data Scheduler. If not, see # +# . # +########################################################################## + +include $(top_srcdir)/config/Make-inc.mk + +CXX = @MPICXX@ + +AM_CPPFLAGS = -I m4 + +TESTS = mpi_tests + +check_PROGRAMS = mpi_tests + +COMMON_SOURCES = \ + $(END) + +mpi_tests_SOURCES = \ + $(top_srcdir)/tests/catch.hpp \ + mpi-tests-main.cpp \ + mpi-remote-transfers.cpp \ + ../fake-daemon.cpp \ + ../fake-daemon.hpp \ + ../test-env.cpp \ + ../test-env.hpp \ + config-template.cpp \ + config-template.hpp \ + $(COMMON_SOURCES) \ + $(END) + +mpi_tests_CPPFLAGS = \ + @BOOST_CPPFLAGS@ \ + -I$(top_srcdir)/tests \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/rpc \ + -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ + -D__NORNS_DEBUG__ \ + $(END) + +mpi_tests_CXXFLAGS = \ + -Wall -Wextra \ + $(END) + +mpi_tests_LDFLAGS = \ + @MPILIBS@ \ + @BOOST_ASIO_LIB@ \ + @BOOST_LDFLAGS@ \ + @BOOST_FILESYSTEM_LIB@ \ + @BOOST_PROGRAM_OPTIONS_LIB@ \ + @BOOST_SYSTEM_LIB@ \ + @BOOST_THREAD_LIB@ \ + @BOOST_REGEX_LIB@ \ + @PROTOBUF_LIBS@ \ + -no-install \ + -Wl,-rpath,$(top_builddir)/lib/.libs \ + $(top_builddir)/src/liburd_aux.la \ + $(top_builddir)/lib/libnorns_debug.la \ + $(top_builddir)/lib/libnornsctl_debug.la \ + $(END) + +EXTRA_mpi_tests_DEPENDENCIES = \ + $(top_builddir)/src/liburd_aux.la \ + $(top_builddir)/lib/libnorns_debug.la \ + $(top_builddir)/lib/libnornsctl_debug.la \ + $(END) + +BUILT_SOURCES = \ + config-template.cpp + +edit = $(SED) \ + -e 's|\"|\\"|g' \ + -e 's|^| "|g' \ + -e 's|$$|\\n"|g' + +config-template.cpp: Makefile $(top_srcdir)/etc/norns.conf.in + @( echo "/* This file autogenerated by Makefile */"; \ + echo "#include \"config-template.hpp\""; \ + echo ""; \ + echo "const std::string config_file::cftemplate = "; \ + $(edit) $(top_srcdir)/etc/norns.conf.in ; \ + echo ";"; \ + ) > $@ diff --git a/tests/mpi/mpi-remote-transfers.cpp b/tests/mpi/mpi-remote-transfers.cpp new file mode 100644 index 0000000..5fd3d00 --- /dev/null +++ b/tests/mpi/mpi-remote-transfers.cpp @@ -0,0 +1,256 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include "norns.h" +#include "nornsctl.h" +#include "test-env.hpp" +#include "catch.hpp" + +/******************************************************************************/ +/* tests for push transfers (single files) */ +/******************************************************************************/ +SCENARIO("copy local POSIX file to remote POSIX file", + "[mpi::norns_submit_push_to_posix_file]") { + + GIVEN("two running urd instances (local and remote)") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + struct { + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + } test_context; + + test_env env(false); + + const char* remote_host = "127.0.0.1:42000"; + + // Get the number of processes + int world_size; + if(MPI_Comm_size(MPI_COMM_WORLD, &world_size) != MPI_SUCCESS) { + FAIL("Failed to determine number of processes"); + } + + // Get the rank of the process + int world_rank; + if(MPI_Comm_rank(MPI_COMM_WORLD, &world_rank) != MPI_SUCCESS) { + FAIL("Failed to determine own rank"); + } + + // code for the test client + auto client = [world_rank, &env, &test_context] { + std::string nsid("client"); + bfs::path mountdir; + + // create namespaces + std::tie(std::ignore, mountdir) = + env.create_namespace(nsid, "mnt/" + nsid, 16384); + + // create input data + env.add_to_namespace(nsid, test_context.src_file_at_root, 40000); + env.add_to_namespace(nsid, test_context.src_file_at_subdir, 80000); + env.add_to_namespace(nsid, test_context.src_subdir0); + env.add_to_namespace(nsid, test_context.src_subdir1); + env.add_to_namespace(nsid, test_context.src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{test_context.src_subdir0 / + ("file" + std::to_string(i))}; + env.add_to_namespace(nsid, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{test_context.src_subdir1 / + ("file" + std::to_string(i))}; + env.add_to_namespace(nsid, p, 4096+i*10); + } + + // create input data with special permissions + auto p = + env.add_to_namespace(nsid, test_context.src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, test_context.src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, test_context.src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid, test_context.src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, test_context.src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid, test_context.src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid, test_context.src_file_at_root, + test_context.src_symlink_at_root0); + env.add_to_namespace(nsid, test_context.src_subdir0, + test_context.src_symlink_at_root1); + env.add_to_namespace(nsid, test_context.src_subdir1, + test_context.src_symlink_at_root2); + + env.add_to_namespace(nsid, test_context.src_file_at_root, + test_context.src_symlink_at_subdir0); + env.add_to_namespace(nsid, test_context.src_subdir0, + test_context.src_symlink_at_subdir1); + env.add_to_namespace(nsid, test_context.src_subdir1, + test_context.src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(env.basedir(), mountdir / out_symlink, ec); + REQUIRE(!ec); + }; + + // code for the test servers + auto server = [world_rank, &env, &test_context] { + std::string nsid("server" + std::to_string(world_rank)); + bfs::path mountdir; + + // create namespaces + std::tie(std::ignore, mountdir) = + env.create_namespace(nsid, "mnt/" + nsid, 16384); + + // create required output directories + env.add_to_namespace(nsid, test_context.dst_subdir1); + }; + + + // register code + std::vector> functors; + functors.reserve(world_size); + + functors.push_back(client); + + for(int i = 1; i < world_size; ++i) { + functors.push_back(server); + } + + // run registered code for each process + functors.at(world_rank)(); + + // wait for everyone to finish preparing the test environment + MPI_Barrier(MPI_COMM_WORLD); + + if(world_rank != 0) { + // servers do nothing but wait for client to complete + MPI_Barrier(MPI_COMM_WORLD); + env.notify_success(); + return; + } + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + // cp -r /a/contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to DST namespace's root\n" + " cp -r /a/contents.* -> / = /contents.* ") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("client", + test_context.src_subdir0.c_str()), + NORNS_REMOTE_PATH("server1", + remote_host, + test_context.dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + +#if 0 + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace("client", src_subdir0); + bfs::path dst = + env.get_from_namespace("server1", dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } +#endif + } + } + } + } + + MPI_Barrier(MPI_COMM_WORLD); + env.notify_success(); + } +} diff --git a/tests/mpi/mpi-tests-main.cpp b/tests/mpi/mpi-tests-main.cpp new file mode 100644 index 0000000..ff7d6f6 --- /dev/null +++ b/tests/mpi/mpi-tests-main.cpp @@ -0,0 +1,48 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#define CATCH_CONFIG_RUNNER +#include +#include "catch.hpp" + +int +main(int argc, char* argv[]) { + + if(MPI_Init(&argc, &argv) != MPI_SUCCESS) { + std::cerr << "Failed to initialize MPI\n"; + return 1; + } + + int result = Catch::Session().run(argc, argv); + + if(MPI_Finalize() != MPI_SUCCESS) { + std::cerr << "Failed to finalize MPI\n"; + return 1; + } + + return result; +} diff --git a/tests/test-env.cpp b/tests/test-env.cpp index ec19662..c42067d 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -233,7 +233,7 @@ void test_env::notify_success() { } } -std::tuple +std::tuple test_env::create_namespace(const std::string& nsid, const bfs::path& mnt, size_t quota) { @@ -246,7 +246,7 @@ test_env::create_namespace(const std::string& nsid, const bfs::path& mnt, m_namespaces.emplace(nsid, abs_dir); - return std::make_tuple(nsid.c_str(), abs_dir); + return std::make_tuple(nsid, abs_dir); } bfs::path test_env::add_to_namespace(const std::string& nsid, const bfs::path& dirname) { diff --git a/tests/test-env.hpp b/tests/test-env.hpp index 76c4733..709a62f 100644 --- a/tests/test-env.hpp +++ b/tests/test-env.hpp @@ -50,7 +50,7 @@ struct test_env { bfs::path basedir() const; - std::tuple create_namespace(const std::string& nsid, const bfs::path& mnt, size_t quota); + std::tuple create_namespace(const std::string& nsid, const bfs::path& mnt, size_t quota); bfs::path add_to_namespace(const std::string& nsid, const bfs::path& dirname); bfs::path add_to_namespace(const std::string& nsid, const bfs::path& filename, std::size_t size); bfs::path add_to_namespace(const std::string& nsid, const bfs::path& src, const bfs::path& linkname); -- GitLab From fbd22abd8f1f265ac1efd2cb83b1324f2581b53d Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 28 Mar 2019 17:47:56 +0100 Subject: [PATCH 2/4] Sync test server with Catch2 rewind infrastructure --- tests/mpi/Makefile.am | 2 + tests/mpi/commands.hpp | 37 +++ tests/mpi/mpi-helpers.cpp | 111 +++++++++ tests/mpi/mpi-helpers.hpp | 59 +++++ tests/mpi/mpi-remote-transfers.cpp | 353 +++++++++++++++++------------ tests/mpi/mpi-tests-main.cpp | 48 +++- 6 files changed, 459 insertions(+), 151 deletions(-) create mode 100644 tests/mpi/commands.hpp create mode 100644 tests/mpi/mpi-helpers.cpp create mode 100644 tests/mpi/mpi-helpers.hpp diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am index 7c131bc..f86278c 100644 --- a/tests/mpi/Makefile.am +++ b/tests/mpi/Makefile.am @@ -42,6 +42,8 @@ mpi_tests_SOURCES = \ $(top_srcdir)/tests/catch.hpp \ mpi-tests-main.cpp \ mpi-remote-transfers.cpp \ + mpi-helpers.hpp \ + mpi-helpers.cpp \ ../fake-daemon.cpp \ ../fake-daemon.hpp \ ../test-env.cpp \ diff --git a/tests/mpi/commands.hpp b/tests/mpi/commands.hpp new file mode 100644 index 0000000..c8b4150 --- /dev/null +++ b/tests/mpi/commands.hpp @@ -0,0 +1,37 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef COMMANDS_HPP +#define COMMANDS_HPP + +enum class server_command : int { + accept = 0, + restart, + shutdown +}; + +#endif // COMMANDS_HPP diff --git a/tests/mpi/mpi-helpers.cpp b/tests/mpi/mpi-helpers.cpp new file mode 100644 index 0000000..f60148e --- /dev/null +++ b/tests/mpi/mpi-helpers.cpp @@ -0,0 +1,111 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include +#include "mpi-helpers.hpp" + +#ifdef MPI_TEST_DEBUG +#include +#include +#endif // MPI_TEST_DEBUG + + +namespace mpi { + +void +initialize(int* argc, char** argv[]) { + if(::MPI_Init(argc, argv) != MPI_SUCCESS) { + throw std::runtime_error("Failed to initialize MPI"); + } +} + +void +finalize() { + if(::MPI_Finalize() != MPI_SUCCESS) { + throw std::runtime_error("Failed to finalize MPI"); + } +} + + +int +get_rank() { + int world_rank; + if(::MPI_Comm_rank(MPI_COMM_WORLD, &world_rank) != MPI_SUCCESS) { + throw std::runtime_error("Failed to determine own rank"); + } + + return world_rank; +} + +void +barrier() { + +#ifdef MPI_TEST_DEBUG + std::cerr << __PRETTY_FUNCTION__ << "\n"; + std::cerr << "Entering MPI_Barrier()\n"; +#endif // MPI_TEST_DEBUG + + ::MPI_Barrier(MPI_COMM_WORLD); + +#ifdef MPI_TEST_DEBUG + std::cerr << "Exiting MPI_Barrier()\n"; +#endif // MPI_TEST_DEBUG + +} + +server_command +broadcast_command(server_command cmd) { + + int c = static_cast(cmd); + +#ifdef MPI_TEST_DEBUG + std::stringstream ss; + ss << __PRETTY_FUNCTION__ << "(" << c << ")" << "\n"; + std::cerr << ss.str(); + std::cerr << "Entering MPI_Bcast()\n"; +#endif // MPI_TEST_DEBUG + + ::MPI_Bcast(&c, 1, MPI_INT, 0, MPI_COMM_WORLD); + +#ifdef MPI_TEST_DEBUG + + MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) { + std::stringstream ss; + ss << "command was " << c << "\n"; + std::cerr << ss.str(); + } +#endif + +#ifdef MPI_TEST_DEBUG + std::cerr << "Exiting MPI_Bcast()\n"; +#endif // MPI_TEST_DEBUG + + return static_cast(c); +} + +} // namespace mpi diff --git a/tests/mpi/mpi-helpers.hpp b/tests/mpi/mpi-helpers.hpp new file mode 100644 index 0000000..a2a8b54 --- /dev/null +++ b/tests/mpi/mpi-helpers.hpp @@ -0,0 +1,59 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef MPI_HELPERS_HPP +#define MPI_HELPERS_HPP + +#include +#include "commands.hpp" + +// #define MPI_TEST_DEBUG + +namespace mpi { + +void +initialize(int* argc, char** argv[]); + +void +finalize(); + +int +get_rank(); + +server_command +broadcast_command(server_command cmd = server_command::accept); + +void +barrier(); + +} // namespace mpi + +#define MPI_RANK_EQ(r) (mpi::get_rank() == r) +#define MPI_RANK_NEQ(r) (mpi::get_rank() != r) +#define MPI_TEST_RUN_IF(expr) if(expr) + +#endif // MPI_HELPERS_HPP diff --git a/tests/mpi/mpi-remote-transfers.cpp b/tests/mpi/mpi-remote-transfers.cpp index 5fd3d00..e6ace59 100644 --- a/tests/mpi/mpi-remote-transfers.cpp +++ b/tests/mpi/mpi-remote-transfers.cpp @@ -25,76 +25,131 @@ * . * *************************************************************************/ -#include +#include "mpi-helpers.hpp" +#include "commands.hpp" #include "norns.h" #include "nornsctl.h" #include "test-env.hpp" #include "catch.hpp" +namespace test_data { + +struct { + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname +} context; + +} + + /******************************************************************************/ /* tests for push transfers (single files) */ /******************************************************************************/ SCENARIO("copy local POSIX file to remote POSIX file", "[mpi::norns_submit_push_to_posix_file]") { + using test_data::context; + GIVEN("two running urd instances (local and remote)") { /**********************************************************************/ /* setup common environment */ /**********************************************************************/ - struct { - // define input names - const bfs::path src_file_at_root = "/file0"; - const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; - const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; - const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; - const bfs::path src_subdir0 = "/input_dir0"; - const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; - const bfs::path src_empty_dir = "/empty_dir0"; - - const bfs::path src_noperms_file0 = "/noperms_file0"; - const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible - const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible - const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible - const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible - const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible - - const bfs::path src_symlink_at_root0 = "/symlink0"; - const bfs::path src_symlink_at_root1 = "/symlink1"; - const bfs::path src_symlink_at_root2 = "/symlink2"; - const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; - const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; - const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; - - const bfs::path dst_root = "/"; - const bfs::path dst_subdir0 = "/output_dir0"; - const bfs::path dst_subdir1 = "/output_dir1"; - const bfs::path dst_file_at_root0 = "/file0"; // same basename - const bfs::path dst_file_at_root1 = "/file1"; // different basename - const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname - const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename - const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename - const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname - } test_context; - - test_env env(false); + const char* remote_host = "127.0.0.1:42000"; - // Get the number of processes - int world_size; - if(MPI_Comm_size(MPI_COMM_WORLD, &world_size) != MPI_SUCCESS) { - FAIL("Failed to determine number of processes"); - } + const char* hostname; - // Get the rank of the process - int world_rank; - if(MPI_Comm_rank(MPI_COMM_WORLD, &world_rank) != MPI_SUCCESS) { - FAIL("Failed to determine own rank"); + if((hostname = ::getenv("MPICH_INTERFACE_HOSTNAME")) != NULL) { + std::cerr << "hostname" << hostname << "\n"; } + + // code for the test servers + MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) { + + bool shutdown = false; + + do { + test_env env(false); + + std::string nsid("server" + std::to_string(mpi::get_rank())); + bfs::path mountdir; + + // create namespaces + std::tie(std::ignore, mountdir) = + env.create_namespace(nsid, "mnt/" + nsid, 16384); + + // create required output directories + env.add_to_namespace(nsid, context.dst_subdir1); + + // sync with client after preparing the test environment + mpi::barrier(); + + bool restart = false; + + // servers do nothing but wait for client to complete + do { + int command = 0; + server_command cmd = mpi::broadcast_command(); + + switch(cmd) { + case server_command::shutdown: + shutdown = true; + break; + + case server_command::restart: + restart = true; + break; + + default: + continue; + } + } while(!restart && !shutdown); + + env.notify_success(); + + } while(!shutdown); + + return; + }; // MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) + // code for the test client - auto client = [world_rank, &env, &test_context] { + MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { + + test_env env(false); + std::string nsid("client"); bfs::path mountdir; @@ -103,154 +158,166 @@ SCENARIO("copy local POSIX file to remote POSIX file", env.create_namespace(nsid, "mnt/" + nsid, 16384); // create input data - env.add_to_namespace(nsid, test_context.src_file_at_root, 40000); - env.add_to_namespace(nsid, test_context.src_file_at_subdir, 80000); - env.add_to_namespace(nsid, test_context.src_subdir0); - env.add_to_namespace(nsid, test_context.src_subdir1); - env.add_to_namespace(nsid, test_context.src_empty_dir); + env.add_to_namespace(nsid, context.src_file_at_root, 40000); + env.add_to_namespace(nsid, context.src_file_at_subdir, 80000); + env.add_to_namespace(nsid, context.src_subdir0); + env.add_to_namespace(nsid, context.src_subdir1); + env.add_to_namespace(nsid, context.src_empty_dir); for(int i=0; i<10; ++i) { - const bfs::path p{test_context.src_subdir0 / + const bfs::path p{context.src_subdir0 / ("file" + std::to_string(i))}; env.add_to_namespace(nsid, p, 4096+i*10); } for(int i=0; i<10; ++i) { - const bfs::path p{test_context.src_subdir1 / + const bfs::path p{context.src_subdir1 / ("file" + std::to_string(i))}; env.add_to_namespace(nsid, p, 4096+i*10); } // create input data with special permissions auto p = - env.add_to_namespace(nsid, test_context.src_noperms_file0, 0); + env.add_to_namespace(nsid, context.src_noperms_file0, 0); env.remove_access(p); - p = env.add_to_namespace(nsid, test_context.src_noperms_file1, 0); + p = env.add_to_namespace(nsid, context.src_noperms_file1, 0); env.remove_access(p); - p = env.add_to_namespace(nsid, test_context.src_noperms_file2, 0); + p = env.add_to_namespace(nsid, context.src_noperms_file2, 0); env.remove_access(p.parent_path()); - p = env.add_to_namespace(nsid, test_context.src_noperms_subdir0); + p = env.add_to_namespace(nsid, context.src_noperms_subdir0); env.remove_access(p); - p = env.add_to_namespace(nsid, test_context.src_noperms_subdir1); + p = env.add_to_namespace(nsid, context.src_noperms_subdir1); env.remove_access(p); - p = env.add_to_namespace(nsid, test_context.src_noperms_subdir2); + p = env.add_to_namespace(nsid, context.src_noperms_subdir2); env.remove_access(p.parent_path()); // add symlinks to the namespace - env.add_to_namespace(nsid, test_context.src_file_at_root, - test_context.src_symlink_at_root0); - env.add_to_namespace(nsid, test_context.src_subdir0, - test_context.src_symlink_at_root1); - env.add_to_namespace(nsid, test_context.src_subdir1, - test_context.src_symlink_at_root2); - - env.add_to_namespace(nsid, test_context.src_file_at_root, - test_context.src_symlink_at_subdir0); - env.add_to_namespace(nsid, test_context.src_subdir0, - test_context.src_symlink_at_subdir1); - env.add_to_namespace(nsid, test_context.src_subdir1, - test_context.src_symlink_at_subdir2); + env.add_to_namespace(nsid, context.src_file_at_root, + context.src_symlink_at_root0); + env.add_to_namespace(nsid, context.src_subdir0, + context.src_symlink_at_root1); + env.add_to_namespace(nsid, context.src_subdir1, + context.src_symlink_at_root2); + + env.add_to_namespace(nsid, context.src_file_at_root, + context.src_symlink_at_subdir0); + env.add_to_namespace(nsid, context.src_subdir0, + context.src_symlink_at_subdir1); + env.add_to_namespace(nsid, context.src_subdir1, + context.src_symlink_at_subdir2); // manually create a symlink leading outside namespace 0 boost::system::error_code ec; const bfs::path out_symlink = "/out_symlink"; bfs::create_symlink(env.basedir(), mountdir / out_symlink, ec); REQUIRE(!ec); - }; - - // code for the test servers - auto server = [world_rank, &env, &test_context] { - std::string nsid("server" + std::to_string(world_rank)); - bfs::path mountdir; - // create namespaces - std::tie(std::ignore, mountdir) = - env.create_namespace(nsid, "mnt/" + nsid, 16384); - - // create required output directories - env.add_to_namespace(nsid, test_context.dst_subdir1); - }; - - - // register code - std::vector> functors; - functors.reserve(world_size); - - functors.push_back(client); - - for(int i = 1; i < world_size; ++i) { - functors.push_back(server); - } + /******************************************************************/ + /* begin tests */ + /******************************************************************/ + // cp -r /a/contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to DST namespace's root\n" + " cp -r /a/contents.* -> / = /contents.* ") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("client", + context.src_subdir0.c_str()), + NORNS_REMOTE_PATH("server1", + remote_host, + context.dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); - // run registered code for each process - functors.at(world_rank)(); + // wait until the task completes + rv = norns_wait(&task, NULL); - // wait for everyone to finish preparing the test environment - MPI_Barrier(MPI_COMM_WORLD); + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); - if(world_rank != 0) { - // servers do nothing but wait for client to complete - MPI_Barrier(MPI_COMM_WORLD); - env.notify_success(); - return; - } + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - /**********************************************************************/ - /* begin tests */ - /**********************************************************************/ - // cp -r /a/contents.* -> / = /contents.* - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " - "namespace's root to DST namespace's root\n" - " cp -r /a/contents.* -> / = /contents.* ") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - norns_iotask_t task = - NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH("client", - test_context.src_subdir0.c_str()), - NORNS_REMOTE_PATH("server1", - remote_host, - test_context.dst_root.c_str())); +#if 0 + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace("client", src_subdir0); + bfs::path dst = + env.get_from_namespace("server1", dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } +#endif + } + } + } + } - norns_error_t rv = norns_submit(&task); + WHEN("copying the contents of a NORNS_LOCAL_PATH arbitrary subdir " + "to DST namespace's root\n" + " cp -r /a/b/c/.../contents.* -> / = /contents.*") { - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); +///XXX wrong test case + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("client", + context.src_subdir0.c_str()), + NORNS_REMOTE_PATH("server1", + remote_host, + context.dst_root.c_str())); - // wait until the task completes - rv = norns_wait(&task, NULL); + norns_error_t rv = norns_submit(&task); - THEN("norns_wait() returns NORNS_SUCCESS") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); - THEN("norns_error() reports NORNS_EFINISHED") { - norns_stat_t stats; - rv = norns_error(&task, &stats); + // wait until the task completes + rv = norns_wait(&task, NULL); + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(stats.st_status == NORNS_EFINISHED); -#if 0 - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace("client", src_subdir0); - bfs::path dst = - env.get_from_namespace("server1", dst_root); + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - REQUIRE(compare_directories(src, dst) == true); - } + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + +#if 0 + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace("client", src_subdir0); + bfs::path dst = + env.get_from_namespace("server1", dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } #endif + } } } } - } - MPI_Barrier(MPI_COMM_WORLD); - env.notify_success(); + env.notify_success(); + + } // MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) } + + std::cout << "Check!\n"; } diff --git a/tests/mpi/mpi-tests-main.cpp b/tests/mpi/mpi-tests-main.cpp index ff7d6f6..2436c58 100644 --- a/tests/mpi/mpi-tests-main.cpp +++ b/tests/mpi/mpi-tests-main.cpp @@ -26,23 +26,55 @@ *************************************************************************/ #define CATCH_CONFIG_RUNNER -#include +#include "mpi-helpers.hpp" +#include "commands.hpp" #include "catch.hpp" +bool +is_top_level_section(const Catch::SectionInfo& info) { + const auto top_level_name("Given: "); + return info.name.rfind(top_level_name, info.name.length()) != + std::string::npos; +} + +struct TestListener : Catch::TestEventListenerBase { + + using TestEventListenerBase::TestEventListenerBase; // inherit constructor + + virtual void + sectionStarting(const Catch::SectionInfo& sectionInfo) override { + + (void) sectionInfo; + + MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { + mpi::broadcast_command(server_command::restart); + mpi::barrier(); // wait until servers finish starting up:w + } + } + + virtual void + sectionEnded(const Catch::SectionStats& sectionStats) override { + (void) sectionStats; + } + +}; + +CATCH_REGISTER_LISTENER(TestListener) + int main(int argc, char* argv[]) { - if(MPI_Init(&argc, &argv) != MPI_SUCCESS) { - std::cerr << "Failed to initialize MPI\n"; - return 1; - } + mpi::initialize(&argc, &argv); int result = Catch::Session().run(argc, argv); - if(MPI_Finalize() != MPI_SUCCESS) { - std::cerr << "Failed to finalize MPI\n"; - return 1; + MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { + std::cerr << "at main()\n"; + mpi::barrier(); + mpi::broadcast_command(server_command::shutdown); } + mpi::finalize(); + return result; } -- GitLab From 5992fd01de0b52de02a842cd7753f35032e22813 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 3 Apr 2019 15:50:38 +0200 Subject: [PATCH 3/4] Add mpi server for testing --- etc/norns.conf.in | 2 +- src/config/settings.cpp | 70 ++-- src/config/settings.hpp | 15 +- src/urd.cpp | 131 ++++--- src/urd.hpp | 4 + tests/fake-daemon.cpp | 8 +- tests/mpi/Makefile.am | 1 + tests/mpi/commands.hpp | 19 +- tests/mpi/mpi-helpers.cpp | 56 ++- tests/mpi/mpi-helpers.hpp | 15 +- tests/mpi/mpi-remote-transfers.cpp | 601 ++++++++++++++++++++--------- tests/mpi/mpi-server.cpp | 139 +++++++ tests/mpi/mpi-tests-main.cpp | 50 ++- tests/test-env.cpp | 54 ++- tests/test-env.hpp | 6 +- 15 files changed, 885 insertions(+), 286 deletions(-) create mode 100644 tests/mpi/mpi-server.cpp diff --git a/etc/norns.conf.in b/etc/norns.conf.in index 4fa1a43..6b0b531 100644 --- a/etc/norns.conf.in +++ b/etc/norns.conf.in @@ -19,7 +19,7 @@ global_settings: [ pidfile: "@localstatedir@/urd.pid", # address to bind to - bind_address: "127.0.0.1", + bind_address: "127.0.0.1:42000", # incoming port for remote connections remote_port: 42000, diff --git a/src/config/settings.cpp b/src/config/settings.cpp index f188118..a32b7af 100644 --- a/src/config/settings.cpp +++ b/src/config/settings.cpp @@ -58,7 +58,7 @@ settings::settings(const std::string& progname, uint32_t dry_run_duration, const bfs::path& global_socket, const bfs::path& control_socket, - const std::string& bind_address, + const std::string& configured_address, uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, @@ -76,7 +76,8 @@ settings::settings(const std::string& progname, m_dry_run_duration(dry_run_duration), m_global_socket(global_socket), m_control_socket(control_socket), - m_bind_address(bind_address), + m_configured_address(configured_address), + m_lookup_address(), m_remote_port(remote_port), m_daemon_pidfile(pidfile), m_workers_in_pool(workers), @@ -97,7 +98,8 @@ settings::load_defaults() { m_dry_run_duration = defaults::dry_run_duration; m_global_socket = defaults::global_socket; m_control_socket = defaults::control_socket; - m_bind_address = defaults::bind_address; + m_configured_address = defaults::bind_address; + m_lookup_address.clear(); m_remote_port = defaults::remote_port; m_daemon_pidfile = defaults::pidfile; m_workers_in_pool = defaults::workers_in_pool; @@ -134,7 +136,9 @@ settings::load_from_file(const bfs::path& filename) { m_dry_run_duration = defaults::dry_run_duration; m_global_socket = gsettings.get_as(keywords::global_socket); m_control_socket = gsettings.get_as(keywords::control_socket); - m_bind_address = gsettings.get_as(keywords::bind_address); + m_configured_address = + gsettings.get_as(keywords::bind_address); + m_lookup_address.clear(); m_remote_port = gsettings.get_as(keywords::remote_port); m_daemon_pidfile = gsettings.get_as(keywords::pidfile); m_workers_in_pool = gsettings.get_as(keywords::workers); @@ -159,24 +163,30 @@ settings::load_from_file(const bfs::path& filename) { std::string settings::to_string() const { - std::string str = std::string("settings {\n") + - " m_progname: " + m_progname + ",\n" + - " m_daemonize: " + (m_daemonize ? "true" : "false") + ",\n" + - " m_use_syslog: " + (m_use_syslog ? "true" : "false") + ",\n" + - " m_use_console: " + (m_use_console ? "true" : "false") + ",\n" + - " m_log_file: " + m_log_file.string() + ",\n" + - " m_log_file_max_size: " + std::to_string(m_log_file_max_size) + ",\n" + - " m_dry_run: " + (m_dry_run ? "true" : "false") + ",\n" + - " m_dry_run_duration: " + std::to_string(m_dry_run_duration) + ",\n" + - " m_global_socket: " + m_global_socket.string() + ",\n" + - " m_control_socket: " + m_control_socket.string() + ",\n" + - " m_bind_address: " + m_bind_address + ",\n" + - " m_remote_port: " + std::to_string(m_remote_port) + ",\n" + - " m_pidfile: " + m_daemon_pidfile.string() + ",\n" + - " m_workers: " + std::to_string(m_workers_in_pool) + ",\n" + + std::string str = + std::string("settings {\n") + + " m_progname: " + m_progname + ",\n" + + " m_daemonize: " + (m_daemonize ? "true" : "false") + ",\n" + + " m_use_syslog: " + (m_use_syslog ? "true" : "false") + ",\n" + + " m_use_console: " + (m_use_console ? "true" : "false") + ",\n" + + " m_log_file: " + m_log_file.string() + ",\n" + + " m_log_file_max_size: " + + std::to_string(m_log_file_max_size) + ",\n" + + " m_dry_run: " + (m_dry_run ? "true" : "false") + ",\n" + + " m_dry_run_duration: " + + std::to_string(m_dry_run_duration) + ",\n" + + " m_global_socket: " + m_global_socket.string() + ",\n" + + " m_control_socket: " + m_control_socket.string() + ",\n" + + " m_configured_address: " + m_configured_address + ",\n" + + " m_lookup_address: " + + (m_lookup_address.empty() ? + m_lookup_address : "{undetermined}") + ",\n" + + " m_remote_port: " + std::to_string(m_remote_port) + ",\n" + + " m_pidfile: " + m_daemon_pidfile.string() + ",\n" + + " m_workers: " + std::to_string(m_workers_in_pool) + ",\n" + " m_staging_directory: " + m_staging_directory.string() + ",\n" + - " m_backlog_size: " + std::to_string(m_backlog_size) + ",\n" + - " m_config_file: " + m_config_file.string() + ",\n" + + " m_backlog_size: " + std::to_string(m_backlog_size) + ",\n" + + " m_config_file: " + m_config_file.string() + ",\n" + "};"; //TODO: add m_default_namespaces return str; @@ -283,13 +293,23 @@ settings::control_socket(const bfs::path& control_socket) { } std::string -settings::bind_address() const { - return m_bind_address; +settings::configured_address() const { + return m_configured_address; } void -settings::bind_address(const std::string& bind_address) { - m_bind_address = bind_address; +settings::configured_address(const std::string& configured_address) { + m_configured_address = configured_address; +} + +std::string +settings::lookup_address() const { + return m_lookup_address; +} + +void +settings::lookup_address(const std::string& lookup_address) { + m_lookup_address = lookup_address; } in_port_t diff --git a/src/config/settings.hpp b/src/config/settings.hpp index d0c62b2..0842497 100644 --- a/src/config/settings.hpp +++ b/src/config/settings.hpp @@ -114,7 +114,7 @@ struct settings { uint32_t dry_run_duration, const bfs::path& global_socket, const bfs::path& control_socket, - const std::string& bind_address, + const std::string& configured_address, uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, @@ -205,10 +205,16 @@ struct settings { control_socket(const bfs::path& control_socket); std::string - bind_address() const; + configured_address() const; void - bind_address(const std::string& bind_address); + configured_address(const std::string& configured_address); + + std::string + lookup_address() const; + + void + lookup_address(const std::string& lookup_address); in_port_t remote_port() const; @@ -262,7 +268,8 @@ struct settings { uint32_t m_dry_run_duration; bfs::path m_global_socket; bfs::path m_control_socket; - std::string m_bind_address; + std::string m_configured_address; + std::string m_lookup_address; in_port_t m_remote_port; bfs::path m_daemon_pidfile; uint32_t m_workers_in_pool; diff --git a/src/urd.cpp b/src/urd.cpp index b407795..849308e 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -63,8 +63,67 @@ #include "hermes.hpp" #include "rpcs.hpp" #include "context.hpp" +#include "utils/file-handle.hpp" #include "urd.hpp" +namespace { + +void +write_pidfile(const bfs::path& pidfile) { + + /* Check if daemon already exists: + * First instance of the daemon will lock the file so that other + * instances understand that an instance is already running. + */ + norns::utils::file_handle fh( + ::open(pidfile.c_str(), O_RDWR | O_CREAT, 0640)); + + if(!fh) { + LOGGER_ERRNO("Failed to create daemon lock file"); + exit(EXIT_FAILURE); + } + + if(::lockf(fh.native(), F_TLOCK, 0) < 0) { + LOGGER_ERRNO("Failed to acquire lock on pidfile"); + LOGGER_ERROR("Another instance of this daemon may already be running"); + exit(EXIT_FAILURE); + } + + /* record pid in lockfile */ + std::string pid(std::to_string(::getpid())); + + ssize_t n = static_cast(pid.length()); + + if(::write(fh.native(), pid.c_str(), n) != n) { + LOGGER_ERRNO("Failed to write pidfile"); + exit(EXIT_FAILURE); + } +} + +bool +append_to_pidfile(const bfs::path& pidfile, + const std::string& entry) { + + const std::string tabbed_entry("\n" + entry + "\n"); + + norns::utils::file_handle fh( + ::open(pidfile.c_str(), O_WRONLY | O_APPEND, 0640)); + + if(!fh) { + return false; + } + + ssize_t n = static_cast(tabbed_entry.length()); + + if(::write(fh.native(), tabbed_entry.c_str(), n) != n) { + return false; + } + + return true; +} + +} + namespace norns { urd::urd() : @@ -164,29 +223,8 @@ pid_t urd::daemonize() { * First instance of the daemon will lock the file so that other * instances understand that an instance is already running. */ - int pfd; - - if((pfd = open(m_settings->pidfile().c_str(), O_RDWR | O_CREAT, 0640)) == -1) { - LOGGER_ERRNO("Failed to create daemon lock file"); - exit(EXIT_FAILURE); - } - - if(lockf(pfd, F_TLOCK, 0) < 0) { - LOGGER_ERRNO("Failed to acquire lock on pidfile"); - LOGGER_ERROR("Another instance of this daemon may already be running"); - exit(EXIT_FAILURE); - } + ::write_pidfile(m_settings->pidfile()); - /* record pid in lockfile */ - std::string pidstr(std::to_string(getpid())); - - if(write(pfd, pidstr.c_str(), pidstr.length()) != - static_cast(pidstr.length())) { - LOGGER_ERRNO("Failed to write pidfile"); - exit(EXIT_FAILURE); - } - - close(pfd); close(dev_null); /* Manage signals */ @@ -1130,15 +1168,10 @@ void urd::init_event_handlers() { ::umask(old_mask); try { - - const std::string bind_address = - m_settings->bind_address() + ":" + - std::to_string(m_settings->remote_port()); - m_network_service = std::make_shared( hermes::transport::ofi_tcp, - bind_address, + m_settings->configured_address(), true); } catch(const std::exception& e) { @@ -1147,20 +1180,6 @@ void urd::init_event_handlers() { exit(EXIT_FAILURE); } -#if 0 - // setup socket for remote connections - try { - m_ipc_service->register_endpoint(m_settings->remote_port()); - } - catch(const std::exception& e) { - LOGGER_ERROR("Failed to create socket for remote connections: {}", - e.what()); - teardown(); - exit(EXIT_FAILURE); - } -#endif - - LOGGER_INFO(" * Installing message handlers..."); /* user-level functionalities */ @@ -1466,7 +1485,8 @@ void urd::print_greeting() { LOGGER_INFO("{}", gsep); } -void urd::print_configuration() { +void +urd::print_configuration() { LOGGER_INFO(""); LOGGER_INFO("[[ Configuration ]]"); LOGGER_INFO(" - running as daemon?: {}", (m_settings->daemonize() ? "yes" : "no")); @@ -1485,11 +1505,25 @@ void urd::print_configuration() { LOGGER_INFO(" - control socket: {}", m_settings->control_socket()); LOGGER_INFO(" - global socket: {}", m_settings->global_socket()); LOGGER_INFO(" - staging directory: {}", m_settings->staging_directory()); - LOGGER_INFO(" - port for remote requests: {}", m_settings->remote_port()); + LOGGER_INFO(" - bind address: {}", m_settings->configured_address()); LOGGER_INFO(" - workers: {}", m_settings->workers_in_pool()); LOGGER_INFO(""); } +void +urd::store_runtime_configuration() { + + const std::string addr(m_network_service->self_address()); + + m_settings->lookup_address(addr); + + if(!::append_to_pidfile(m_settings->pidfile(), addr)) { + LOGGER_ERRNO("Failed to append entry to pidfile"); + exit(EXIT_FAILURE); + } + +} + void urd::print_farewell() { const char farewell[] = "Stopping {} daemon (pid {})"; const auto fsep = std::string(sizeof(farewell) - 4 + @@ -1570,6 +1604,11 @@ int urd::run() { teardown(); return EXIT_SUCCESS; } + else { + /* keep a pidfile even in foreground mode so that we can detect + * concurrent daemons with the same configuration */ + ::write_pidfile(m_settings->pidfile()); + } // print useful information print_greeting(); @@ -1587,12 +1626,16 @@ int urd::run() { // everything is set up load_transfer_plugins(); + // store runtime values of several settings in m_settings + store_runtime_configuration(); + // start the listener for remote transfers // N.B. This call returns immediately m_network_service->run(); LOGGER_INFO(""); LOGGER_INFO("[[ Start up successful, awaiting requests... ]]"); + LOGGER_INFO("[[ (public address: {}) ]]", m_settings->lookup_address()); // N.B. This call blocks here, which means that everything after it // will only run when a shutdown command is received diff --git a/src/urd.hpp b/src/urd.hpp index e45325c..3243829 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -103,6 +103,10 @@ private: void load_transfer_plugins(); void load_default_namespaces(); void check_configuration(); + + void + store_runtime_configuration(); + void print_greeting(); void print_configuration(); void print_farewell(); diff --git a/tests/fake-daemon.cpp b/tests/fake-daemon.cpp index ed83c7a..73ae912 100644 --- a/tests/fake-daemon.cpp +++ b/tests/fake-daemon.cpp @@ -30,15 +30,18 @@ #include #endif +#include #include #include "nornsctl.h" #include "fake-daemon.hpp" +namespace bfs = boost::filesystem; + const norns::config::settings fake_daemon::default_cfg( "test_urd", /* progname */ false, /* daemonize */ false, /* use syslog */ - false, /* use console */ + true, /* use console */ {},// "./test_urd.log", /* log file */ 0, /* unused */ false, /* dry run */ @@ -170,6 +173,9 @@ void fake_daemon::run() { } } + // disown from father + ::setsid(); + m_daemon.configure(m_config); m_daemon.run(); m_daemon.teardown(); diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am index f86278c..ab9cd98 100644 --- a/tests/mpi/Makefile.am +++ b/tests/mpi/Makefile.am @@ -44,6 +44,7 @@ mpi_tests_SOURCES = \ mpi-remote-transfers.cpp \ mpi-helpers.hpp \ mpi-helpers.cpp \ + commands.hpp \ ../fake-daemon.cpp \ ../fake-daemon.hpp \ ../test-env.cpp \ diff --git a/tests/mpi/commands.hpp b/tests/mpi/commands.hpp index c8b4150..25ddbea 100644 --- a/tests/mpi/commands.hpp +++ b/tests/mpi/commands.hpp @@ -28,10 +28,23 @@ #ifndef COMMANDS_HPP #define COMMANDS_HPP +#include + enum class server_command : int { - accept = 0, - restart, - shutdown + accepting_commands = 0, + start_section, + end_section, + shutdown, + MAX +}; + +constexpr const static +std::array(server_command::MAX)> +server_command_names = { + "ACCEPTING_COMMANDS", + "START_SECTION", + "END_SECTION", + "SHUTDOWN" }; #endif // COMMANDS_HPP diff --git a/tests/mpi/mpi-helpers.cpp b/tests/mpi/mpi-helpers.cpp index f60148e..2a59d71 100644 --- a/tests/mpi/mpi-helpers.cpp +++ b/tests/mpi/mpi-helpers.cpp @@ -26,6 +26,8 @@ *************************************************************************/ #include +#include +#include #include #include "mpi-helpers.hpp" @@ -34,9 +36,12 @@ #include #endif // MPI_TEST_DEBUG +namespace bfs = boost::filesystem; namespace mpi { +std::vector< std::pair > test_hosts; + void initialize(int* argc, char** argv[]) { if(::MPI_Init(argc, argv) != MPI_SUCCESS) { @@ -51,6 +56,17 @@ finalize() { } } +int +get_size() { + int world_size; + + if(::MPI_Comm_size(MPI_COMM_WORLD, &world_size) != MPI_SUCCESS) { + throw std::runtime_error("Failed to determine MPI_COMM_WORLD size"); + } + + return world_size; +} + int get_rank() { @@ -70,7 +86,9 @@ barrier() { std::cerr << "Entering MPI_Barrier()\n"; #endif // MPI_TEST_DEBUG - ::MPI_Barrier(MPI_COMM_WORLD); + if(::MPI_Barrier(MPI_COMM_WORLD) != MPI_SUCCESS) { + throw std::runtime_error("MPI_Barrier()"); + } #ifdef MPI_TEST_DEBUG std::cerr << "Exiting MPI_Barrier()\n"; @@ -85,18 +103,20 @@ broadcast_command(server_command cmd) { #ifdef MPI_TEST_DEBUG std::stringstream ss; - ss << __PRETTY_FUNCTION__ << "(" << c << ")" << "\n"; + ss << __PRETTY_FUNCTION__ << "(" << server_command_names[c] << ")" << "\n"; std::cerr << ss.str(); std::cerr << "Entering MPI_Bcast()\n"; #endif // MPI_TEST_DEBUG - ::MPI_Bcast(&c, 1, MPI_INT, 0, MPI_COMM_WORLD); + if(::MPI_Bcast(&c, 1, MPI_INT, 0, MPI_COMM_WORLD) != MPI_SUCCESS) { + throw std::runtime_error("MPI_Bcast()"); + } #ifdef MPI_TEST_DEBUG MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) { std::stringstream ss; - ss << "command was " << c << "\n"; + ss << "command was " << server_command_names[c] << "\n"; std::cerr << ss.str(); } #endif @@ -108,4 +128,32 @@ broadcast_command(server_command cmd) { return static_cast(c); } +void +read_hosts(const std::string& filename) { + + bfs::ifstream ifs(filename); + + std::stringstream ss; + + std::string line; + while(std::getline(ifs, line)) { + + // ignore comments + if(boost::regex_match(line, boost::regex("^\\s*?#\\s*?.*$"))) { + continue; + } + + boost::match_results captures; + + // parse host + if(boost::regex_match( + line, + captures, + boost::regex("^\\s*?(\\S*?)\\s*?:\\s*?(\\d+)\\s*$"))) { + test_hosts.emplace_back( + std::make_pair(captures[1], std::stoi(captures[2]))); + } + } +} + } // namespace mpi diff --git a/tests/mpi/mpi-helpers.hpp b/tests/mpi/mpi-helpers.hpp index a2a8b54..17f5f36 100644 --- a/tests/mpi/mpi-helpers.hpp +++ b/tests/mpi/mpi-helpers.hpp @@ -28,28 +28,39 @@ #ifndef MPI_HELPERS_HPP #define MPI_HELPERS_HPP +#include +#include #include #include "commands.hpp" -// #define MPI_TEST_DEBUG +#define MPI_TEST_DEBUG namespace mpi { +extern std::vector< std::pair > test_hosts; + void initialize(int* argc, char** argv[]); void finalize(); +int +get_size(); + int get_rank(); + server_command -broadcast_command(server_command cmd = server_command::accept); +broadcast_command(server_command cmd = server_command::accepting_commands); void barrier(); +void +read_hosts(const std::string& filename); + } // namespace mpi #define MPI_RANK_EQ(r) (mpi::get_rank() == r) diff --git a/tests/mpi/mpi-remote-transfers.cpp b/tests/mpi/mpi-remote-transfers.cpp index e6ace59..e8b69a4 100644 --- a/tests/mpi/mpi-remote-transfers.cpp +++ b/tests/mpi/mpi-remote-transfers.cpp @@ -25,6 +25,7 @@ * . * *************************************************************************/ +#include #include "mpi-helpers.hpp" #include "commands.hpp" #include "norns.h" @@ -32,6 +33,9 @@ #include "test-env.hpp" #include "catch.hpp" + +extern std::vector< std::pair > mpi::test_hosts; + namespace test_data { struct { @@ -63,261 +67,504 @@ struct { const bfs::path dst_subdir1 = "/output_dir1"; const bfs::path dst_file_at_root0 = "/file0"; // same basename const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_root2 = "/file2"; + const bfs::path dst_file_at_root3 = "/file3"; const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + const bfs::path dst_file_at_subdir4 = "/a/b/c/d/file4"; + const bfs::path dst_file_at_subdir5 = "/a/b/c/d/file5"; } context; } - /******************************************************************************/ /* tests for push transfers (single files) */ /******************************************************************************/ SCENARIO("copy local POSIX file to remote POSIX file", "[mpi::norns_submit_push_to_posix_file]") { - using test_data::context; + auto ctx = test_data::context; - GIVEN("two running urd instances (local and remote)") { + GIVEN("a local running urd instances and several remote ones)") { /**********************************************************************/ /* setup common environment */ /**********************************************************************/ + const std::string bind_address( + mpi::test_hosts.at(mpi::get_rank()).first + ":" + + std::to_string(mpi::test_hosts.at(mpi::get_rank()).second)); + + const std::string remote_host( + mpi::test_hosts.at(1).first + ":" + + std::to_string(mpi::test_hosts.at(1).second)); + + if(::setenv("TEST_FORCE_BIND_ADDRESS", bind_address.c_str(), 1)) { + throw std::runtime_error( + "Failed to set TEST_FORCE_BIND_ADDRESS env var"); + } + + test_env env; - const char* remote_host = "127.0.0.1:42000"; + std::string nsid("client"); + bfs::path mountdir; - const char* hostname; + // create namespaces + std::tie(std::ignore, mountdir) = + env.create_namespace(nsid, "mnt/" + nsid, 16384); - if((hostname = ::getenv("MPICH_INTERFACE_HOSTNAME")) != NULL) { - std::cerr << "hostname" << hostname << "\n"; + // create input data + env.add_to_namespace(nsid, ctx.src_file_at_root, 40000); + env.add_to_namespace(nsid, ctx.src_file_at_subdir, 80000); + env.add_to_namespace(nsid, ctx.src_subdir0); + env.add_to_namespace(nsid, ctx.src_subdir1); + env.add_to_namespace(nsid, ctx.src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{ctx.src_subdir0 / + ("subfile" + std::to_string(i))}; + env.add_to_namespace(nsid, p, 4096+i*10); } + for(int i=0; i<10; ++i) { + const bfs::path p{ctx.src_subdir1 / + ("subfile" + std::to_string(i))}; + env.add_to_namespace(nsid, p, 4096+i*10); + } - // code for the test servers - MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) { + // create input data with special permissions + auto p = + env.add_to_namespace(nsid, ctx.src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, ctx.src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, ctx.src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid, ctx.src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid, ctx.src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid, ctx.src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid, ctx.src_file_at_root, + ctx.src_symlink_at_root0); + env.add_to_namespace(nsid, ctx.src_subdir0, + ctx.src_symlink_at_root1); + env.add_to_namespace(nsid, ctx.src_subdir1, + ctx.src_symlink_at_root2); + + env.add_to_namespace(nsid, ctx.src_file_at_root, + ctx.src_symlink_at_subdir0); + env.add_to_namespace(nsid, ctx.src_subdir0, + ctx.src_symlink_at_subdir1); + env.add_to_namespace(nsid, ctx.src_subdir1, + ctx.src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(env.basedir(), mountdir / out_symlink, ec); + REQUIRE(!ec); + + const std::string nsid0("client"); + const std::string nsid1("server0"); + + /******************************************************************/ + /* begin tests */ + /******************************************************************/ + // cp -r ns0://file0.txt + // -> ns1:// = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_REMOTE_PATH at DST namespace's root " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); - bool shutdown = false; + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - do { - test_env env(false); + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } + } + } - std::string nsid("server" + std::to_string(mpi::get_rank())); - bfs::path mountdir; + // cp -r ns0://file0.txt + // -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at DST namespace's root " + "(changing the name)") { - // create namespaces - std::tie(std::ignore, mountdir) = - env.create_namespace(nsid, "mnt/" + nsid, 16384); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_root1.c_str())); - // create required output directories - env.add_to_namespace(nsid, context.dst_subdir1); + norns_error_t rv = norns_submit(&task); - // sync with client after preparing the test environment - mpi::barrier(); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); - bool restart = false; + // wait until the task completes + rv = norns_wait(&task, NULL); - // servers do nothing but wait for client to complete - do { - int command = 0; - server_command cmd = mpi::broadcast_command(); + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); - switch(cmd) { - case server_command::shutdown: - shutdown = true; - break; + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - case server_command::restart: - restart = true; - break; + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - default: - continue; } - } while(!restart && !shutdown); + } + } + } - env.notify_success(); + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file0.txt = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root") { - } while(!shutdown); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_root2.c_str())); - return; - }; // MPI_TEST_RUN_IF(MPI_RANK_NEQ(0)) + norns_error_t rv = norns_submit(&task); - // code for the test client - MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); - test_env env(false); + // wait until the task completes + rv = norns_wait(&task, NULL); - std::string nsid("client"); - bfs::path mountdir; + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - // create namespaces - std::tie(std::ignore, mountdir) = - env.create_namespace(nsid, "mnt/" + nsid, 16384); + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - // create input data - env.add_to_namespace(nsid, context.src_file_at_root, 40000); - env.add_to_namespace(nsid, context.src_file_at_subdir, 80000); - env.add_to_namespace(nsid, context.src_subdir0); - env.add_to_namespace(nsid, context.src_subdir1); - env.add_to_namespace(nsid, context.src_empty_dir); + } - for(int i=0; i<10; ++i) { - const bfs::path p{context.src_subdir0 / - ("file" + std::to_string(i))}; - env.add_to_namespace(nsid, p, 4096+i*10); + } + } + } + + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_root3.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } + } + } + + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } } + } + + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); - for(int i=0; i<10; ++i) { - const bfs::path p{context.src_subdir1 / - ("file" + std::to_string(i))}; - env.add_to_namespace(nsid, p, 4096+i*10); + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir4.c_str())); + + norns_error_t rv = norns_submit(&task); - // create input data with special permissions - auto p = - env.add_to_namespace(nsid, context.src_noperms_file0, 0); - env.remove_access(p); - - p = env.add_to_namespace(nsid, context.src_noperms_file1, 0); - env.remove_access(p); - - p = env.add_to_namespace(nsid, context.src_noperms_file2, 0); - env.remove_access(p.parent_path()); - - p = env.add_to_namespace(nsid, context.src_noperms_subdir0); - env.remove_access(p); - - p = env.add_to_namespace(nsid, context.src_noperms_subdir1); - env.remove_access(p); - - p = env.add_to_namespace(nsid, context.src_noperms_subdir2); - env.remove_access(p.parent_path()); - - // add symlinks to the namespace - env.add_to_namespace(nsid, context.src_file_at_root, - context.src_symlink_at_root0); - env.add_to_namespace(nsid, context.src_subdir0, - context.src_symlink_at_root1); - env.add_to_namespace(nsid, context.src_subdir1, - context.src_symlink_at_root2); - - env.add_to_namespace(nsid, context.src_file_at_root, - context.src_symlink_at_subdir0); - env.add_to_namespace(nsid, context.src_subdir0, - context.src_symlink_at_subdir1); - env.add_to_namespace(nsid, context.src_subdir1, - context.src_symlink_at_subdir2); - - // manually create a symlink leading outside namespace 0 - boost::system::error_code ec; - const bfs::path out_symlink = "/out_symlink"; - bfs::create_symlink(env.basedir(), mountdir / out_symlink, ec); - REQUIRE(!ec); - - /******************************************************************/ - /* begin tests */ - /******************************************************************/ - // cp -r /a/contents.* -> / = /contents.* - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " - "namespace's root to DST namespace's root\n" - " cp -r /a/contents.* -> / = /contents.* ") { - - norns_iotask_t task = - NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH("client", - context.src_subdir0.c_str()), - NORNS_REMOTE_PATH("server1", - remote_host, - context.dst_root.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - // wait until the task completes - rv = norns_wait(&task, NULL); + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } + } + } - THEN("norns_error() reports NORNS_EFINISHED") { - norns_stat_t stats; - rv = norns_error(&task, &stats); + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(stats.st_status == NORNS_EFINISHED); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir5.c_str())); -#if 0 - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace("client", src_subdir0); - bfs::path dst = - env.get_from_namespace("server1", dst_root); + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - REQUIRE(compare_directories(src, dst) == true); - } -#endif - } } } } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the parents names)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir2.c_str())); - WHEN("copying the contents of a NORNS_LOCAL_PATH arbitrary subdir " - "to DST namespace's root\n" - " cp -r /a/b/c/.../contents.* -> / = /contents.*") { + norns_error_t rv = norns_submit(&task); -///XXX wrong test case - norns_iotask_t task = - NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH("client", - context.src_subdir0.c_str()), - NORNS_REMOTE_PATH("server1", - remote_host, - context.dst_root.c_str())); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); - norns_error_t rv = norns_submit(&task); + // wait until the task completes + rv = norns_wait(&task, NULL); - THEN("norns_submit() returns NORNS_SUCCESS") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - // wait until the task completes - rv = norns_wait(&task, NULL); + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); - THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0.c_str(), + ctx.src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1.c_str(), + remote_host.c_str(), + ctx.dst_file_at_subdir3.c_str())); - THEN("norns_error() reports NORNS_EFINISHED") { - norns_stat_t stats; - rv = norns_error(&task, &stats); + norns_error_t rv = norns_submit(&task); - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(stats.st_status == NORNS_EFINISHED); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); -#if 0 - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace("client", src_subdir0); - bfs::path dst = - env.get_from_namespace("server1", dst_root); + // wait until the task completes + rv = norns_wait(&task, NULL); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_error() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_error(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - REQUIRE(compare_directories(src, dst) == true); - } -#endif - } } } } + } - env.notify_success(); - } // MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) - } - std::cout << "Check!\n"; + + + + + + env.notify_success(); + } } diff --git a/tests/mpi/mpi-server.cpp b/tests/mpi/mpi-server.cpp new file mode 100644 index 0000000..1f420eb --- /dev/null +++ b/tests/mpi/mpi-server.cpp @@ -0,0 +1,139 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include +#include "mpi-helpers.hpp" +#include "test-env.hpp" + +namespace test_data { + +struct { + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname +} context; + +} + +namespace server { + +std::atomic shutdown(false); + +void +install_signal_handlers() { + + for(auto signum : { SIGINT, SIGILL, SIGFPE, + SIGSEGV, SIGTERM, SIGABRT }) { + struct sigaction sa; + sa.sa_handler = [](int signum) -> void { + server::shutdown = true; + }; + + if(::sigaction(signum, &sa, NULL) != 0) { + throw std::runtime_error("Failed to set signal handler"); + } + } +} + +} + +int +main(int argc, char* argv[]) { + + using test_data::context; + + server::install_signal_handlers(); + + std::string hostfile(argv[1]); + + mpi::read_hosts(hostfile); + + mpi::initialize(&argc, &argv); + + const std::string bind_address( + mpi::test_hosts.at(mpi::get_rank() + 1).first + ":" + + std::to_string(mpi::test_hosts.at(mpi::get_rank() + 1).second)); + + if(::setenv("TEST_FORCE_BIND_ADDRESS", bind_address.c_str(), 1)) { + throw std::runtime_error( + "Failed to set TEST_FORCE_BIND_ADDRESS env var"); + } + + std::cout << "Server booting (" << bind_address << ")\n"; + + test_env env; + + std::string nsid("server" + std::to_string(mpi::get_rank())); + bfs::path mountdir; + + // create namespaces + std::tie(std::ignore, mountdir) = + env.create_namespace(nsid, "mnt/" + nsid, 16384); + + // create required output directories + env.add_to_namespace(nsid, context.dst_subdir1); + + std::cout << "Server ready\n"; + + while(!server::shutdown) { + ::usleep(1000); + } + + std::cout << "Shutting down\n"; + + env.notify_success(); +} diff --git a/tests/mpi/mpi-tests-main.cpp b/tests/mpi/mpi-tests-main.cpp index 2436c58..0332f76 100644 --- a/tests/mpi/mpi-tests-main.cpp +++ b/tests/mpi/mpi-tests-main.cpp @@ -26,16 +26,14 @@ *************************************************************************/ #define CATCH_CONFIG_RUNNER +#include +#include +#include #include "mpi-helpers.hpp" #include "commands.hpp" #include "catch.hpp" -bool -is_top_level_section(const Catch::SectionInfo& info) { - const auto top_level_name("Given: "); - return info.name.rfind(top_level_name, info.name.length()) != - std::string::npos; -} +namespace bfs = boost::filesystem; struct TestListener : Catch::TestEventListenerBase { @@ -43,20 +41,13 @@ struct TestListener : Catch::TestEventListenerBase { virtual void sectionStarting(const Catch::SectionInfo& sectionInfo) override { - (void) sectionInfo; - - MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { - mpi::broadcast_command(server_command::restart); - mpi::barrier(); // wait until servers finish starting up:w - } } virtual void sectionEnded(const Catch::SectionStats& sectionStats) override { (void) sectionStats; } - }; CATCH_REGISTER_LISTENER(TestListener) @@ -64,17 +55,36 @@ CATCH_REGISTER_LISTENER(TestListener) int main(int argc, char* argv[]) { - mpi::initialize(&argc, &argv); + Catch::Session session; + + std::string hostfile; + + auto cli = session.cli() + | Catch::clara::Opt(hostfile, "filename") + ["-H"] + ["--hosts-file"] + ("file containing the hosts running the test (one per line)") + .required(); - int result = Catch::Session().run(argc, argv); + session.cli(cli); - MPI_TEST_RUN_IF(MPI_RANK_EQ(0)) { - std::cerr << "at main()\n"; - mpi::barrier(); - mpi::broadcast_command(server_command::shutdown); + int rv; + if((rv = session.applyCommandLine(argc, argv)) != 0) { + return rv; } + if(hostfile.empty()) { + std::cerr << "Missing host file.\n"; + return 1; + } + + mpi::read_hosts(hostfile); + + mpi::initialize(&argc, &argv); + + rv = session.run(argc, argv); + mpi::finalize(); - return result; + return rv; } diff --git a/tests/test-env.cpp b/tests/test-env.cpp index c42067d..aba89b0 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -33,7 +33,12 @@ #include #include +#ifndef TEST_ENV_DISABLE_CATCH2 #include "catch.hpp" +#else +#define REQUIRE(...) +#endif + #include "test-env.hpp" #include "norns.h" #include "nornsctl.h" @@ -90,6 +95,17 @@ create_config_file(const bfs::path& basedir, outstr = boost::regex_replace(outstr, boost::regex("(staging_directory:)\\s*?\".*?\"(,?)$"), "\\1 \"" + stdir.string() + "\"\\2"); + + const char* var; + + if((var = ::getenv("TEST_FORCE_BIND_ADDRESS")) != NULL) { + + const std::string bind_address(var); + + outstr = boost::regex_replace(outstr, + boost::regex("(bind_address:)\\s*?\".*?\"(,?)$"), + "\\1 \"" + bind_address + "\"\\2"); + } for(const auto& r : reps) { outstr = boost::regex_replace(outstr, boost::regex(r.first), r.second); @@ -114,7 +130,7 @@ patch_libraries(const bfs::path& config_file) { #endif } -test_env::test_env(bool requires_remote_peer) : +test_env::test_env() : m_test_succeeded(false), m_uid(generate_testid()), m_base_dir(bfs::absolute(bfs::current_path() / m_uid)) { @@ -129,11 +145,21 @@ test_env::test_env(bool requires_remote_peer) : #ifndef USE_REAL_DAEMON const std::string alias(".local"); - const auto local_config = ::create_config_file(m_base_dir, alias); - ::patch_libraries(local_config); - m_ltd.configure(local_config, alias); + const auto config_file = ::create_config_file(m_base_dir, alias); + ::patch_libraries(config_file); + m_ltd.configure(config_file, alias); + + + const bfs::path pidfile = m_base_dir / ("config" + alias); + + m_ltd.run(); + + + + +#if 0 // if the test requires a remote peer, we need to spawn another daemon if(requires_remote_peer) { const std::string alias(".remote"); @@ -151,6 +177,7 @@ test_env::test_env(bool requires_remote_peer) : ::patch_libraries(local_config); } #endif +#endif } @@ -205,6 +232,25 @@ void test_env::cleanup() { } +std::string +test_env::daemon_lookup_address() const { + + bfs::ifstream pidfile(m_ltd.m_config.pidfile()); + + std::string line; + + const boost::regex expr("^.*?\\+.*?:\\/\\/.+$"); + + while(std::getline(pidfile, line)) { + if(boost::regex_match(line, expr)) { + return line; // only return the first address found + } + } + + return ""; +} + + test_env::~test_env() { if(m_namespaces.size() != 0) { diff --git a/tests/test-env.hpp b/tests/test-env.hpp index 709a62f..55afc5a 100644 --- a/tests/test-env.hpp +++ b/tests/test-env.hpp @@ -44,7 +44,7 @@ namespace bfs = boost::filesystem; struct test_env { - test_env(bool requires_remote_peer = false); + test_env(); test_env(const fake_daemon_cfg& cfg); ~test_env(); @@ -66,6 +66,10 @@ struct test_env { void notify_success(); void cleanup(); + std::string + daemon_lookup_address() const; + + bool m_test_succeeded; std::string m_uid; bfs::path m_base_dir; -- GitLab From d855287c0dbf7d948a0b3fb101d7534bce3fc858 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 3 Apr 2019 16:31:52 +0200 Subject: [PATCH 4/4] Restore missing entry in Makefile.am --- tests/mpi/Makefile.am | 54 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/mpi/Makefile.am b/tests/mpi/Makefile.am index ab9cd98..094e0d1 100644 --- a/tests/mpi/Makefile.am +++ b/tests/mpi/Makefile.am @@ -33,7 +33,7 @@ AM_CPPFLAGS = -I m4 TESTS = mpi_tests -check_PROGRAMS = mpi_tests +check_PROGRAMS = mpi_tests mpi_server COMMON_SOURCES = \ $(END) @@ -107,3 +107,55 @@ config-template.cpp: Makefile $(top_srcdir)/etc/norns.conf.in $(edit) $(top_srcdir)/etc/norns.conf.in ; \ echo ";"; \ ) > $@ + +mpi_server_SOURCES = \ + mpi-server.cpp \ + mpi-helpers.hpp \ + mpi-helpers.cpp \ + commands.hpp \ + ../fake-daemon.cpp \ + ../fake-daemon.hpp \ + ../test-env.cpp \ + ../test-env.hpp \ + config-template.cpp \ + config-template.hpp \ + $(COMMON_SOURCES) \ + $(END) + +mpi_server_CPPFLAGS = \ + @BOOST_CPPFLAGS@ \ + -I$(top_srcdir)/tests \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/rpc \ + -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ + -D__NORNS_DEBUG__ \ + -DTEST_ENV_DISABLE_CATCH2 \ + $(END) + +mpi_server_CXXFLAGS = \ + -Wall -Wextra \ + $(END) + +mpi_server_LDFLAGS = \ + @MPILIBS@ \ + @BOOST_ASIO_LIB@ \ + @BOOST_LDFLAGS@ \ + @BOOST_FILESYSTEM_LIB@ \ + @BOOST_PROGRAM_OPTIONS_LIB@ \ + @BOOST_SYSTEM_LIB@ \ + @BOOST_THREAD_LIB@ \ + @BOOST_REGEX_LIB@ \ + @PROTOBUF_LIBS@ \ + -no-install \ + -Wl,-rpath,$(top_builddir)/lib/.libs \ + $(top_builddir)/src/liburd_aux.la \ + $(top_builddir)/lib/libnorns_debug.la \ + $(top_builddir)/lib/libnornsctl_debug.la \ + $(END) + +EXTRA_mpi_server_DEPENDENCIES = \ + $(top_builddir)/src/liburd_aux.la \ + $(top_builddir)/lib/libnorns_debug.la \ + $(top_builddir)/lib/libnornsctl_debug.la \ + $(END) -- GitLab