From 86e4783b70d0d14354d93396529f1f11e20e75b7 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 11 Oct 2018 17:29:13 +0200 Subject: [PATCH 01/51] Add basic test for remote copies The test infrastructure now supports creating a secondary "remote" daemon that binds to a different port in the localhost interface. With this, we can simulate remote transfers between different urd instances. --- tests/Makefile.am | 1 + tests/api-copy-remote-data.cpp | 1524 ++++++++++++++++++++++++++++++++ tests/fake-daemon.cpp | 31 +- tests/fake-daemon.hpp | 2 +- tests/test-env.cpp | 83 +- tests/test-env.hpp | 5 +- 6 files changed, 1607 insertions(+), 39 deletions(-) create mode 100644 tests/api-copy-remote-data.cpp diff --git a/tests/Makefile.am b/tests/Makefile.am index 2ff26e6..0f07a00 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -50,6 +50,7 @@ api_SOURCES = \ api-namespace-register.cpp \ api-namespace-unregister.cpp \ api-copy-local-data.cpp \ + api-copy-remote-data.cpp \ api-remove-local-data.cpp \ api-job-register.cpp \ api-job-update.cpp \ diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp new file mode 100644 index 0000000..ff54033 --- /dev/null +++ b/tests/api-copy-remote-data.cpp @@ -0,0 +1,1524 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "norns.h" +#include "nornsctl.h" +#include "test-env.hpp" +#include "compare-files.hpp" +#include "catch.hpp" + +namespace bfs = boost::filesystem; + +SCENARIO("copy local POSIX file to remote POSIX file", "[api::norns_submit_remote_copy_local_posix_files]") { + GIVEN("a running urd instance") { + + test_env env(true); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "localhost"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 4096); + env.add_to_namespace(nsid0, src_file_at_subdir, 8192); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + +#if 0 + /**************************************************************************************************************/ + /* tests for error conditions */ + /**************************************************************************************************************/ + // - trying to copy a non-existing file + WHEN("copying a non-existing NORNS_LOCAL_PATH file") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_invalid_file.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + + // - trying to copy a non-existing directory + WHEN("copying a non-existing NORNS_LOCAL_PATH directory") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_invalid_dir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + + // - trying to copy an empty directory + WHEN("copying an empty NORNS_LOCAL_PATH directory") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_empty_dir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_SUCCESS and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + REQUIRE(stats.st_task_error == NORNS_SUCCESS); + REQUIRE(stats.st_sys_errno == 0); + + REQUIRE(bfs::exists(dst_mnt / dst_file_at_root0)); + } + } + } + } + +//FIXME: DISABLED in CI until impersonation is implemented or capabilities can be added to the docker service +#ifdef __SETCAP_TESTS__ + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from \"/\" without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access a parent") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from \"/\" without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access a parent") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } +#endif + + // symlink leading out of namespace + WHEN("copying a NORNS_LOCAL_PATH through a symbolic link that leads " + "out of the src namespace") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, out_symlink.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } +#endif + + + /**********************************************************************/ + /* tests for single files */ + /**********************************************************************/ + // cp -r ns0://file0.txt -> ns1:// = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " + "another NORNS_REMOTE_PATH at dst namespace's root " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + +// THEN("NORNS_SUCCESS is returned") { +// REQUIRE(rv == NORNS_SUCCESS); +// +// THEN("Files are equal") { +// +// bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); +// bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); +// +// REQUIRE(compare_files(src, dst) == true); +// } +// } + } + } + +#if 0 + // cp -r ns0://file0.txt -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " + "another NORNS_LOCAL_PATH at dst namespace's root (changing the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../d/file0.txt -> ns1://file0.txt = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at dst namespace's root (keeping the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../d/file0.txt -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at dst namespace's root (changing the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://file0.txt -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (keeping the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://file0.txt -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (keeping the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the parents names)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir2.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir2); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " + "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir3.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir3); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + /**************************************************************************************************************/ + /* tests for directories */ + /**************************************************************************************************************/ + // cp -r /a/contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " + "dst namespace's root") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = env.get_from_namespace(nsid1, dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_LOCAL_PATH arbitrary subdir to " + "dst namespace's root") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = env.get_from_namespace(nsid1, dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + // cp -r /a/contents.* -> /c = /c/contents.* + // (c did not exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " + "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = env.get_from_namespace(nsid1, dst_subdir0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + // cp -r /a/contents.* -> /c = /c/contents.* + // (c did exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " + "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = env.get_from_namespace(nsid1, dst_subdir1); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* + // (c did not exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " + "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = env.get_from_namespace(nsid1, dst_subdir0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* + // (c did exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " + "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Copied files are identical to original") { + bfs::path src = env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = env.get_from_namespace(nsid1, dst_subdir1); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + /**************************************************************************************************************/ + /* tests for soft links */ + /**************************************************************************************************************/ + WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " + "through a symlink also located at '/'" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_root0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root0); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH subdir from src namespace's '/' " + "through a symlink also located at '/'" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_root1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directories are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root1); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink also located at '/'" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_root2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directories are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root2); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " + "through a symlink located in a subdir" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir0); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH subdir from src namespace's '/' " + "through a symlink also located at subdir" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directories are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir1); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink also located at a subdir" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directories are equal") { + + bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir2); + bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } +#endif + + env.notify_success(); + } + +#ifndef USE_REAL_DAEMON + GIVEN("a non-running urd instance") { + WHEN("attempting to request a transfer") { + + norns_iotask_t task = NORNS_IOTASK( + NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), + NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_ECONNFAILED is returned") { + REQUIRE(rv == NORNS_ECONNFAILED); + } + } + } +#endif +} + + +#if 0 +SCENARIO("copy local memory buffer to local POSIX file", "[api::norns_submit_copy_buffer_to_file]") { + GIVEN("a running urd instance") { + + test_env env; + + const char* nsid0 = "tmp0"; + bfs::path dst_mnt; + + // create namespaces + std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + // output names + const bfs::path dst_file_at_root0 = "/file0"; + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0/"; // existing + const bfs::path dst_subdir1 = "/output_dir0"; // existing but does not look as a directory + const bfs::path dst_subdir2 = "/output_dir0/a/b/c/d/"; // existing + const bfs::path dst_subdir3 = "/output_dir0/a/b/c/d"; // existing but does not look as a directory + const bfs::path dst_subdir4 = "/output_dir1/"; // non-existing + const bfs::path dst_subdir5 = "/output_dir1/a/b/c/d/"; // non-existing + + // create required output directories + env.add_to_namespace(nsid0, dst_subdir0); + env.add_to_namespace(nsid0, dst_subdir2); + + /**************************************************************************************************************/ + /* tests for error conditions */ + /**************************************************************************************************************/ + //TODO + // - copy a valid but removed memory region (cannot control => undefined behavior) + // - providing a non-existing directory path (i.e. finished with /) as output name + // - providing an existing path that points to a directory as output name + WHEN("copying an invalid memory region to a local POSIX file") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION((void*) 0x42, region_size), + NORNS_LOCAL_PATH(nsid0, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and EFAULT") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EFAULT); + } + } + } + } + + WHEN("copying a valid memory region to a local POSIX file located at '/'") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Output file contains buffer data") { + + bfs::path dst = env.get_from_namespace(nsid0, dst_file_at_root0); + + REQUIRE(compare(input_data, dst) == true); + } + } + } + } + + WHEN("copying a valid memory region to a local POSIX file located at a subdir") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Output file contains buffer data") { + + bfs::path dst = env.get_from_namespace(nsid0, dst_file_at_subdir0); + + REQUIRE(compare(input_data, dst) == true); + } + } + } + } + + WHEN("copying a valid memory region to a local POSIX /") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a local POSIX existing directory at /") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a local POSIX existing directory at /") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and EISDIR") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EISDIR); + } + } + } + } + + WHEN("copying a valid memory region to a local POSIX existing directory at an arbitary subdir") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir2.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a local POSIX existing directory at an arbitary subdir") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir3.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and EISDIR") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EISDIR); + } + } + } + } + + + // i.e. a destination path that 'looks like' a directory + WHEN("copying a valid memory region to a local POSIX non-existing directory") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir4.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a local POSIX non-existing directory") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_LOCAL_PATH(nsid0, dst_subdir5.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + env.notify_success(); + } +} +#endif diff --git a/tests/fake-daemon.cpp b/tests/fake-daemon.cpp index def2477..573ada2 100644 --- a/tests/fake-daemon.cpp +++ b/tests/fake-daemon.cpp @@ -34,7 +34,7 @@ #include "nornsctl.h" #include "fake-daemon.hpp" -norns::config::settings test_cfg( +norns::config::settings default_cfg( "test_urd", /* progname */ false, /* daemonize */ false, /* use syslog */ @@ -71,25 +71,30 @@ void fake_daemon::configure(const bfs::path& config_file, m_config.load_from_file(config_file); - m_config.progname() = test_cfg.progname(); - m_config.daemonize() = test_cfg.daemonize(); - m_config.use_syslog() = test_cfg.use_syslog(); + m_config.progname() = default_cfg.progname(); + m_config.daemonize() = default_cfg.daemonize(); + m_config.use_syslog() = default_cfg.use_syslog(); + m_config.use_console() = default_cfg.use_console(); m_config.dry_run() = override_cfg.m_dry_run; m_config.dry_run_duration() = override_cfg.m_dry_run_duration; - m_config.remote_port() = test_cfg.remote_port(); - m_config.workers_in_pool() = test_cfg.workers_in_pool(); + m_config.remote_port() = default_cfg.remote_port(); + m_config.workers_in_pool() = default_cfg.workers_in_pool(); m_config.config_file() = config_file; m_config.default_namespaces().clear(); } -void fake_daemon::configure(const bfs::path& config_file) { +void fake_daemon::configure(const bfs::path& config_file, + const std::string& alias) { + m_config.load_from_file(config_file); - m_config.progname() = test_cfg.progname(); - m_config.daemonize() = test_cfg.daemonize(); - m_config.use_syslog() = test_cfg.use_syslog(); - m_config.dry_run() = test_cfg.dry_run(); - m_config.remote_port() = test_cfg.remote_port(); - m_config.workers_in_pool() = test_cfg.workers_in_pool(); + + m_config.progname() = default_cfg.progname() + alias; + m_config.daemonize() = default_cfg.daemonize(); + m_config.use_syslog() = default_cfg.use_syslog(); + m_config.use_console() = default_cfg.use_console(); + m_config.dry_run() = default_cfg.dry_run(); + m_config.dry_run_duration() = default_cfg.dry_run_duration(); + m_config.workers_in_pool() = default_cfg.workers_in_pool(); m_config.config_file() = config_file; m_config.default_namespaces().clear(); } diff --git a/tests/fake-daemon.hpp b/tests/fake-daemon.hpp index 7929372..e6cb5ea 100644 --- a/tests/fake-daemon.hpp +++ b/tests/fake-daemon.hpp @@ -49,7 +49,7 @@ struct fake_daemon { fake_daemon(); ~fake_daemon(); void configure(const bfs::path& config_file, const fake_daemon_cfg& override_cfg); - void configure(const bfs::path& config_file); + void configure(const bfs::path& config_file, const std::string& alias = ""); void run(); int stop(); diff --git a/tests/test-env.cpp b/tests/test-env.cpp index db5fa3f..c711243 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -40,7 +40,6 @@ #include "config-template.hpp" //#define __DEBUG_OUTPUT__ - namespace bfs = boost::filesystem; namespace { @@ -50,40 +49,58 @@ std::string generate_testid() { return std::string("test_") + seed.substr(0, 8); } + + #ifndef USE_REAL_DAEMON -bfs::path create_config_file(const bfs::path& basedir) { - const bfs::path cfgdir = basedir / "config"; - bool res = bfs::create_directory(cfgdir); - REQUIRE(res == true); +using replacement_list = + std::vector< + std::pair>; + +bfs::path +create_config_file(const bfs::path& basedir, + const std::string& alias, + const replacement_list& reps = replacement_list()) { + + const bfs::path cfgdir = basedir / ("config" + alias); + + if(!bfs::exists(cfgdir)) { + bool res = bfs::create_directory(cfgdir); + REQUIRE(res == true); + } + + const std::string name = "test.conf" + alias; + + const bfs::path config_file = cfgdir / name; + + auto outstr = boost::regex_replace(config_file::cftemplate, + boost::regex("@localstatedir@"), + cfgdir.string()); + + for(const auto& r : reps) { + outstr = boost::regex_replace(outstr, boost::regex(r.first), r.second); + } - const bfs::path config_file = cfgdir / "test.conf"; bfs::ofstream outf(config_file); - outf << boost::regex_replace(config_file::cftemplate, - boost::regex("@localstatedir@"), - cfgdir.string()); + outf << outstr; outf.close(); return config_file; } -bfs::path patch_libraries(const bfs::path& basedir) { - - const auto config_file = create_config_file(basedir); +void +patch_libraries(const bfs::path& config_file) { int rv = ::setenv("NORNS_CONFIG_FILE", config_file.c_str(), 1); REQUIRE(rv != -1); libnorns_reload_config_file(); libnornsctl_reload_config_file(); - - return config_file; } #endif - } -test_env::test_env() : +test_env::test_env(bool requires_remote_peer) : m_test_succeeded(false), m_uid(generate_testid()), m_base_dir(bfs::absolute(bfs::current_path() / m_uid)) { @@ -97,9 +114,28 @@ test_env::test_env() : } #ifndef USE_REAL_DAEMON - const auto config_file = patch_libraries(m_base_dir); - m_td.configure(config_file); - m_td.run(); + const std::string alias(".local"); + const auto local_config = ::create_config_file(m_base_dir, alias); + ::patch_libraries(local_config); + m_ltd.configure(local_config, alias); + m_ltd.run(); + + // if the test requires a remote peer, we need to spawn another daemon + if(requires_remote_peer) { + const std::string alias(".remote"); + const auto remote_config = + ::create_config_file(m_base_dir, alias, + { {"(remote_port:)\\s*?42000\\s*?,$", "\\1 50000,"} }); + + // temporarily patch the libraries so that we can + // ping the "remote" daemon + ::patch_libraries(remote_config); + m_rtd.configure(remote_config, alias); + m_rtd.run(); + + // restore the libraries for the client + ::patch_libraries(local_config); + } #endif } @@ -118,9 +154,10 @@ test_env::test_env(const fake_daemon_cfg& cfg) : } #ifndef USE_REAL_DAEMON - const auto config_file = patch_libraries(m_base_dir); - m_td.configure(config_file, cfg); - m_td.run(); + const auto config_file = ::create_config_file(m_base_dir, ""); + ::patch_libraries(config_file); + m_ltd.configure(config_file, cfg); + m_ltd.run(); #else (void) cfg; #endif @@ -166,7 +203,7 @@ test_env::~test_env() { } #ifndef USE_REAL_DAEMON - int ret = m_td.stop(); + int ret = m_ltd.stop(); //REQUIRE(ret == 0); #endif diff --git a/tests/test-env.hpp b/tests/test-env.hpp index b8360f0..76c4733 100644 --- a/tests/test-env.hpp +++ b/tests/test-env.hpp @@ -44,7 +44,7 @@ namespace bfs = boost::filesystem; struct test_env { - test_env(); + test_env(bool requires_remote_peer = false); test_env(const fake_daemon_cfg& cfg); ~test_env(); @@ -71,7 +71,8 @@ struct test_env { bfs::path m_base_dir; #ifndef USE_REAL_DAEMON - fake_daemon m_td; + fake_daemon m_ltd; + fake_daemon m_rtd; #endif std::set m_dirs; -- GitLab From e0ee97ae2f7d544851aa3092f61afe7771670e30 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 16 Oct 2018 17:23:03 +0200 Subject: [PATCH 02/51] Rename m_namespace_name to m_name_in_namespace --- src/io/transferors/local-path-to-remote-path.cpp | 8 ++++++++ .../local_posix_path/detail/local-path-impl.cpp | 4 ++-- .../local_posix_path/detail/local-path-impl.hpp | 2 +- .../remote_posix_path/detail/remote-path-impl.cpp | 6 ++++-- .../remote_posix_path/detail/remote-path-impl.hpp | 2 ++ .../shared_posix_path/detail/shared-path-impl.cpp | 4 ++-- .../shared_posix_path/detail/shared-path-impl.hpp | 2 +- tests/api-copy-remote-data.cpp | 9 ++++++--- 8 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/io/transferors/local-path-to-remote-path.cpp b/src/io/transferors/local-path-to-remote-path.cpp index 1f79d4c..9a05aeb 100644 --- a/src/io/transferors/local-path-to-remote-path.cpp +++ b/src/io/transferors/local-path-to-remote-path.cpp @@ -66,6 +66,14 @@ local_path_to_remote_path_transferor::transfer( (void) src; (void) dst; + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] transfer: {} -> {}", task_info->id(), + d_src.canonical_path(), d_dst.to_string()); + LOGGER_WARN("Transfer not implemented"); return std::make_error_code(static_cast(0)); diff --git a/src/resources/local_posix_path/detail/local-path-impl.cpp b/src/resources/local_posix_path/detail/local-path-impl.cpp index 35de7e4..eadf4b3 100644 --- a/src/resources/local_posix_path/detail/local-path-impl.cpp +++ b/src/resources/local_posix_path/detail/local-path-impl.cpp @@ -47,14 +47,14 @@ using local_path_resource = resource_impl; local_path_resource::resource_impl(const std::shared_ptr parent, const bfs::path& name) : - m_namespace_name(name), + m_name_in_namespace(name), m_canonical_path(parent->mount() / name), m_is_collection(bfs::is_directory(m_canonical_path)), m_parent(std::static_pointer_cast(std::move(parent))) { } std::string local_path_resource::name() const { - return m_namespace_name.string(); + return m_name_in_namespace.string(); } resource_type diff --git a/src/resources/local_posix_path/detail/local-path-impl.hpp b/src/resources/local_posix_path/detail/local-path-impl.hpp index ce7588b..df2e9b4 100644 --- a/src/resources/local_posix_path/detail/local-path-impl.hpp +++ b/src/resources/local_posix_path/detail/local-path-impl.hpp @@ -63,7 +63,7 @@ struct resource_impl : public resource { bfs::path canonical_path() const; - const bfs::path m_namespace_name; // absolute pathname w.r.t. backend's mount point + const bfs::path m_name_in_namespace; // absolute pathname w.r.t. backend's mount point const bfs::path m_canonical_path; // canonical pathname const bool m_is_collection; const std::shared_ptr m_parent; diff --git a/src/resources/remote_posix_path/detail/remote-path-impl.cpp b/src/resources/remote_posix_path/detail/remote-path-impl.cpp index ef04869..7d71cc7 100644 --- a/src/resources/remote_posix_path/detail/remote-path-impl.cpp +++ b/src/resources/remote_posix_path/detail/remote-path-impl.cpp @@ -43,8 +43,10 @@ namespace detail { // remote alias for convenience using remote_path_resource = resource_impl; -remote_path_resource::resource_impl(const std::shared_ptr parent) : - m_parent(std::static_pointer_cast(std::move(parent))) { } +remote_path_resource::resource_impl( + const std::shared_ptr parent) + : m_parent(std::static_pointer_cast( + std::move(parent))) {} std::string remote_path_resource::name() const { return "PENDING"; diff --git a/src/resources/remote_posix_path/detail/remote-path-impl.hpp b/src/resources/remote_posix_path/detail/remote-path-impl.hpp index 6abfb11..8865e02 100644 --- a/src/resources/remote_posix_path/detail/remote-path-impl.hpp +++ b/src/resources/remote_posix_path/detail/remote-path-impl.hpp @@ -55,6 +55,8 @@ struct resource_impl : public resource { const std::shared_ptr parent() const override final; std::string to_string() const override final; + + const bfs::path m_name_in_namespace; const std::shared_ptr m_parent; }; diff --git a/src/resources/shared_posix_path/detail/shared-path-impl.cpp b/src/resources/shared_posix_path/detail/shared-path-impl.cpp index 0f83fbd..6834c39 100644 --- a/src/resources/shared_posix_path/detail/shared-path-impl.cpp +++ b/src/resources/shared_posix_path/detail/shared-path-impl.cpp @@ -45,13 +45,13 @@ using shared_path_resource = resource_impl; shared_path_resource::resource_impl(const std::shared_ptr parent, const bfs::path& name) : - m_namespace_name(name), + m_name_in_namespace(name), m_canonical_path(parent->mount() / name), m_is_collection(bfs::is_directory(m_canonical_path)), m_parent(std::static_pointer_cast(std::move(parent))) { } std::string shared_path_resource::name() const { - return m_namespace_name.string(); + return m_name_in_namespace.string(); } resource_type shared_path_resource::type() const { diff --git a/src/resources/shared_posix_path/detail/shared-path-impl.hpp b/src/resources/shared_posix_path/detail/shared-path-impl.hpp index 270b407..c55251a 100644 --- a/src/resources/shared_posix_path/detail/shared-path-impl.hpp +++ b/src/resources/shared_posix_path/detail/shared-path-impl.hpp @@ -61,7 +61,7 @@ struct resource_impl : public resource { const std::shared_ptr parent() const override final; std::string to_string() const override final; - const bfs::path m_namespace_name; // absolute pathname w.r.t. backend's mount point + const bfs::path m_name_in_namespace; // absolute pathname w.r.t. backend's mount point const bfs::path m_canonical_path; // canonical pathname const bool m_is_collection; const std::shared_ptr m_parent; diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp index ff54033..1c38ab9 100644 --- a/tests/api-copy-remote-data.cpp +++ b/tests/api-copy-remote-data.cpp @@ -33,7 +33,8 @@ namespace bfs = boost::filesystem; -SCENARIO("copy local POSIX file to remote POSIX file", "[api::norns_submit_remote_copy_local_posix_files]") { +SCENARIO("copy local POSIX file to remote POSIX file", + "[api::norns_submit_remote_copy_local_posix_files]") { GIVEN("a running urd instance") { test_env env(true); @@ -44,8 +45,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", "[api::norns_submit_remot bfs::path src_mnt, dst_mnt; // create namespaces - std::tie(std::ignore, src_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); - std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid1, "mnt/tmp1", 16384); + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); // define input names const bfs::path src_file_at_root = "/file0"; -- GitLab From 305b9a92fcce75a169730677c9ab35fa798893de Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Feb 2019 15:02:33 +0100 Subject: [PATCH 03/51] Implemented remote single file transfers --- .gitmodules | 3 + configure.ac | 3 + src/Makefile.am | 28 +- src/api/request.cpp | 11 +- src/backends.hpp | 7 +- src/backends/backend-base.cpp | 10 +- src/backends/backend-base.hpp | 14 +- src/backends/lustre-fs.cpp | 10 +- src/backends/lustre-fs.hpp | 10 +- src/backends/nvml-dax.cpp | 10 +- src/backends/nvml-dax.hpp | 10 +- src/backends/posix-fs.cpp | 12 +- src/backends/posix-fs.hpp | 10 +- src/backends/process-memory.cpp | 8 +- src/backends/process-memory.hpp | 6 +- src/backends/remote-backend.cpp | 41 ++- src/backends/remote-backend.hpp | 6 +- src/common/types.cpp | 2 + src/common/types.hpp | 1 + src/externals/hermes | 1 + src/io/task-copy.hpp | 98 ++++++ src/io/task-info.cpp | 26 +- src/io/task-info.hpp | 21 +- src/io/task-manager.cpp | 220 ++++++++++++- src/io/task-manager.hpp | 18 ++ src/io/task-move.hpp | 98 ++++++ src/io/task-noop.hpp | 78 +++++ src/io/task-remote-transfer.hpp | 98 ++++++ src/io/task-remove.hpp | 81 +++++ ...th-to-remote-path.hpp => task-unknown.hpp} | 42 +-- src/io/task.cpp | 175 ----------- src/io/task.hpp | 155 +++++++-- src/io/transferor-registry.hpp | 19 +- src/io/transferors.hpp | 3 +- .../transferors/local-path-to-local-path.cpp | 53 ++++ .../transferors/local-path-to-local-path.hpp | 10 + .../local-path-to-remote-resource.cpp | 147 +++++++++ .../local-path-to-remote-resource.hpp | 98 ++++++ .../transferors/local-path-to-shared-path.cpp | 17 + .../transferors/local-path-to-shared-path.hpp | 8 + src/io/transferors/memory-to-local-path.cpp | 47 +++ src/io/transferors/memory-to-local-path.hpp | 8 + src/io/transferors/memory-to-remote-path.cpp | 17 + src/io/transferors/memory-to-remote-path.hpp | 8 + src/io/transferors/memory-to-shared-path.cpp | 17 + src/io/transferors/memory-to-shared-path.hpp | 8 + .../remote-resource-to-local-path.cpp | 294 ++++++++++++++++++ .../remote-resource-to-local-path.hpp | 98 ++++++ src/io/transferors/transferor.hpp | 7 + src/logger.hpp | 110 ++++++- src/namespaces/namespace-manager.hpp | 2 +- src/resources.hpp | 1 + .../detail/remote-path-impl.cpp | 2 +- .../detail/remote-path-info.cpp | 5 + .../detail/remote-resource-impl.cpp | 102 ++++++ .../detail/remote-resource-impl.hpp | 88 ++++++ .../detail/remote-resource-info.cpp} | 99 +++--- .../detail/remote-resource-info.hpp | 82 +++++ .../remote_resource/remote-resource.hpp | 47 +++ src/resources/resource-type.hpp | 3 + src/rpcs.cpp | 15 + src/rpcs.hpp | 216 +++++++++++++ src/urd.cpp | 270 +++++++++++++--- src/urd.hpp | 17 +- tests/Makefile.am | 2 + tests/api-copy-remote-data.cpp | 277 ++++++++++------- tests/api-task-submit.cpp | 33 +- 67 files changed, 3005 insertions(+), 538 deletions(-) create mode 100644 .gitmodules create mode 160000 src/externals/hermes create mode 100644 src/io/task-copy.hpp create mode 100644 src/io/task-move.hpp create mode 100644 src/io/task-noop.hpp create mode 100644 src/io/task-remote-transfer.hpp create mode 100644 src/io/task-remove.hpp rename src/io/{transferors/local-path-to-remote-path.hpp => task-unknown.hpp} (66%) create mode 100644 src/io/transferors/local-path-to-remote-resource.cpp create mode 100644 src/io/transferors/local-path-to-remote-resource.hpp create mode 100644 src/io/transferors/remote-resource-to-local-path.cpp create mode 100644 src/io/transferors/remote-resource-to-local-path.hpp create mode 100644 src/resources/remote_resource/detail/remote-resource-impl.cpp create mode 100644 src/resources/remote_resource/detail/remote-resource-impl.hpp rename src/{io/transferors/local-path-to-remote-path.cpp => resources/remote_resource/detail/remote-resource-info.cpp} (56%) create mode 100644 src/resources/remote_resource/detail/remote-resource-info.hpp create mode 100644 src/resources/remote_resource/remote-resource.hpp create mode 100644 src/rpcs.cpp create mode 100644 src/rpcs.hpp diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0617326 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/externals/hermes"] + path = src/externals/hermes + url = ssh://git@bscssrg.bsc.es:2222/hpc/hermes.git diff --git a/configure.ac b/configure.ac index 8ad287a..8b6fbbe 100644 --- a/configure.ac +++ b/configure.ac @@ -119,6 +119,9 @@ AS_IF([test "x$is_enabled_build_tests" = "xyes"], AC_CONFIG_FILES(tests/Makefile) ], []) +# check for mercury +PKG_CHECK_MODULES([MERCURY], [mercury >= 0.26]) + # check for libyaml-cpp PKG_CHECK_MODULES([YAMLCPP], [yaml-cpp >= 0.5.1]) diff --git a/src/Makefile.am b/src/Makefile.am index e994b4a..66ffdb8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -48,6 +48,11 @@ liburd_resources_la_SOURCES = \ resources/memory_buffer/detail/memory-region-impl.hpp \ resources/memory_buffer/detail/memory-region-info.cpp \ resources/memory_buffer/detail/memory-region-info.hpp \ + resources/remote_resource/remote-resource.hpp \ + resources/remote_resource/detail/remote-resource-impl.cpp \ + resources/remote_resource/detail/remote-resource-impl.hpp \ + resources/remote_resource/detail/remote-resource-info.cpp \ + resources/remote_resource/detail/remote-resource-info.hpp \ resources/remote_posix_path/remote-path.hpp \ resources/remote_posix_path/detail/remote-path-impl.cpp \ resources/remote_posix_path/detail/remote-path-impl.hpp \ @@ -67,9 +72,11 @@ liburd_resources_la_CXXFLAGS = \ liburd_resources_la_CPPFLAGS = \ -DSPDLOG_ENABLE_SYSLOG \ + -DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \ @BOOST_CPPFLAGS@ \ -I$(top_srcdir)/include \ -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ -I$(top_srcdir)/src/resources \ -I$(top_srcdir)/rpc \ -I$(top_builddir)/rpc @@ -81,6 +88,7 @@ liburd_resources_la_LDFLAGS = \ @BOOST_PROGRAM_OPTIONS_LIB@ \ @BOOST_SYSTEM_LIB@ \ @BOOST_THREAD_LIB@ \ + @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ -pthread @@ -137,12 +145,16 @@ liburd_aux_la_SOURCES = \ config/parsers.hpp \ config/defaults.hpp \ io.hpp \ - io/task.cpp \ io/task.hpp \ + io/task-copy.hpp \ io/task-info.cpp \ io/task-info.hpp \ io/task-manager.cpp \ io/task-manager.hpp \ + io/task-move.hpp \ + io/task-noop.hpp \ + io/task-remote-transfer.hpp \ + io/task-remove.hpp \ io/task-stats.cpp \ io/task-stats.hpp \ io/transferors.hpp \ @@ -151,8 +163,10 @@ liburd_aux_la_SOURCES = \ io/transferors/local-path-to-local-path.hpp \ io/transferors/local-path-to-shared-path.cpp \ io/transferors/local-path-to-shared-path.hpp \ - io/transferors/local-path-to-remote-path.cpp \ - io/transferors/local-path-to-remote-path.hpp \ + io/transferors/local-path-to-remote-resource.cpp \ + io/transferors/local-path-to-remote-resource.hpp \ + io/transferors/remote-resource-to-local-path.cpp \ + io/transferors/remote-resource-to-local-path.hpp \ io/transferors/memory-to-local-path.cpp \ io/transferors/memory-to-local-path.hpp \ io/transferors/memory-to-shared-path.cpp \ @@ -164,6 +178,8 @@ liburd_aux_la_SOURCES = \ job.hpp \ logger.hpp \ resources.hpp \ + rpcs.cpp \ + rpcs.hpp \ signal-listener.hpp \ thread-pool.hpp \ thread-pool/thread-pool.hpp \ @@ -183,9 +199,11 @@ liburd_aux_la_CXXFLAGS = \ liburd_aux_la_CPPFLAGS = \ -DSPDLOG_ENABLE_SYSLOG \ + -DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \ @BOOST_CPPFLAGS@ \ -I$(top_srcdir)/include \ -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ -I$(top_srcdir)/src/resources \ -I$(top_srcdir)/rpc \ -I$(top_builddir)/rpc @@ -197,6 +215,7 @@ liburd_aux_la_LDFLAGS = \ @BOOST_PROGRAM_OPTIONS_LIB@ \ @BOOST_SYSTEM_LIB@ \ @BOOST_THREAD_LIB@ \ + @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ @YAMLCPP_LIBS@ \ liburd_resources.la \ @@ -256,9 +275,11 @@ urd_CXXFLAGS = \ urd_CPPFLAGS = \ -DSPDLOG_ENABLE_SYSLOG \ + -DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \ @BOOST_CPPFLAGS@ \ -I$(top_srcdir)/include \ -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ -I$(top_srcdir)/rpc \ -I$(top_builddir)/rpc @@ -269,6 +290,7 @@ urd_LDFLAGS = \ @BOOST_PROGRAM_OPTIONS_LIB@ \ @BOOST_SYSTEM_LIB@ \ @BOOST_THREAD_LIB@ \ + @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ liburd_aux.la diff --git a/src/api/request.cpp b/src/api/request.cpp index c032852..9dcca44 100644 --- a/src/api/request.cpp +++ b/src/api/request.cpp @@ -166,6 +166,7 @@ create_from(const norns::rpc::Request_Task_Resource& res) { using norns::data::local_path_info; using norns::data::shared_path_info; using norns::data::remote_path_info; + using norns::data::remote_resource_info; assert(is_valid(res)); @@ -187,9 +188,13 @@ create_from(const norns::rpc::Request_Task_Resource& res) { // R_REMOTE assert(res.type() & R_REMOTE); - return std::make_shared(res.path().nsid(), - res.path().hostname(), - res.path().datapath()); + //return std::make_shared(res.path().nsid(), + // res.path().hostname(), + // res.path().datapath()); + + return std::make_shared(res.path().hostname(), + res.path().nsid(), + res.path().datapath()); } assert(res.type() & NORNS_NULL_RESOURCE); diff --git a/src/backends.hpp b/src/backends.hpp index bf9ea43..7565c82 100644 --- a/src/backends.hpp +++ b/src/backends.hpp @@ -38,8 +38,11 @@ namespace norns { namespace storage { - static const auto process_memory_backend = std::make_shared(); - static const auto remote_backend = std::make_shared(); + + constexpr static const auto memory_nsid = "[[internal::memory]]"; + + static const auto process_memory_backend = std::make_shared(memory_nsid); +// static const auto remote_backend = std::make_shared(); } // namespace storage } // namespace norns diff --git a/src/backends/backend-base.cpp b/src/backends/backend-base.cpp index 88b7d1a..313e531 100644 --- a/src/backends/backend-base.cpp +++ b/src/backends/backend-base.cpp @@ -41,8 +41,11 @@ backend_factory::get() { } std::shared_ptr -backend_factory::create(const backend_type type, bool track, - const bfs::path& mount, uint32_t quota) const { +backend_factory::create(const backend_type type, + const std::string& nsid, + bool track, + const bfs::path& mount, + uint32_t quota) const { boost::system::error_code ec; @@ -58,7 +61,8 @@ backend_factory::create(const backend_type type, bool track, const auto& it = m_registrar.find(id); if(it != m_registrar.end()){ - return std::shared_ptr(it->second(track, canonical_mount, quota)); + return std::shared_ptr( + it->second(nsid, track, canonical_mount, quota)); } else{ throw std::invalid_argument("Unrecognized backend type!"); diff --git a/src/backends/backend-base.hpp b/src/backends/backend-base.hpp index cbe7ee6..3154e14 100644 --- a/src/backends/backend-base.hpp +++ b/src/backends/backend-base.hpp @@ -59,6 +59,7 @@ protected: public: virtual ~backend() {}; + virtual std::string nsid() const = 0; virtual bool is_tracked() const = 0; virtual bool is_empty() const = 0; virtual bfs::path mount() const = 0; @@ -85,8 +86,13 @@ public: class backend_factory { - using creator_function = std::function< - std::shared_ptr(bool track, const bfs::path&, uint32_t)>; + using creator_function = + std::function< + std::shared_ptr( + const std::string&, + bool track, + const bfs::path&, + uint32_t)>; public: @@ -140,8 +146,8 @@ public: private: std::shared_ptr - create(const backend_type type, bool track, const bfs::path& mount, - uint32_t quota) const; + create(const backend_type type, const std::string&, bool track, + const bfs::path& mount, uint32_t quota) const; protected: backend_factory() {} diff --git a/src/backends/lustre-fs.cpp b/src/backends/lustre-fs.cpp index 5e614ae..1ae3d0c 100644 --- a/src/backends/lustre-fs.cpp +++ b/src/backends/lustre-fs.cpp @@ -33,11 +33,17 @@ namespace norns { namespace storage { -lustre::lustre(bool track, const bfs::path& mount, uint32_t quota) - : m_track(track), +lustre::lustre(const std::string& nsid, bool track, const bfs::path& mount, uint32_t quota) + : m_nsid(nsid), + m_track(track), m_mount(mount), m_quota(quota) { } +std::string +lustre::nsid() const { + return m_nsid; +} + bool lustre::is_tracked() const { return m_track; diff --git a/src/backends/lustre-fs.hpp b/src/backends/lustre-fs.hpp index 7489f89..d09ec8d 100644 --- a/src/backends/lustre-fs.hpp +++ b/src/backends/lustre-fs.hpp @@ -41,8 +41,9 @@ namespace storage { class lustre final : public storage::backend { public: - lustre(bool track, const bfs::path& mount, uint32_t quota); + lustre(const std::string& nsid, bool track, const bfs::path& mount, uint32_t quota); + std::string nsid() const override final; bool is_tracked() const override final; bool is_empty() const override final; bfs::path mount() const override final; @@ -57,9 +58,10 @@ public: std::string to_string() const override final; protected: - bool m_track; - bfs::path m_mount; - uint32_t m_quota; + std::string m_nsid; + bool m_track; + bfs::path m_mount; + uint32_t m_quota; }; //NORNS_REGISTER_BACKEND(backend_type::lustre, lustre); diff --git a/src/backends/nvml-dax.cpp b/src/backends/nvml-dax.cpp index 6b08824..0fe2ee3 100644 --- a/src/backends/nvml-dax.cpp +++ b/src/backends/nvml-dax.cpp @@ -33,11 +33,17 @@ namespace norns { namespace storage { -nvml_dax::nvml_dax(bool track, const bfs::path& mount, uint32_t quota) - : m_track(track), +nvml_dax::nvml_dax(const std::string& nsid, bool track, const bfs::path& mount, uint32_t quota) + : m_nsid(nsid), + m_track(track), m_mount(mount), m_quota(quota) { } +std::string +nvml_dax::nsid() const { + return m_nsid; +} + bool nvml_dax::is_tracked() const { return m_track; diff --git a/src/backends/nvml-dax.hpp b/src/backends/nvml-dax.hpp index 76b0cbe..075502c 100644 --- a/src/backends/nvml-dax.hpp +++ b/src/backends/nvml-dax.hpp @@ -40,8 +40,9 @@ namespace storage { class nvml_dax final : public storage::backend { public: - nvml_dax(bool track, const bfs::path& mount, uint32_t quota); + nvml_dax(const std::string& nsid, bool track, const bfs::path& mount, uint32_t quota); + std::string nsid() const override final; bool is_tracked() const override final; bool is_empty() const override final; bfs::path mount() const override final; @@ -56,9 +57,10 @@ public: std::string to_string() const final; private: - bool m_track; - bfs::path m_mount; - uint32_t m_quota; + std::string m_nsid; + bool m_track; + bfs::path m_mount; + uint32_t m_quota; }; //NORNS_REGISTER_BACKEND(backend_type::nvml, nvml_dax); diff --git a/src/backends/posix-fs.cpp b/src/backends/posix-fs.cpp index 0db04a6..5bc8ad6 100644 --- a/src/backends/posix-fs.cpp +++ b/src/backends/posix-fs.cpp @@ -55,12 +55,20 @@ bool contains(const bfs::path& p1, const bfs::path& p2) { namespace norns { namespace storage { -posix_filesystem::posix_filesystem(bool track, const bfs::path& mount, +posix_filesystem::posix_filesystem(const std::string& nsid, + bool track, + const bfs::path& mount, uint32_t quota) - : m_track(track), + : m_nsid(nsid), + m_track(track), m_mount(mount), m_quota(quota) { } +std::string +posix_filesystem::nsid() const { + return m_nsid; +} + bool posix_filesystem::is_tracked() const { return m_track; diff --git a/src/backends/posix-fs.hpp b/src/backends/posix-fs.hpp index 23cdbb5..5e558b8 100644 --- a/src/backends/posix-fs.hpp +++ b/src/backends/posix-fs.hpp @@ -41,8 +41,9 @@ namespace storage { class posix_filesystem final : public storage::backend { public: - posix_filesystem(bool track, const bfs::path& mount, uint32_t quota); + posix_filesystem(const std::string& nsid, bool track, const bfs::path& mount, uint32_t quota); + std::string nsid() const override final; bool is_tracked() const override final; bool is_empty() const override final; bfs::path mount() const override final; @@ -57,9 +58,10 @@ public: std::string to_string() const override final; private: - bool m_track; - bfs::path m_mount; - uint32_t m_quota; + std::string m_nsid; + bool m_track; + bfs::path m_mount; + uint32_t m_quota; }; //NORNS_REGISTER_BACKEND(backend_type::posix_filesystem, posix_filesystem); diff --git a/src/backends/process-memory.cpp b/src/backends/process-memory.cpp index f2a05b2..cb32c3a 100644 --- a/src/backends/process-memory.cpp +++ b/src/backends/process-memory.cpp @@ -33,7 +33,13 @@ namespace norns { namespace storage { namespace detail { -process_memory::process_memory() { } +process_memory::process_memory(const std::string& nsid) : + m_nsid(nsid) { } + +std::string +process_memory::nsid() const { + return m_nsid; +} bool process_memory::is_tracked() const { diff --git a/src/backends/process-memory.hpp b/src/backends/process-memory.hpp index d3727af..9db0236 100644 --- a/src/backends/process-memory.hpp +++ b/src/backends/process-memory.hpp @@ -42,8 +42,9 @@ namespace detail { class process_memory final : public storage::backend { public: - process_memory(); + process_memory(const std::string& nsid); + std::string nsid() const override final; bool is_tracked() const override final; bool is_empty() const override final; bfs::path mount() const override final; @@ -56,6 +57,9 @@ public: bool accepts(resource_info_ptr res) const override final; std::string to_string() const override final; + +private: + std::string m_nsid; }; // no need to register it since it's never going to be created diff --git a/src/backends/remote-backend.cpp b/src/backends/remote-backend.cpp index d5ae203..15b7a65 100644 --- a/src/backends/remote-backend.cpp +++ b/src/backends/remote-backend.cpp @@ -28,12 +28,19 @@ #include "backend-base.hpp" #include "resources.hpp" #include "remote-backend.hpp" +#include "logger.hpp" namespace norns { namespace storage { namespace detail { -remote_backend::remote_backend() {} +remote_backend::remote_backend(const std::string& nsid) : + m_nsid(nsid) {} + +std::string +remote_backend::nsid() const { + return m_nsid; +} bool remote_backend::is_tracked() const { @@ -45,11 +52,13 @@ remote_backend::is_empty() const { return false; } -bfs::path remote_backend::mount() const { +bfs::path +remote_backend::mount() const { return ""; } -uint32_t remote_backend::quota() const { +uint32_t +remote_backend::quota() const { return 0; } @@ -57,34 +66,44 @@ backend::resource_ptr remote_backend::new_resource(const resource_info_ptr& rinfo, const bool is_collection, std::error_code& ec) const { - (void) rinfo; (void) is_collection; (void) ec; - return std::make_shared(shared_from_this()); //XXX + + const auto d_rinfo = + std::static_pointer_cast(rinfo); + + return std::make_shared(shared_from_this(), d_rinfo); } backend::resource_ptr -remote_backend::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { - (void) rinfo; +remote_backend::get_resource(const resource_info_ptr& rinfo, + std::error_code& ec) const { (void) ec; - return std::make_shared(shared_from_this()); //XXX + + const auto d_rinfo = + std::static_pointer_cast(rinfo); + + return std::make_shared(shared_from_this(), d_rinfo); } void -remote_backend::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { +remote_backend::remove(const resource_info_ptr& rinfo, + std::error_code& ec) const { (void) rinfo; (void) ec; } std::size_t -remote_backend::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { +remote_backend::get_size(const resource_info_ptr& rinfo, + std::error_code& ec) const { (void) rinfo; (void) ec; return 0; //XXX } -bool remote_backend::accepts(resource_info_ptr res) const { +bool +remote_backend::accepts(resource_info_ptr res) const { switch(res->type()) { case data::resource_type::local_posix_path: return true; diff --git a/src/backends/remote-backend.hpp b/src/backends/remote-backend.hpp index 3e944bc..b4d5483 100644 --- a/src/backends/remote-backend.hpp +++ b/src/backends/remote-backend.hpp @@ -41,8 +41,9 @@ namespace detail { class remote_backend final : public storage::backend { public: - remote_backend(); + remote_backend(const std::string& nsid); + std::string nsid() const override final; bool is_tracked() const override final; bool is_empty() const override final; bfs::path mount() const override final; @@ -55,6 +56,9 @@ public: bool accepts(resource_info_ptr res) const override final; std::string to_string() const override final; + +private: + std::string m_nsid; }; // no need to register it since it's never going to be created diff --git a/src/common/types.cpp b/src/common/types.cpp index 2edaa8d..f838144 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -55,6 +55,8 @@ std::string to_string(iotask_type type) { return "DATA_MOVE"; case iotask_type::remove: return "DATA_REMOVE"; + case iotask_type::remote_transfer: + return "DATA_TRANSFER"; default: return "UNKNOWN_IOTASK"; } diff --git a/src/common/types.hpp b/src/common/types.hpp index 098ad00..aeffde8 100644 --- a/src/common/types.hpp +++ b/src/common/types.hpp @@ -48,6 +48,7 @@ enum class iotask_type { copy, move, remove, + remote_transfer, noop, unknown }; diff --git a/src/externals/hermes b/src/externals/hermes new file mode 160000 index 0000000..57619ec --- /dev/null +++ b/src/externals/hermes @@ -0,0 +1 @@ +Subproject commit 57619ec9e554cca491b2d495f4a64aa411f3a564 diff --git a/src/io/task-copy.hpp b/src/io/task-copy.hpp new file mode 100644 index 0000000..6a81448 --- /dev/null +++ b/src/io/task-copy.hpp @@ -0,0 +1,98 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_TASK_COPY_HPP__ +#define __IO_TASK_COPY_HPP__ + +namespace norns { +namespace io { + +///////////////////////////////////////////////////////////////////////////////// +// specializations for copy tasks +///////////////////////////////////////////////////////////////////////////////// +template<> +inline void +task::operator()() { + + const auto tid = m_task_info->id(); + const auto type = m_task_info->type(); + const auto auth = m_task_info->auth(); + const auto src_backend = m_task_info->src_backend(); + const auto src_rinfo = m_task_info->src_rinfo(); + const auto dst_backend = m_task_info->dst_backend(); + const auto dst_rinfo = m_task_info->dst_rinfo(); + std::error_code ec; + + // helper lambda for error reporting + const auto log_error = [&] (const std::string& msg) { + + m_task_info->update_status(task_status::finished_with_error, + urd_error::system_error, ec); + std::string r_msg = "[{}] " + msg + ": {}"; + + LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); + LOGGER_WARN("[{}] I/O task completed with error", tid); + + }; + + LOGGER_WARN("[{}] Starting I/O task", tid); + LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); + LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); + LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); + + m_task_info->update_status(task_status::running); + + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + log_error("Could not access input data " + src_rinfo->to_string()); + return; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + log_error("Could not create output data " + dst_rinfo->to_string()); + return; + } + + ec = m_transferor->transfer(auth, m_task_info, src, dst); + + if(ec) { + log_error("Transfer failed"); + return; + } + + LOGGER_WARN("[{}] I/O task completed successfully", tid); + m_task_info->update_status(task_status::finished, urd_error::success, + std::make_error_code(static_cast(ec.value()))); +} + +} // namespace io +} // namespace norns + +#endif // __IO_TASK_COPY_HPP__ diff --git a/src/io/task-info.cpp b/src/io/task-info.cpp index 11d80de..085bae5 100644 --- a/src/io/task-info.cpp +++ b/src/io/task-info.cpp @@ -35,9 +35,12 @@ namespace norns { namespace io { task_info::task_info(const iotask_id tid, const iotask_type type, - const auth::credentials& auth, - const backend_ptr src_backend, const resource_info_ptr src_rinfo, - const backend_ptr dst_backend, const resource_info_ptr dst_rinfo) : + const auth::credentials& auth, + const backend_ptr src_backend, + const resource_info_ptr src_rinfo, + const backend_ptr dst_backend, + const resource_info_ptr dst_rinfo, + const boost::any& ctx) : m_id(tid), m_type(type), m_auth(auth), @@ -45,6 +48,7 @@ task_info::task_info(const iotask_id tid, const iotask_type type, m_src_rinfo(src_rinfo), m_dst_backend(dst_backend), m_dst_rinfo(dst_rinfo), + m_ctx(ctx), m_status(task_status::pending), m_task_error(urd_error::success), m_sys_error(), @@ -52,6 +56,10 @@ task_info::task_info(const iotask_id tid, const iotask_type type, m_sent_bytes(), m_total_bytes() { + if(!src_rinfo) { + return; + } + std::error_code ec; m_total_bytes = src_backend->get_size(src_rinfo, ec); @@ -61,6 +69,8 @@ task_info::task_info(const iotask_id tid, const iotask_type type, } } +task_info::~task_info() { } + iotask_id task_info::id() const { return m_id; @@ -96,6 +106,16 @@ task_info::dst_rinfo() const { return m_dst_rinfo; } +boost::any +task_info::context() const { + return m_ctx; +} + +void +task_info::set_context(const boost::any& ctx) { + m_ctx = ctx; +} + task_status task_info::status() const { boost::shared_lock lock(m_mutex); diff --git a/src/io/task-info.hpp b/src/io/task-info.hpp index c59dedb..566d021 100644 --- a/src/io/task-info.hpp +++ b/src/io/task-info.hpp @@ -28,6 +28,7 @@ #ifndef __TASK_INFO_HPP__ #define __TASK_INFO_HPP__ +#include #include #include "backends.hpp" #include "resources.hpp" @@ -45,10 +46,16 @@ struct task_info { using backend_ptr = std::shared_ptr; using resource_info_ptr = std::shared_ptr; - task_info(const iotask_id tid, const iotask_type type, - const auth::credentials& creds, - const backend_ptr src_backend, const resource_info_ptr src_rinfo, - const backend_ptr dst_backend, const resource_info_ptr dst_rinfo); + task_info(const iotask_id tid, + const iotask_type type, + const auth::credentials& creds, + const backend_ptr src_backend, + const resource_info_ptr src_rinfo, + const backend_ptr dst_backend, + const resource_info_ptr dst_rinfo, + const boost::any& ctx = {}); + + ~task_info(); iotask_id id() const; iotask_type type() const; @@ -58,6 +65,9 @@ struct task_info { backend_ptr dst_backend() const; resource_info_ptr dst_rinfo() const; + boost::any context() const; + void set_context(const boost::any& ctx); + task_status status() const; void update_status(const task_status st); void update_status(const task_status st, const urd_error ec, @@ -92,6 +102,9 @@ struct task_info { const backend_ptr m_dst_backend; const resource_info_ptr m_dst_rinfo; + // optional task context + boost::any m_ctx; + // general task status task_status m_status; urd_error m_task_error; diff --git a/src/io/task-manager.cpp b/src/io/task-manager.cpp index 0651d5d..03cadde 100644 --- a/src/io/task-manager.cpp +++ b/src/io/task-manager.cpp @@ -137,7 +137,8 @@ task_manager::register_transfer_plugin(const data::resource_type t1, std::tuple> -task_manager::create_task(iotask_type type, const auth::credentials& auth, +task_manager::create_task(iotask_type type, + const auth::credentials& auth, const std::vector& backend_ptrs, const std::vector& rinfo_ptrs) { @@ -294,6 +295,223 @@ task_manager::create_task(iotask_type type, const auth::credentials& auth, return std::make_tuple(urd_error::success, tid); } +std::tuple> +task_manager::create_local_initiated_task(iotask_type type, + const auth::credentials& auth, + const std::vector& backend_ptrs, + const std::vector& rinfo_ptrs) { + + boost::unique_lock lock(m_mutex); + + // fetch an iotask_id for this task + iotask_id tid = ++m_id_base; + + if(m_task_info.count(tid) != 0) { + --m_id_base; + return std::make_tuple(urd_error::too_many_tasks, boost::none); + } + + // immediately-invoked lambda to create the appropriate metadata for the task + // and register it into m_task_info + const auto task_info_ptr = [&]() { + + assert(backend_ptrs.size() == 1 || backend_ptrs.size() == 2); + + const backend_ptr src_backend = backend_ptrs[0]; + const resource_info_ptr src_rinfo = rinfo_ptrs[0]; + const backend_ptr dst_backend = (backend_ptrs.size() == 1 ? + nullptr : backend_ptrs[1]); + const resource_info_ptr dst_rinfo = (rinfo_ptrs.size() == 1 ? + nullptr : rinfo_ptrs[1]); + + auto it = m_task_info.end(); + std::tie(it, std::ignore) = m_task_info.emplace(tid, + std::make_shared(tid, type, auth, + src_backend, src_rinfo, + dst_backend, dst_rinfo)); + return it->second; + }(); + + + std::shared_ptr tx_ptr; + + if(type == iotask_type::copy || type == iotask_type::move) { + + assert(backend_ptrs.size() == 2); + + tx_ptr = m_transferor_registry.get(rinfo_ptrs[0]->type(), + rinfo_ptrs[1]->type()); + if(!tx_ptr) { + return std::make_tuple(urd_error::not_supported, boost::none); + } + + LOGGER_DEBUG("Selected plugin: {}", tx_ptr->to_string()); + + if(!tx_ptr->validate(rinfo_ptrs[0], rinfo_ptrs[1])) { + return std::make_tuple(urd_error::bad_args, boost::none); + } + } + + if(m_dry_run) { + type = iotask_type::noop; + } + + switch(type) { + case iotask_type::remove: + { + assert(backend_ptrs.size() == 1); + + return std::make_tuple( + urd_error::success, + generic_task(type, + io::task( + std::move(task_info_ptr)))); + } + + case iotask_type::copy: + { + return std::make_tuple( + urd_error::success, + generic_task(type, + io::task( + std::move(task_info_ptr), std::move(tx_ptr)))); + break; + } + + case iotask_type::move: + { + + return std::make_tuple( + urd_error::success, + generic_task(type, + io::task( + std::move(task_info_ptr), std::move(tx_ptr)))); + break; + } + + case iotask_type::noop: + { + return std::make_tuple( + urd_error::success, + generic_task(type, + io::task( + std::move(task_info_ptr), m_dry_run_duration))); + break; + } + + default: + break; + } + + return std::make_tuple(urd_error::bad_args, boost::none); +} + +std::tuple> +task_manager::create_remote_initiated_task(iotask_type task_type, + const auth::credentials& auth, + const boost::any& ctx, + const backend_ptr src_backend, + const resource_info_ptr src_rinfo, + const backend_ptr dst_backend, + const resource_info_ptr dst_rinfo) { + + boost::unique_lock lock(m_mutex); + + // fetch an iotask_id for this task + iotask_id tid = ++m_id_base; + + if(m_task_info.count(tid) != 0) { + --m_id_base; + return std::make_tuple(urd_error::too_many_tasks, boost::none); + } + + // immediately-invoked lambda to create the appropriate metadata for the task + // and register it into m_task_info + const auto task_info_ptr = [&]() { + + auto it = m_task_info.end(); + std::tie(it, std::ignore) = m_task_info.emplace(tid, + std::make_shared(tid, task_type, auth, + src_backend, src_rinfo, + dst_backend, dst_rinfo, + ctx)); + return it->second; + }(); + + auto tx_ptr = + m_transferor_registry.get(src_rinfo->type(), dst_rinfo->type()); + + if(!tx_ptr) { + return std::make_tuple(urd_error::not_supported, boost::none); + } + + LOGGER_DEBUG("Selected plugin: {}", tx_ptr->to_string()); + + if(!tx_ptr->validate(src_rinfo, dst_rinfo)) { + return std::make_tuple(urd_error::bad_args, boost::none); + } + + return std::make_tuple( + urd_error::success, + generic_task(task_type, + io::task( + std::move(task_info_ptr), std::move(tx_ptr)))); +} + + +urd_error +task_manager::enqueue_task(io::generic_task&& t) { + + // helper lambda to register the completion of tasks so that we can keep track + // of the consumed bandwidth by each task + // N.B: we use capture-by-value here so that the task_info_ptr is valid when + // the callback is invoked. + const auto completion_callback = [=]() { + assert(t.info()->status() == task_status::finished || + t.info()->status() == task_status::finished_with_error); + + LOGGER_DEBUG("Task {} finished [{} MiB/s]", + t.info()->id(), t.info()->bandwidth()); + + auto bw = t.info()->bandwidth(); + + // bw might be nan if the task did not finish correctly + if(!std::isnan(bw)) { + + const auto key = std::make_pair(t.info()->src_rinfo()->nsid(), + t.info()->dst_rinfo()->nsid()); + + if(!m_bandwidth_backlog.count(key)) { + m_bandwidth_backlog.emplace(key, + boost::circular_buffer(m_backlog_size)); + } + + m_bandwidth_backlog.at(key).push_back(bw); + } + }; + + switch(t.m_type) { + case iotask_type::remove: + case iotask_type::noop: + { + m_runners.submit_and_forget(t); + break; + } + + case iotask_type::copy: + case iotask_type::move: + { + m_runners.submit_with_epilog_and_forget(t, completion_callback); + break; + } + + default: + return urd_error::bad_args; + } + + return urd_error::success; +} + std::shared_ptr task_manager::find(iotask_id tid) const { diff --git a/src/io/task-manager.hpp b/src/io/task-manager.hpp index 5529ef0..36a10c7 100644 --- a/src/io/task-manager.hpp +++ b/src/io/task-manager.hpp @@ -91,6 +91,24 @@ struct task_manager { const std::vector>& backend_ptrs, const std::vector>& rinfo_ptrs); + std::tuple> + create_local_initiated_task(iotask_type type, + const auth::credentials& auth, + const std::vector& backend_ptrs, + const std::vector& rinfo_ptrs); + + std::tuple> + create_remote_initiated_task(iotask_type task_type, + const auth::credentials& auth, + const boost::any& ctx, + const backend_ptr src_backend, + const resource_info_ptr src_rinfo, + const backend_ptr dst_backend, + const resource_info_ptr dst_rinfo); + + urd_error + enqueue_task(io::generic_task&& t); + std::shared_ptr find(iotask_id) const; diff --git a/src/io/task-move.hpp b/src/io/task-move.hpp new file mode 100644 index 0000000..8a64ac6 --- /dev/null +++ b/src/io/task-move.hpp @@ -0,0 +1,98 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_TASK_MOVE_HPP__ +#define __IO_TASK_MOVE_HPP__ + +namespace norns { +namespace io { + +///////////////////////////////////////////////////////////////////////////////// +// specializations for move tasks +///////////////////////////////////////////////////////////////////////////////// +template<> +inline void +task::operator()() { + + std::error_code ec; + + const auto tid = m_task_info->id(); + const auto type = m_task_info->type(); + const auto src_backend = m_task_info->src_backend(); + const auto src_rinfo = m_task_info->src_rinfo(); + const auto dst_backend = m_task_info->dst_backend(); + const auto dst_rinfo = m_task_info->dst_rinfo(); + const auto auth = m_task_info->auth(); + + // helper lambda for error reporting + const auto log_error = [&] (const std::string& msg) { + m_task_info->update_status(task_status::finished_with_error, + urd_error::system_error, ec); + + std::string r_msg = "[{}] " + msg + ": {}"; + + LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); + LOGGER_WARN("[{}] I/O task completed with error", tid); + }; + + LOGGER_WARN("[{}] Starting I/O task", tid); + LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); + LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); + LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); + + m_task_info->update_status(task_status::running); + + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + log_error("Could not access input data " + src_rinfo->to_string()); + return; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + log_error("Could not create output data " + dst_rinfo->to_string()); + return; + } + + ec = m_transferor->transfer(auth, m_task_info, src, dst); + + if(ec) { + log_error("Transfer failed"); + return; + } + + LOGGER_WARN("[{}] I/O task completed successfully", tid); + m_task_info->update_status(task_status::finished, urd_error::success, + std::make_error_code(static_cast(ec.value()))); +} + +} // namespace io +} // namespace norns + +#endif // __IO_TASK_MOVE_HPP__ diff --git a/src/io/task-noop.hpp b/src/io/task-noop.hpp new file mode 100644 index 0000000..73de719 --- /dev/null +++ b/src/io/task-noop.hpp @@ -0,0 +1,78 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_TASK_NOOP_HPP__ +#define __IO_TASK_NOOP_HPP__ + +namespace norns { +namespace io { + +///////////////////////////////////////////////////////////////////////////////// +// specializations for noop tasks +///////////////////////////////////////////////////////////////////////////////// +template <> +struct task { + + using task_info_ptr = std::shared_ptr; + + task(const task_info_ptr&& task_info, uint32_t sleep_duration) + : m_task_info(std::move(task_info)), + m_sleep_duration(sleep_duration) { } + + task(const task& other) = default; + task(task&& rhs) = default; + task& operator=(const task& other) = default; + task& operator=(task&& rhs) = default; + + void operator()() { + const auto tid = m_task_info->id(); + + LOGGER_WARN("[{}] Starting noop I/O task", tid); + + LOGGER_DEBUG("[{}] Sleep for {} usecs", tid, m_sleep_duration); + usleep(m_sleep_duration); + + m_task_info->update_status(task_status::running); + + LOGGER_WARN("[{}] noop I/O task \"running\"", tid); + + LOGGER_DEBUG("[{}] Sleep for {} usecs", tid, m_sleep_duration); + usleep(m_sleep_duration); + + m_task_info->update_status(task_status::finished); + + LOGGER_WARN("[{}] noop I/O task completed successfully", tid); + } + + task_info_ptr m_task_info; + uint32_t m_sleep_duration; +}; + +} // namespace io +} // namespace norns + +#endif // __IO_TASK_NOOP_HPP__ diff --git a/src/io/task-remote-transfer.hpp b/src/io/task-remote-transfer.hpp new file mode 100644 index 0000000..1f9ba96 --- /dev/null +++ b/src/io/task-remote-transfer.hpp @@ -0,0 +1,98 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_TASK_REMOTE_TRANSFER_HPP__ +#define __IO_TASK_REMOTE_TRANSFER_HPP__ + +namespace norns { +namespace io { + +///////////////////////////////////////////////////////////////////////////////// +// specializations for remote transfer tasks +///////////////////////////////////////////////////////////////////////////////// +template<> +inline void +task::operator()() { + + const auto tid = m_task_info->id(); + const auto type = m_task_info->type(); + const auto auth = m_task_info->auth(); + const auto src_backend = m_task_info->src_backend(); + const auto src_rinfo = m_task_info->src_rinfo(); + const auto dst_backend = m_task_info->dst_backend(); + const auto dst_rinfo = m_task_info->dst_rinfo(); + std::error_code ec; + + // helper lambda for error reporting + const auto log_error = [&] (const std::string& msg) { + + m_task_info->update_status(task_status::finished_with_error, + urd_error::system_error, ec); + std::string r_msg = "[{}] " + msg + ": {}"; + + LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); + LOGGER_WARN("[{}] I/O task completed with error", tid); + + }; + + LOGGER_WARN("[{}] Starting I/O task", tid); + LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); + LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); + LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); + + m_task_info->update_status(task_status::running); + + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + log_error("Could not access input data " + src_rinfo->to_string()); + return; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + log_error("Could not create output data " + dst_rinfo->to_string()); + return; + } + + ec = m_transferor->transfer(auth, m_task_info, src, dst); + + if(ec) { + log_error("Transfer failed"); + return; + } + + LOGGER_WARN("[{}] I/O task completed successfully", tid); + m_task_info->update_status(task_status::finished, urd_error::success, + std::make_error_code(static_cast(ec.value()))); +} + +} // namespace io +} // namespace norns + +#endif // __IO_TASK_REMOTE_TRANSFER_HPP__ diff --git a/src/io/task-remove.hpp b/src/io/task-remove.hpp new file mode 100644 index 0000000..49ac5a6 --- /dev/null +++ b/src/io/task-remove.hpp @@ -0,0 +1,81 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_TASK_REMOVE_HPP__ +#define __IO_TASK_REMOVE_HPP__ + +namespace norns { +namespace io { + +///////////////////////////////////////////////////////////////////////////////// +// specializations for remove tasks +///////////////////////////////////////////////////////////////////////////////// +template<> +inline void +task::operator()() { + + std::error_code ec; + + const auto tid = m_task_info->id(); + const auto type = m_task_info->type(); + const auto src_backend = m_task_info->src_backend(); + const auto src_rinfo = m_task_info->src_rinfo(); + //const auto auth = m_task_info->auth(); + + // helper lambda for error reporting + const auto log_error = [&] (const std::string& msg) { + m_task_info->update_status(task_status::finished_with_error, + urd_error::system_error, ec); + + std::string r_msg = "[{}] " + msg + ": {}"; + + LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); + LOGGER_WARN("[{}] I/O task completed with error", tid); + }; + + LOGGER_WARN("[{}] Starting I/O task", tid); + LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); + LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); + + m_task_info->update_status(task_status::running); + + src_backend->remove(src_rinfo, ec); + + if(ec) { + log_error("Failed to remove resource " + src_rinfo->to_string()); + return; + } + + LOGGER_WARN("[{}] I/O task completed successfully", tid); + m_task_info->update_status(task_status::finished, urd_error::success, + std::make_error_code(static_cast(ec.value()))); +} + +} // namespace io +} // namespace norns + +#endif // __IO_TASK_REMOVE_HPP__ diff --git a/src/io/transferors/local-path-to-remote-path.hpp b/src/io/task-unknown.hpp similarity index 66% rename from src/io/transferors/local-path-to-remote-path.hpp rename to src/io/task-unknown.hpp index db5fbd3..c3d5f8d 100644 --- a/src/io/transferors/local-path-to-remote-path.hpp +++ b/src/io/task-unknown.hpp @@ -1,5 +1,5 @@ /************************************************************************* - * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * * Centro Nacional de Supercomputacion * * All rights reserved. * * * @@ -25,38 +25,26 @@ * . * *************************************************************************/ -#ifndef __IO_LOCAL_PATH_TO_REMOTE_PATH_TX__ -#define __IO_LOCAL_PATH_TO_REMOTE_PATH_TX__ - -#include -#include -#include "transferor.hpp" +#ifndef __IO_TASK_UNKNOWN_HPP__ +#define __IO_TASK_UNKNOWN_HPP__ namespace norns { - -// forward declarations -namespace auth { -struct credentials; -} - -namespace data { -struct resource_info; -struct resource; -} - namespace io { -struct local_path_to_remote_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; - std::string to_string() const override final; +///////////////////////////////////////////////////////////////////////////////// +// specializations for unknown tasks +///////////////////////////////////////////////////////////////////////////////// +template <> +struct task { + task() { } + task(const task& other) = default; + task(task&& rhs) = default; + task& operator=(const task& other) = default; + task& operator=(task&& rhs) = default; + void operator()() { } }; } // namespace io } // namespace norns -#endif /* __LOCAL_PATH_TO_REMOTE_PATH_TX__ */ +#endif // __IO_TASK_UNKNOWN_HPP__ diff --git a/src/io/task.cpp b/src/io/task.cpp index c4ed40a..7c4efb9 100644 --- a/src/io/task.cpp +++ b/src/io/task.cpp @@ -36,185 +36,10 @@ namespace norns { namespace io { -///////////////////////////////////////////////////////////////////////////////// -// specializations for copy tasks -///////////////////////////////////////////////////////////////////////////////// -template<> -void -task::operator()() { - std::error_code ec; - const auto tid = m_task_info->id(); - const auto type = m_task_info->type(); - const auto src_backend = m_task_info->src_backend(); - const auto src_rinfo = m_task_info->src_rinfo(); - const auto dst_backend = m_task_info->dst_backend(); - const auto dst_rinfo = m_task_info->dst_rinfo(); - const auto auth = m_task_info->auth(); - // helper lambda for error reporting - const auto log_error = [&] (const std::string& msg) { - m_task_info->update_status(task_status::finished_with_error, - urd_error::system_error, ec); - std::string r_msg = "[{}] " + msg + ": {}"; - - LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); - LOGGER_WARN("[{}] I/O task completed with error", tid); - }; - - LOGGER_WARN("[{}] Starting I/O task", tid); - LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); - LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); - LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); - - m_task_info->update_status(task_status::running); - - auto src = src_backend->get_resource(src_rinfo, ec); - - if(ec) { - log_error("Could not access input data " + src_rinfo->to_string()); - return; - } - - auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); - - if(ec) { - log_error("Could not create output data " + dst_rinfo->to_string()); - return; - } - - ec = m_transferor->transfer(auth, m_task_info, src, dst); - - if(ec) { - log_error("Transfer failed"); - return; - } - - LOGGER_WARN("[{}] I/O task completed successfully", tid); - m_task_info->update_status(task_status::finished, urd_error::success, - std::make_error_code(static_cast(ec.value()))); -} - - -///////////////////////////////////////////////////////////////////////////////// -// specializations for move tasks -///////////////////////////////////////////////////////////////////////////////// -template<> -void -task::operator()() { - - std::error_code ec; - - const auto tid = m_task_info->id(); - const auto type = m_task_info->type(); - const auto src_backend = m_task_info->src_backend(); - const auto src_rinfo = m_task_info->src_rinfo(); - const auto dst_backend = m_task_info->dst_backend(); - const auto dst_rinfo = m_task_info->dst_rinfo(); - const auto auth = m_task_info->auth(); - - // helper lambda for error reporting - const auto log_error = [&] (const std::string& msg) { - m_task_info->update_status(task_status::finished_with_error, - urd_error::system_error, ec); - - std::string r_msg = "[{}] " + msg + ": {}"; - - LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); - LOGGER_WARN("[{}] I/O task completed with error", tid); - }; - - LOGGER_WARN("[{}] Starting I/O task", tid); - LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); - LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); - LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); - - m_task_info->update_status(task_status::running); - - auto src = src_backend->get_resource(src_rinfo, ec); - - if(ec) { - log_error("Could not access input data " + src_rinfo->to_string()); - return; - } - - auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); - - if(ec) { - log_error("Could not create output data " + dst_rinfo->to_string()); - return; - } - - ec = m_transferor->transfer(auth, m_task_info, src, dst); - - if(ec) { - log_error("Transfer failed"); - return; - } - - LOGGER_WARN("[{}] I/O task completed successfully", tid); - m_task_info->update_status(task_status::finished, urd_error::success, - std::make_error_code(static_cast(ec.value()))); -} - - -///////////////////////////////////////////////////////////////////////////////// -// specializations for remove tasks -///////////////////////////////////////////////////////////////////////////////// -template<> -void -task::operator()() { - - std::error_code ec; - - const auto tid = m_task_info->id(); - const auto type = m_task_info->type(); - const auto src_backend = m_task_info->src_backend(); - const auto src_rinfo = m_task_info->src_rinfo(); - const auto auth = m_task_info->auth(); - - // helper lambda for error reporting - const auto log_error = [&] (const std::string& msg) { - m_task_info->update_status(task_status::finished_with_error, - urd_error::system_error, ec); - - std::string r_msg = "[{}] " + msg + ": {}"; - - LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); - LOGGER_WARN("[{}] I/O task completed with error", tid); - }; - - LOGGER_WARN("[{}] Starting I/O task", tid); - LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); - LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); - - m_task_info->update_status(task_status::running); - - src_backend->remove(src_rinfo, ec); - - if(ec) { - log_error("Failed to remove resource " + src_rinfo->to_string()); - return; - } - - LOGGER_WARN("[{}] I/O task completed successfully", tid); - m_task_info->update_status(task_status::finished, urd_error::success, - std::make_error_code(static_cast(ec.value()))); -} - -///////////////////////////////////////////////////////////////////////////////// -// specializations for unknown tasks -///////////////////////////////////////////////////////////////////////////////// -template<> -void -task::operator()() { - - const auto tid = m_task_info->id(); - - LOGGER_CRITICAL("[{}] Unknown task type detected!", tid); -} } // namespace io } // namespace norns diff --git a/src/io/task.hpp b/src/io/task.hpp index cd02e3a..95b1c67 100644 --- a/src/io/task.hpp +++ b/src/io/task.hpp @@ -30,6 +30,7 @@ #include #include +#include #include "logger.hpp" #include "common.hpp" @@ -61,48 +62,146 @@ struct task { : m_task_info(std::move(task_info)), m_transferor(std::move(tx_ptr)) { } + task(const task& other) = default; + task(task&& rhs) = default; + task& operator=(const task& other) = default; + task& operator=(task&& rhs) = default; + void operator()(); - const task_info_ptr m_task_info; - const transferor_ptr m_transferor; + task_info_ptr m_task_info; + transferor_ptr m_transferor; }; -///////////////////////////////////////////////////////////////////////////////// -// specializations for noop tasks -///////////////////////////////////////////////////////////////////////////////// -template <> -struct task { - - using task_info_ptr = std::shared_ptr; - - task(const task_info_ptr&& task_info, uint32_t sleep_duration) - : m_task_info(std::move(task_info)), - m_sleep_duration(sleep_duration) { } - - void operator()() { - const auto tid = m_task_info->id(); +} // namespace io +} // namespace norns - LOGGER_WARN("[{}] Starting noop I/O task", tid); - LOGGER_DEBUG("[{}] Sleep for {} usecs", tid, m_sleep_duration); - usleep(m_sleep_duration); +#include "task-copy.hpp" +#include "task-move.hpp" +#include "task-remove.hpp" +#include "task-remote-transfer.hpp" +#include "task-noop.hpp" +#include "task-unknown.hpp" - m_task_info->update_status(task_status::running); - LOGGER_WARN("[{}] noop I/O task \"running\"", tid); +namespace norns { +namespace io { - LOGGER_DEBUG("[{}] Sleep for {} usecs", tid, m_sleep_duration); - usleep(m_sleep_duration); +struct generic_task { + + generic_task() : + m_type(iotask_type::unknown), + m_impl(task()) {} + + template + generic_task(iotask_type type, + TaskImpl&& impl) : + m_type(type), + m_impl(std::forward(impl)) { } + + generic_task(const generic_task& other) = default; + generic_task(generic_task&& rhs) = default; + generic_task& operator=(const generic_task& other) = default; + generic_task& operator=(generic_task&& rhs) = default; + + iotask_id + id() const { + switch(m_type) { + case iotask_type::noop: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info->id(); + } + + case iotask_type::copy: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info->id(); + } + + case iotask_type::move: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info->id(); + } + + case iotask_type::remove: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info->id(); + } + + default: + return static_cast(0); + } + } - m_task_info->update_status(task_status::finished); + std::shared_ptr + info() const { + switch(m_type) { + case iotask_type::noop: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info; + } + + case iotask_type::copy: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info; + } + + case iotask_type::move: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info; + } + + case iotask_type::remove: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info; + } + + default: + return {}; + } + } - LOGGER_WARN("[{}] noop I/O task completed successfully", tid); + void + operator()() { + switch(m_type) { + case iotask_type::noop: + boost::get>(m_impl)(); + break; + case iotask_type::copy: + boost::get>(m_impl)(); + break; + case iotask_type::move: + boost::get>(m_impl)(); + break; + case iotask_type::remove: + boost::get>(m_impl)(); + break; + case iotask_type::remote_transfer: + boost::get>(m_impl)(); + break; + default: + break; + } } - const task_info_ptr m_task_info; - const uint32_t m_sleep_duration; -}; + iotask_type m_type; + boost::variant< + io::task, + io::task, + io::task, + io::task, + io::task, + io::task> m_impl; +}; } // namespace io } // namespace norns diff --git a/src/io/transferor-registry.hpp b/src/io/transferor-registry.hpp index cd6123e..83a4745 100644 --- a/src/io/transferor-registry.hpp +++ b/src/io/transferor-registry.hpp @@ -47,27 +47,18 @@ struct transferor; struct transferor_registry { - struct transferor_hash { - template - std::size_t operator()(const std::pair &x) const { - std::size_t seed = 0; - boost::hash_combine(seed, x.first); - boost::hash_combine(seed, x.second); - return seed; - } - }; - bool add(const data::resource_type t1, const data::resource_type t2, std::shared_ptr&& tr); std::shared_ptr get(const data::resource_type t1, const data::resource_type t2) const; - std::unordered_map, + using key_type = + std::pair; + + std::unordered_map, - transferor_hash> m_transferors; + boost::hash> m_transferors; }; } // namespace io diff --git a/src/io/transferors.hpp b/src/io/transferors.hpp index 12b4232..8bf44aa 100644 --- a/src/io/transferors.hpp +++ b/src/io/transferors.hpp @@ -30,9 +30,10 @@ #include "transferors/local-path-to-local-path.hpp" #include "transferors/local-path-to-shared-path.hpp" -#include "transferors/local-path-to-remote-path.hpp" +#include "transferors/local-path-to-remote-resource.hpp" #include "transferors/memory-to-local-path.hpp" #include "transferors/memory-to-shared-path.hpp" #include "transferors/memory-to-remote-path.hpp" +#include "transferors/remote-resource-to-local-path.hpp" #endif /* __IO_TRANSFERORS_HPP__ */ diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 851b00d..bb61540 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -231,6 +231,59 @@ local_path_to_local_path_transferor::transfer( d_dst.canonical_path()); } + +std::error_code +local_path_to_local_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + + (void) auth; + const auto src_backend = task_info->src_backend(); + const auto dst_backend = task_info->dst_backend(); + + std::error_code ec; + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + LOGGER_ERROR("[{}] Could not access input resource '{}': {}", + task_info->id(), + src_rinfo->to_string(), + ec.message()); + return ec; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + LOGGER_ERROR("[{}] Could not create output resource '{}': {}", + task_info->id(), + src_rinfo->to_string(), + ec.message()); + return ec; + } + + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] transfer: {} -> {}", + task_info->id(), + d_src.canonical_path(), + d_dst.canonical_path()); + + if(bfs::is_directory(d_src.canonical_path())) { + return ::copy_directory(task_info, d_src.canonical_path(), + d_dst.canonical_path()); + } + + return ::copy_file(task_info, + d_src.canonical_path(), + d_dst.canonical_path()); +} + std::string local_path_to_local_path_transferor::to_string() const { return "transferor[local_path => local_path]"; diff --git a/src/io/transferors/local-path-to-local-path.hpp b/src/io/transferors/local-path-to-local-path.hpp index 2d4f802..98f1e4d 100644 --- a/src/io/transferors/local-path-to-local-path.hpp +++ b/src/io/transferors/local-path-to-local-path.hpp @@ -50,10 +50,20 @@ struct local_path_to_local_path_transferor : public transferor { bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) const override final; + + __attribute__((deprecated)) std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp new file mode 100644 index 0000000..e0fa877 --- /dev/null +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -0,0 +1,147 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include +#include +#include + +#include "utils.hpp" +#include "logger.hpp" +#include "resources.hpp" +#include "auth.hpp" +#include "io/task-info.hpp" +#include "backends/posix-fs.hpp" +#include "hermes.hpp" +#include "rpcs.hpp" +#include "local-path-to-remote-resource.hpp" + + +namespace norns { +namespace io { + +local_path_to_remote_resource_transferor::local_path_to_remote_resource_transferor( + std::shared_ptr remote_endpoint) : + m_remote_endpoint(remote_endpoint) { } + +bool +local_path_to_remote_resource_transferor::validate( + const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) const { + + (void) src_info; + (void) dst_info; + + LOGGER_WARN("Validation not implemented"); + + return true; +} + +std::error_code +local_path_to_remote_resource_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] start_transfer: {} -> {}", + task_info->id(), d_src.canonical_path(), d_dst.to_string()); + + hermes::endpoint endp = m_remote_endpoint->lookup(d_dst.address()); + + try { + hermes::mapped_buffer input_data(d_src.canonical_path().string()); + + std::vector bufvec{ + hermes::mutable_buffer{(void*) input_data.data(), input_data.size()} + }; + + auto buffers = + m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); + + norns::rpc::remote_transfer::input args( + m_remote_endpoint->self_address(), + d_src.parent()->nsid(), + d_dst.parent()->nsid(), + static_cast(backend_type::posix_filesystem), + static_cast(data::resource_type::local_posix_path), + d_dst.name(), + buffers); + + auto start = std::chrono::steady_clock::now(); + + auto rpc = + m_remote_endpoint->post(endp, args); + + auto resp = rpc.get(); + + double usecs = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + + task_info->record_transfer(input_data.size(), usecs); + + LOGGER_DEBUG("Remote request completed with retval {} " + "({} bytes, {} usecs)", + resp.at(0).retval(), input_data.size(), usecs); + + return std::make_error_code(static_cast(0)); + } + catch(const std::exception& ex) { + LOGGER_ERROR(ex.what()); + return std::make_error_code(static_cast(-1)); + } +} + +std::error_code +local_path_to_remote_resource_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + + (void) auth; + (void) task_info; + (void) src_rinfo; + (void) dst_rinfo; + + return std::make_error_code(static_cast(0)); +} + +std::string +local_path_to_remote_resource_transferor::to_string() const { + return "transferor[local_path => remote_resource]"; +} + +} // namespace io +} // namespace norns diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp new file mode 100644 index 0000000..852e173 --- /dev/null +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -0,0 +1,98 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_LOCAL_PATH_TO_REMOTE_RESOURCE_TX__ +#define __IO_LOCAL_PATH_TO_REMOTE_RESOURCE_TX__ + +#include +#include +#include "transferor.hpp" + +namespace hermes { +class async_engine; + +template class request; + +} // namespace hermes + +namespace norns { + +// forward declarations +namespace auth { +struct credentials; +} + +namespace data { +struct resource_info; +struct resource; +} + +namespace rpc { +struct remote_transfer; +} + +namespace io { + +struct local_path_to_remote_resource_transferor : public transferor { + + local_path_to_remote_resource_transferor( + std::shared_ptr remote_endpoint); + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::string to_string() const override final; + +private: + + void + do_accept(hermes::request&& req); + + std::shared_ptr m_remote_endpoint; + +}; + +} // namespace io +} // namespace norns + +#endif /* __LOCAL_PATH_TO_REMOTE_RESOURCE_TX__ */ diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index f3f1a6f..4ac9cbb 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -71,6 +71,23 @@ local_path_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::error_code +local_path_to_shared_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + LOGGER_WARN("Transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + std::string local_path_to_shared_path_transferor::to_string() const { return "transferor[local_path => shared_path]"; diff --git a/src/io/transferors/local-path-to-shared-path.hpp b/src/io/transferors/local-path-to-shared-path.hpp index 40eb978..18329b8 100644 --- a/src/io/transferors/local-path-to-shared-path.hpp +++ b/src/io/transferors/local-path-to-shared-path.hpp @@ -53,6 +53,14 @@ struct local_path_to_shared_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index 5c4edf3..f42b189 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -183,6 +183,53 @@ memory_region_to_local_path_transferor::transfer( d_src.size(), d_dst.canonical_path()); } +std::error_code +memory_region_to_local_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + + (void) task_info; + std::error_code ec; + const auto src_backend = task_info->src_backend(); + const auto dst_backend = task_info->dst_backend(); + + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + LOGGER_ERROR("[{}] Could not access input resource '{}': {}", + task_info->id(), + src_rinfo->to_string(), + ec.message()); + return ec; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + LOGGER_ERROR("[{}] Could not create output resource '{}': {}", + task_info->id(), + src_rinfo->to_string(), + ec.message()); + return ec; + } + + const auto& d_src = reinterpret_cast(*src); + const auto& d_dst = reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] transfer: [{} {}+{}] -> {}", + task_info->id(), + auth.pid(), + utils::n2hexstr(d_src.address()), + d_src.size(), + d_dst.canonical_path()); + + return ::copy_memory_region(task_info, auth.pid(), + reinterpret_cast(d_src.address()), + d_src.size(), d_dst.canonical_path()); +} + std::string memory_region_to_local_path_transferor::to_string() const { return "transferor[memory_region => local_path]"; diff --git a/src/io/transferors/memory-to-local-path.hpp b/src/io/transferors/memory-to-local-path.hpp index 4102739..c82799a 100644 --- a/src/io/transferors/memory-to-local-path.hpp +++ b/src/io/transferors/memory-to-local-path.hpp @@ -53,6 +53,14 @@ struct memory_region_to_local_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-remote-path.cpp b/src/io/transferors/memory-to-remote-path.cpp index 5135c07..da815f3 100644 --- a/src/io/transferors/memory-to-remote-path.cpp +++ b/src/io/transferors/memory-to-remote-path.cpp @@ -70,6 +70,23 @@ memory_region_to_remote_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::error_code +memory_region_to_remote_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + + (void) auth; + (void) task_info; + (void) src_rinfo; + (void) dst_rinfo; + + LOGGER_WARN("Transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + std::string memory_region_to_remote_path_transferor::to_string() const { return "transferor[memory_region => remote_path]"; diff --git a/src/io/transferors/memory-to-remote-path.hpp b/src/io/transferors/memory-to-remote-path.hpp index 393495d..e35efe6 100644 --- a/src/io/transferors/memory-to-remote-path.hpp +++ b/src/io/transferors/memory-to-remote-path.hpp @@ -53,6 +53,14 @@ struct memory_region_to_remote_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index c573bb2..43502c2 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -70,6 +70,23 @@ memory_region_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::error_code +memory_region_to_shared_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + + (void) auth; + (void) task_info; + (void) src_rinfo; + (void) dst_rinfo; + + LOGGER_WARN("Transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + std::string memory_region_to_shared_path_transferor::to_string() const { return "transferor[memory_region => shared_path]"; diff --git a/src/io/transferors/memory-to-shared-path.hpp b/src/io/transferors/memory-to-shared-path.hpp index 25d8e4e..d39cb9e 100644 --- a/src/io/transferors/memory-to-shared-path.hpp +++ b/src/io/transferors/memory-to-shared-path.hpp @@ -53,6 +53,14 @@ struct memory_region_to_shared_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp new file mode 100644 index 0000000..dcb02c8 --- /dev/null +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -0,0 +1,294 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "utils.hpp" +#include "logger.hpp" +#include "resources.hpp" +#include "auth.hpp" +#include "io/task-info.hpp" +#include "hermes.hpp" +#include "rpcs.hpp" +#include "remote-resource-to-local-path.hpp" + +namespace { + +std::tuple> +create_file(const bfs::path& filename, + std::size_t size) { + + int out_fd = + ::open(filename.c_str(), O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); + + if(out_fd == -1) { + auto ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + + // preallocate output file + if(::fallocate(out_fd, 0, 0, size) == -1) { + // filesystem doesn't support fallocate(), fallback to truncate() + if(errno == EOPNOTSUPP) { + if(::ftruncate(out_fd, size) != 0) { + auto ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + } + auto ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + +retry_close: + if(close(out_fd) == -1) { + if(errno == EINTR) { + goto retry_close; + } + + auto ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + + std::shared_ptr output_data; + + try { + output_data = + std::make_shared( + filename.string(), hermes::access_mode::write_only); + } + catch(const std::exception& ex) { + LOGGER_ERROR(ex.what()); + + auto ec = std::make_error_code(static_cast(-1)); + return std::make_tuple(ec, output_data); + } + + auto ec = std::make_error_code(static_cast(0)); + return std::make_tuple(ec, output_data); +} + +} // anonymous namespace + +namespace norns { +namespace io { + +remote_resource_to_local_path_transferor::remote_resource_to_local_path_transferor( + std::shared_ptr remote_endpoint) : + m_remote_endpoint(remote_endpoint) { } + +bool +remote_resource_to_local_path_transferor::validate( + const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) const { + + (void) src_info; + (void) dst_info; + + LOGGER_WARN("Validation not implemented"); + + return true; +} + +std::error_code +remote_resource_to_local_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) src; + (void) dst; + + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + const auto ctx = boost::any_cast< + std::shared_ptr< + hermes::request>>(task_info->context()); + auto req = std::move(*ctx); + + LOGGER_DEBUG("[{}] accept_transfer: {} -> {}", task_info->id(), + d_src.to_string(), d_dst.canonical_path()); + + LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); + + hermes::exposed_memory remote_buffers = d_src.buffers(); + + LOGGER_DEBUG("remote_buffers{{count={}, total_size={}}}", + remote_buffers.count(), + remote_buffers.size()); + + assert(remote_buffers.count() == 1); + + LOGGER_DEBUG("creating local resource: {}", d_dst.canonical_path()); + + std::error_code ec; + std::shared_ptr output_data; + + std::tie(ec, output_data) = + ::create_file(d_dst.canonical_path(), remote_buffers.size()); + + if(ec) { + if(req.requires_response()) { + m_remote_endpoint->respond( + std::move(req), static_cast(urd_error::snafu));//XXX + } + return ec; + } + + // let's prepare some local buffers + + std::vector bufseq{ + hermes::mutable_buffer{output_data->data(), output_data->size()} + }; + + hermes::exposed_memory local_buffers = + m_remote_endpoint->expose(bufseq, hermes::access_mode::write_only); + + LOGGER_DEBUG("pulling remote data into {}", d_dst.canonical_path()); + + // N.B. IMPORTANT: we NEED to capture output_data by value here so that + // the mapped_buffer doesn't get released before completion_callback() + // is called. + const auto completion_callback = [this, output_data]( + hermes::request&& req) { + + //TODO: hermes offers no way to check for an error yet + LOGGER_DEBUG("pull succeeded"); + + if(req.requires_response()) { + m_remote_endpoint->respond( + std::move(req), static_cast(urd_error::success)); + } + }; + + m_remote_endpoint->async_pull(remote_buffers, + local_buffers, + std::move(req), + completion_callback); + + return std::make_error_code(static_cast(0)); +} + +std::error_code +remote_resource_to_local_path_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src_rinfo, + const std::shared_ptr& dst_rinfo) const { + +#if 0 + (void) auth; + const auto src_backend = task_info->src_backend(); + const auto dst_backend = task_info->dst_backend(); + + std::error_code ec; + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + LOGGER_ERROR("[{}] Could not access input resource '{}': {}", + task_info->id(), + src_rinfo->to_string(), + ec.message()); + return ec; + } + + const auto& d_src = + reinterpret_cast(*src); + + const auto& d_dst = + reinterpret_cast(*dst_rinfo); + + LOGGER_DEBUG("[{}] transfer: {} -> {}://{}@{}", + task_info->id(), + d_src.canonical_path(), + d_dst.nsid(), + d_dst.datapath(), + d_dst.hostname()); + + LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", + dst_rinfo->to_string(), + "xxx"); + + hermes::endpoint endp = m_remote_endpoint->lookup(d_dst.hostname()); + + hermes::mapped_buffer b(d_src.canonical_path().string()); + + std::vector bufvec{ + hermes::mutable_buffer{b.data(), b.size()} + }; + + auto buffers = + m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); + + rpc::remote_transfer::input + args(d_dst.nsid(), d_dst.datapath(), buffers); + + auto rpc = + m_remote_endpoint->post(endp, args); +#endif + +#if 0 + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] transfer: {} -> {}", task_info->id(), + d_src.canonical_path(), d_dst.to_string()); + + LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); + + hermes::endpoint endp = m_remote_endpoint->lookup("127.0.0.1:42000"); + + char* data = new char[42]; + + std::vector bufvec{ + hermes::mutable_buffer{(void*) data, 42} + }; + + auto buffers = + m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); + + norns::rpc::remote_transfer::input args("/tmp/foo", buffers); + + auto rpc = + m_remote_endpoint->post(endp, args); + + +#endif + return std::make_error_code(static_cast(0)); +} + +std::string +remote_resource_to_local_path_transferor::to_string() const { + return "transferor[remote_resource => local_path]"; +} + +} // namespace io +} // namespace norns diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp new file mode 100644 index 0000000..34c2d6f --- /dev/null +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -0,0 +1,98 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_REMOTE_RESOURCE_TO_LOCAL_PATH_TX__ +#define __IO_REMOTE_RESOURCE_TO_LOCAL_PATH_TX__ + +#include + +#include "transferor.hpp" + +namespace hermes { +class async_engine; + +template class request; + +} // namespace hermes + +namespace norns { + +// forward declarations +namespace auth { +struct credentials; +} + +namespace data { +struct resource_info; +struct resource; +} + +namespace rpc { +struct remote_transfer; +} + +namespace io { + +struct remote_resource_to_local_path_transferor : public transferor { + + remote_resource_to_local_path_transferor( + std::shared_ptr remote_endpoint); + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::string + to_string() const override final; + +private: + + void + do_accept(hermes::request&& req); + + std::shared_ptr m_remote_endpoint; +}; + +} // namespace io +} // namespace norns + +#endif // __IO_REMOTE_RESOURCE_TO_LOCAL_PATH_TX__ diff --git a/src/io/transferors/transferor.hpp b/src/io/transferors/transferor.hpp index 4fd98f3..3d11fed 100644 --- a/src/io/transferors/transferor.hpp +++ b/src/io/transferors/transferor.hpp @@ -57,6 +57,13 @@ struct transferor { const std::shared_ptr& src, const std::shared_ptr& dst) const = 0; + virtual std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const = 0; + + virtual std::string to_string() const = 0; }; diff --git a/src/logger.hpp b/src/logger.hpp index 870d6ce..9a6100f 100644 --- a/src/logger.hpp +++ b/src/logger.hpp @@ -31,9 +31,17 @@ #include #include +#include #include #include +#if FMT_VERSION < 50000 +namespace fmt { +template +inline const void *ptr(const T *p) { return p; } +} // namespace fmt +#endif // FMT_VERSION + namespace bfs = boost::filesystem; class logger { @@ -123,30 +131,65 @@ public: return global_logger(); } - // some macros to make it more convenient to use the global logger +// some macros to make it more convenient to use the global logger +#define LOGGER_INFO(...) \ +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->info(__VA_ARGS__); \ + } \ +} while(0); -#define LOGGER_INFO(...) \ - logger::get_global_logger()->info(__VA_ARGS__) #ifdef __LOGGER_ENABLE_DEBUG__ + #define LOGGER_DEBUG(...) \ - logger::get_global_logger()->debug(__VA_ARGS__) +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->debug(__VA_ARGS__); \ + } \ +} while(0); + +#define LOGGER_FLUSH() \ +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->flush(); \ + } \ +} while(0); + #else -#define LOGGER_DEBUG(...) \ - do {} while(0) -#endif + +#define LOGGER_DEBUG(...) do {} while(0); +#define LOGGER_FLUSH() do {} while(0); + +#endif // __LOGGER_ENABLE_DEBUG__ #define LOGGER_WARN(...) \ - logger::get_global_logger()->warn(__VA_ARGS__) +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->warn(__VA_ARGS__); \ + } \ +} while(0); #define LOGGER_ERROR(...) \ - logger::get_global_logger()->error(__VA_ARGS__) +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->error(__VA_ARGS__); \ + } \ +} while(0); #define LOGGER_ERRNO(...) \ - logger::get_global_logger()->error_errno(__VA_ARGS__) +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->error_errno(__VA_ARGS__); \ + } \ +} while(0); #define LOGGER_CRITICAL(...) \ - logger::get_global_logger()->critical(__VA_ARGS__) +do { \ + if(logger::get_global_logger()) { \ + logger::get_global_logger()->critical(__VA_ARGS__); \ + } \ +} while(0); // the following member functions can be used to interact // with a specific logger instance @@ -251,4 +294,49 @@ private: std::string m_type; }; +// override the logging macros of the hermes library +// and replace it with ours, which use the same fmt definitions +// to format message strings +#ifdef HERMES_INFO +#undef HERMES_INFO +#endif +#define HERMES_INFO(...) LOGGER_INFO(__VA_ARGS__) + +#ifdef HERMES_WARNING +#undef HERMES_WARNING +#endif +#define HERMES_WARNING(...) LOGGER_WARN(__VA_ARGS__) + +#ifdef HERMES_DEBUG +#undef HERMES_DEBUG +#endif +#define HERMES_DEBUG(...) LOGGER_DEBUG(__VA_ARGS__) + +#ifdef HERMES_DEBUG2 +#undef HERMES_DEBUG2 +#endif +#define HERMES_DEBUG2(...) LOGGER_DEBUG(__VA_ARGS__) + +#ifdef HERMES_DEBUG3 +#undef HERMES_DEBUG3 +#endif +#define HERMES_DEBUG3(...) LOGGER_DEBUG(__VA_ARGS__) + +#ifdef HERMES_DEBUG4 +#undef HERMES_DEBUG4 +#endif +#define HERMES_DEBUG4(...) // LOGGER_DEBUG(__VA_ARGS__) + +#ifdef HERMES_ERROR +#undef HERMES_ERROR +#endif +#define HERMES_ERROR(...) LOGGER_ERROR(__VA_ARGS__) + +#ifdef HERMES_FATAL +#undef HERMES_FATAL +#endif +#define HERMES_FATAL(...) LOGGER_CRITICAL(__VA_ARGS__) + +#include + #endif /* __LOGGER_HPP__ */ diff --git a/src/namespaces/namespace-manager.hpp b/src/namespaces/namespace-manager.hpp index e29b88d..a35c684 100644 --- a/src/namespaces/namespace-manager.hpp +++ b/src/namespaces/namespace-manager.hpp @@ -94,7 +94,7 @@ struct namespace_manager { if(is_remote) { return static_cast>( - std::make_shared()); + std::make_shared(nsid)); } if(m_namespaces.count(nsid) == 0) { diff --git a/src/resources.hpp b/src/resources.hpp index 03e6b39..b280507 100644 --- a/src/resources.hpp +++ b/src/resources.hpp @@ -35,6 +35,7 @@ #include "resources/local_posix_path/local-path.hpp" #include "resources/memory_buffer/memory-buffer.hpp" #include "resources/shared_posix_path/shared-path.hpp" +#include "resources/remote_resource/remote-resource.hpp" #include "resources/remote_posix_path/remote-path.hpp" #endif /* __RESOURCES_HPP__ */ diff --git a/src/resources/remote_posix_path/detail/remote-path-impl.cpp b/src/resources/remote_posix_path/detail/remote-path-impl.cpp index 7d71cc7..67b251c 100644 --- a/src/resources/remote_posix_path/detail/remote-path-impl.cpp +++ b/src/resources/remote_posix_path/detail/remote-path-impl.cpp @@ -49,7 +49,7 @@ remote_path_resource::resource_impl( std::move(parent))) {} std::string remote_path_resource::name() const { - return "PENDING"; + return "PENDING: " + m_parent->to_string(); } resource_type remote_path_resource::type() const { diff --git a/src/resources/remote_posix_path/detail/remote-path-info.cpp b/src/resources/remote_posix_path/detail/remote-path-info.cpp index 2f0b465..46faf50 100644 --- a/src/resources/remote_posix_path/detail/remote-path-info.cpp +++ b/src/resources/remote_posix_path/detail/remote-path-info.cpp @@ -59,6 +59,11 @@ std::string remote_path_info::to_string() const { return "REMOTE_PATH[\"" + m_hostname + "\", \"" + m_nsid + "\", \"" + m_datapath + "\"]"; } +std::string +remote_path_info::hostname() const { + return m_hostname; +} + std::string remote_path_info::datapath() const { return m_datapath; } diff --git a/src/resources/remote_resource/detail/remote-resource-impl.cpp b/src/resources/remote_resource/detail/remote-resource-impl.cpp new file mode 100644 index 0000000..5e93041 --- /dev/null +++ b/src/resources/remote_resource/detail/remote-resource-impl.cpp @@ -0,0 +1,102 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include + +#include "common.hpp" +#include "resource-type.hpp" +#include "resource.hpp" +#include "remote-resource-info.hpp" +#include "remote-resource-impl.hpp" +#include "backends/remote-backend.hpp" +#include "logger.hpp" + +namespace norns { +namespace data { +namespace detail { + +// local alias for convenience +using remote_resource = resource_impl; + +remote_resource::resource_impl( + const std::shared_ptr parent, + const std::shared_ptr rinfo) : + m_name(rinfo->m_name), + m_address(rinfo->m_address), + m_nsid(rinfo->m_nsid), + m_buffers(rinfo->m_buffers), + m_is_collection(rinfo->m_buffers.count() > 1), + m_parent(std::static_pointer_cast(std::move(parent))) { } + +std::string +remote_resource::name() const { + return m_name; +} + +resource_type +remote_resource::type() const { + return resource_type::remote_resource; +} + +bool +remote_resource::is_collection() const { + return m_is_collection; +} + +const std::shared_ptr +remote_resource::parent() const { + return std::static_pointer_cast(m_parent); +} + +std::string +remote_resource::to_string() const { + return "REMOTE_RESOURCE[" + m_nsid + "@" + m_address + ":" + m_name + "]"; +} + +std::string +remote_resource::address() const { + return m_address; +} + +std::string +remote_resource::nsid() const { + return m_nsid; +} + +bool +remote_resource::has_buffer() const { + return m_buffers.count() != 0; +} + +hermes::exposed_memory +remote_resource::buffers() const { + return m_buffers; +} + +} // namespace detail +} // namespace data +} // namespace norns diff --git a/src/resources/remote_resource/detail/remote-resource-impl.hpp b/src/resources/remote_resource/detail/remote-resource-impl.hpp new file mode 100644 index 0000000..86201f1 --- /dev/null +++ b/src/resources/remote_resource/detail/remote-resource-impl.hpp @@ -0,0 +1,88 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __REMOTE_RESOURCE_IMPL_HPP__ +#define __REMOTE_RESOURCE_IMPL_HPP__ + +#include +#include +#include +#include +#include + +namespace norns { + +// forward declarations +namespace storage { +namespace detail { +class remote_backend; +} +} + +namespace data { + +enum class resource_type; + +namespace detail { + +template <> +struct resource_impl : public resource { + + resource_impl(const std::shared_ptr parent, + const std::shared_ptr rinfo); + + std::string name() const override final; + resource_type type() const override final; + bool is_collection() const override final; + const std::shared_ptr parent() const override final; + std::string to_string() const override final; + + std::string + address() const; + + std::string + nsid() const; + + bool + has_buffer() const; + + hermes::exposed_memory + buffers() const; + + const std::string m_name; + const std::string m_address; + const std::string m_nsid; + const hermes::exposed_memory m_buffers; + const bool m_is_collection; + const std::shared_ptr m_parent; +}; + +} // namespace detail +} // namespace data +} // namespace norns + +#endif /* __REMOTE_RESOURCE_IMPL_HPP__ */ diff --git a/src/io/transferors/local-path-to-remote-path.cpp b/src/resources/remote_resource/detail/remote-resource-info.cpp similarity index 56% rename from src/io/transferors/local-path-to-remote-path.cpp rename to src/resources/remote_resource/detail/remote-resource-info.cpp index 9a05aeb..d85662d 100644 --- a/src/io/transferors/local-path-to-remote-path.cpp +++ b/src/resources/remote_resource/detail/remote-resource-info.cpp @@ -1,5 +1,5 @@ /************************************************************************* - * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * * Centro Nacional de Supercomputacion * * All rights reserved. * * * @@ -25,64 +25,75 @@ * . * *************************************************************************/ -#include -#include -#include -#include - -#include "utils.hpp" -#include "logger.hpp" -#include "resources.hpp" -#include "auth.hpp" -#include "io/task-info.hpp" -#include "backends/posix-fs.hpp" -#include "local-path-to-remote-path.hpp" +#include "resource-type.hpp" +#include "resource-info.hpp" +#include "remote-resource-info.hpp" namespace norns { -namespace io { +namespace data { +namespace detail { -bool -local_path_to_remote_path_transferor::validate( - const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const { +/*! Remote path data */ +remote_resource_info::remote_resource_info( + std::string address, + std::string nsid, + std::string name) : + m_address(address), + m_nsid(nsid), + m_name(name) { } - (void) src_info; - (void) dst_info; +remote_resource_info::remote_resource_info( + std::string address, + std::string nsid, + std::string name, + hermes::exposed_memory buffers) : + m_address(address), + m_nsid(nsid), + m_name(name), + m_buffers(buffers) { } - LOGGER_WARN("Validation not implemented"); +remote_resource_info::~remote_resource_info() { } - return true; +resource_type +remote_resource_info::type() const { + return resource_type::remote_resource; } -std::error_code -local_path_to_remote_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const { +std::string +remote_resource_info::address() const { + return m_address; +} - (void) auth; - (void) task_info; - (void) src; - (void) dst; +std::string +remote_resource_info::nsid() const { + return m_nsid; +} - const auto& d_src = - reinterpret_cast(*src); - const auto& d_dst = - reinterpret_cast(*dst); +bool +remote_resource_info::is_remote() const { + return true; +} - LOGGER_DEBUG("[{}] transfer: {} -> {}", task_info->id(), - d_src.canonical_path(), d_dst.to_string()); +std::string +remote_resource_info::to_string() const { + return "REMOTE_RESOURCE[" + m_nsid + "@" + m_address + ":" + m_name + "]"; +} - LOGGER_WARN("Transfer not implemented"); +std::string +remote_resource_info::name() const { + return m_name; +} - return std::make_error_code(static_cast(0)); +bool +remote_resource_info::has_buffers() const { + return m_buffers.count() != 0; } -std::string -local_path_to_remote_path_transferor::to_string() const { - return "transferor[local_path => remote_path]"; +hermes::exposed_memory +remote_resource_info::buffers() const { + return m_buffers; } -} // namespace io +} // namespace detail +} // namespace data } // namespace norns diff --git a/src/resources/remote_resource/detail/remote-resource-info.hpp b/src/resources/remote_resource/detail/remote-resource-info.hpp new file mode 100644 index 0000000..78359f4 --- /dev/null +++ b/src/resources/remote_resource/detail/remote-resource-info.hpp @@ -0,0 +1,82 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __REMOTE_RESOURCE_INFO_HPP__ +#define __REMOTE_RESOURCE_INFO_HPP__ + +#include +#include +#include "resource-info.hpp" +#include "rpcs.hpp" + +namespace norns { +namespace data { + +enum class resource_type; + +namespace detail { + +/*! Local filesystem path data */ +struct remote_resource_info : public resource_info { + + remote_resource_info(std::string address, + std::string nsid, + std::string name); + + remote_resource_info(std::string address, + std::string nsid, + std::string name, + hermes::exposed_memory buffers); + ~remote_resource_info(); + resource_type type() const override final; + std::string nsid() const override final; + bool is_remote() const override final; + std::string to_string() const override final; + + std::string + address() const; + + std::string + name() const; + + bool + has_buffers() const; + + hermes::exposed_memory + buffers() const; + + std::string m_address; + std::string m_nsid; + std::string m_name; + hermes::exposed_memory m_buffers; +}; + +} // namespace detail +} // namespace data +} // namespace norns + +#endif // __REMOTE_RESOURCE_INFO_HPP__ diff --git a/src/resources/remote_resource/remote-resource.hpp b/src/resources/remote_resource/remote-resource.hpp new file mode 100644 index 0000000..8dceb1e --- /dev/null +++ b/src/resources/remote_resource/remote-resource.hpp @@ -0,0 +1,47 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __REMOTE_RESOURCE_HPP__ +#define __REMOTE_RESOURCE_HPP__ + +#include "resource-type.hpp" +#include "resource.hpp" + +#include "detail/remote-resource-info.hpp" +#include "detail/remote-resource-impl.hpp" + +namespace norns { +namespace data { + +using remote_resource_info = detail::remote_resource_info; +using remote_resource = + detail::resource_impl; + +} // namespace data +} // namespace norns + +#endif // __REMOTE_RESOURCE_HPP__ diff --git a/src/resources/resource-type.hpp b/src/resources/resource-type.hpp index ec9aa9f..9d88a72 100644 --- a/src/resources/resource-type.hpp +++ b/src/resources/resource-type.hpp @@ -39,6 +39,7 @@ enum class resource_type { local_posix_path, shared_posix_path, remote_posix_path, + remote_resource, ignorable }; @@ -56,6 +57,8 @@ static inline std::string to_string(data::resource_type type) { return "SHARED_PATH"; case data::resource_type::remote_posix_path: return "REMOTE_PATH"; + case data::resource_type::remote_resource: + return "REMOTE_RESOURCE"; default: return "UNKNOWN_RESOURCE_TYPE"; } diff --git a/src/rpcs.cpp b/src/rpcs.cpp new file mode 100644 index 0000000..664a7c4 --- /dev/null +++ b/src/rpcs.cpp @@ -0,0 +1,15 @@ +//#include "common.hpp" +#include +#include "rpcs.hpp" + +namespace hermes { namespace detail { + +//============================================================================== +// register request types so that they can be used by users and the engine +// +void +register_user_request_types() { + (void) registered_requests().add(); +} + +}} // namespace hermes::detail diff --git a/src/rpcs.hpp b/src/rpcs.hpp new file mode 100644 index 0000000..c2d9dd5 --- /dev/null +++ b/src/rpcs.hpp @@ -0,0 +1,216 @@ +#ifndef __NORNS_IO_TRANSFERORS_RPCS_HPP__ +#define __NORNS_IO_TRANSFERORS_RPCS_HPP__ + +// C includes +#include +#include +#include + +// C++ includes +#include + +// hermes includes +#include + +#ifndef HG_GEN_PROC_NAME +#define HG_GEN_PROC_NAME(struct_type_name) \ + hermes::detail::hg_proc_ ## struct_type_name +#endif + +// forward declarations +namespace hermes { namespace detail { + +template +hg_return_t post_to_mercury(ExecutionContext* ctx); + +}} // namespace hermes::detail + +//============================================================================== +// definitions for norns::rpc::remote_transfer +namespace hermes { namespace detail { + +// Generate Mercury types and serialization functions (field names match +// those defined by remote_transfer::input and remote_transfer::output). These +// definitions are internal and should not be used directly. Classes +// remote_transfer::input and remote_transfer::output are provided for public use. +MERCURY_GEN_PROC(remote_transfer_in_t, + ((hg_const_string_t) (address)) + ((hg_const_string_t) (in_nsid)) + ((hg_const_string_t) (out_nsid)) + ((uint32_t) (backend_type)) + ((uint32_t) (resource_type)) + ((hg_const_string_t) (resource_name)) + ((hg_bulk_t) (buffers))) + +MERCURY_GEN_PROC(remote_transfer_out_t, + ((int32_t) (retval))) + +}} // namespace hermes::detail + + +namespace norns { +namespace rpc { + +struct remote_transfer { + + // forward declarations of public input/output types for this RPC + class input; + class output; + + // traits used so that the engine knows what to do with the RPC + using self_type = remote_transfer; + using handle_type = hermes::rpc_handle; + using input_type = input; + using output_type = output; + using mercury_input_type = hermes::detail::remote_transfer_in_t; + using mercury_output_type = hermes::detail::remote_transfer_out_t; + + // RPC public identifier + constexpr static const uint16_t public_id = 43; + + // RPC internal Mercury identifier + constexpr static const uint16_t mercury_id = public_id; + + // RPC name + constexpr static const auto name = "remote_transfer"; + + // requires response? + constexpr static const auto requires_response = true; + + // Mercury callback to serialize input arguments + constexpr static const auto mercury_in_proc_cb = + HG_GEN_PROC_NAME(remote_transfer_in_t); + + // Mercury callback to serialize output arguments + constexpr static const auto mercury_out_proc_cb = + HG_GEN_PROC_NAME(remote_transfer_out_t); + + class input { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + input(const std::string& address, + const std::string& in_nsid, + const std::string& out_nsid, + uint32_t backend_type, + uint32_t resource_type, + const std::string& resource_name, + const hermes::exposed_memory& buffers) : + m_address(address), + m_in_nsid(in_nsid), + m_out_nsid(out_nsid), + m_backend_type(backend_type), + m_resource_type(resource_type), + m_resource_name(resource_name), + m_buffers(buffers) { } + + std::string + address() const { + return m_address; + } + + std::string + in_nsid() const { + return m_in_nsid; + } + + std::string + out_nsid() const { + return m_out_nsid; + } + + uint32_t + backend_type() const { + return m_backend_type; + } + + uint32_t + resource_type() const { + return m_resource_type; + } + + std::string + resource_name() const { + return m_resource_name; + } + + hermes::exposed_memory + buffers() const { + return m_buffers; + } + +//TODO: make private + explicit + input(const hermes::detail::remote_transfer_in_t& other) : + m_address(other.address), + m_in_nsid(other.in_nsid), + m_out_nsid(other.out_nsid), + m_backend_type(other.backend_type), + m_resource_type(other.resource_type), + m_resource_name(other.resource_name), + m_buffers(other.buffers) { } + + explicit + operator hermes::detail::remote_transfer_in_t() { + return {m_address.c_str(), + m_in_nsid.c_str(), + m_out_nsid.c_str(), + m_backend_type, + m_resource_type, + m_resource_name.c_str(), + hg_bulk_t(m_buffers)}; + } + + + private: + std::string m_address; + std::string m_in_nsid; + std::string m_out_nsid; + uint32_t m_backend_type; + uint32_t m_resource_type; + std::string m_resource_name; + hermes::exposed_memory m_buffers; + }; + + class output { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + output(int32_t retval) : + m_retval(retval) { } + + int32_t + retval() const { + return m_retval; + } + + void + set_retval(int32_t retval) { + m_retval = retval; + } + + explicit + output(const hermes::detail::remote_transfer_out_t& out) { + m_retval = out.retval; + } + + explicit + operator hermes::detail::remote_transfer_out_t() { + return {m_retval}; + } + + private: + int32_t m_retval; + }; +}; + +} // namespace rpc +} // namespace norns + +#undef HG_GEN_PROC_NAME + +#endif // __NORNS_IO_TRANSFERORS_RPCS_HPP__ diff --git a/src/urd.cpp b/src/urd.cpp index e608360..ae871f3 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -54,6 +54,8 @@ #include "io.hpp" #include "namespaces.hpp" #include "fmt.hpp" +#include "hermes.hpp" +#include "rpcs.hpp" #include "urd.hpp" namespace norns { @@ -204,10 +206,13 @@ urd_error urd::validate_iotask_args(iotask_type type, // handlers for user requests /////////////////////////////////////////////////////////////////////////////// -response_ptr urd::iotask_create_handler(const request_ptr base_request) { +response_ptr +urd::iotask_create_handler(const request_ptr base_request) { // downcast the generic request to the concrete implementation - auto request = utils::static_unique_ptr_cast(std::move(base_request)); + auto request = + utils::static_unique_ptr_cast( + std::move(base_request)); const auto type = request->get<0>(); const auto src_rinfo = request->get<1>(); @@ -219,6 +224,7 @@ response_ptr urd::iotask_create_handler(const request_ptr base_request) { std::vector> rinfo_ptrs; boost::optional tid; boost::optional auth; + boost::optional t; response_ptr resp; urd_error rv = urd_error::success; @@ -235,6 +241,11 @@ response_ptr urd::iotask_create_handler(const request_ptr base_request) { goto log_and_return; } + if(src_rinfo->is_remote()) { + rv = urd_error::not_supported; + goto log_and_return; + } + for(const auto& rinfo : {src_rinfo, dst_rinfo}) { if(rinfo) { nsids.push_back(rinfo->nsid()); @@ -277,7 +288,35 @@ response_ptr urd::iotask_create_handler(const request_ptr base_request) { } #endif - std::tie(rv, tid) = m_task_mgr->create_task(type, *auth, backend_ptrs, rinfo_ptrs); + + //FIXME: use appropriate args for each task rather than a vector of nullptrs + switch(type) { + case iotask_type::move: + case iotask_type::copy: + std::tie(rv, t) = + m_task_mgr->create_local_initiated_task(type, *auth, backend_ptrs, rinfo_ptrs); + break; + case iotask_type::remove: + std::tie(rv, t) = + m_task_mgr->create_local_initiated_task(type, *auth, backend_ptrs, rinfo_ptrs); + break; + case iotask_type::noop: + std::tie(rv, t) = + m_task_mgr->create_local_initiated_task(type, *auth, backend_ptrs, rinfo_ptrs); + break; + default: + rv = urd_error::bad_args; + goto log_and_return; + } + + if(rv == urd_error::success) { + tid = t->id(); + + // enqueue task so that it's eventually run by a worker thread + m_task_mgr->enqueue_task(std::move(*t)); + } + +// std::tie(rv, tid) = m_task_mgr->create_task(type, *auth, backend_ptrs, rinfo_ptrs); log_and_return: resp = std::make_unique(tid.get_value_or(0)); @@ -417,7 +456,7 @@ response_ptr urd::process_add_handler(const request_ptr base_request) { auto request = utils::static_unique_ptr_cast(std::move(base_request)); uint32_t jobid = request->get<0>(); - pid_t uid = request->get<1>(); + //pid_t uid = request->get<1>(); gid_t gid = request->get<2>(); pid_t pid = request->get<3>(); @@ -446,7 +485,7 @@ response_ptr urd::process_remove_handler(const request_ptr base_request) { auto request = utils::static_unique_ptr_cast(std::move(base_request)); uint32_t jobid = request->get<0>(); - pid_t uid = request->get<1>(); + //pid_t uid = request->get<1>(); gid_t gid = request->get<2>(); pid_t pid = request->get<3>(); @@ -495,7 +534,7 @@ urd_error urd::create_namespace(const std::string& nsid, backend_type type, } if(auto bptr = storage::backend_factory::create_from( - type, track, mount, quota)) { + type, nsid, track, mount, quota)) { m_namespace_mgr->add(nsid, bptr); return urd_error::success; } @@ -625,6 +664,95 @@ response_ptr urd::unknown_request_handler(const request_ptr /*base_request*/) { return resp; } +// N.B. This function is called by the progress thread internal to +// m_network_endpoint rather than by the main execution thread +void +urd::remote_transfer_handler(hermes::request&& req) { + + const auto args = req.args(); + + LOGGER_WARN("incoming rpc::remote_transfer(from: \"{}@{}:{}\", to: \"{}:{}\")", + args.in_nsid(), + args.address(), + args.resource_name(), + args.out_nsid(), + args.resource_name()); + + urd_error rv = urd_error::success; + boost::optional t; + std::shared_ptr src_backend; + boost::optional> dst_backend; + auto dst_rtype = static_cast(args.resource_type()); + auth::credentials auth; //XXX fake credentials for now + + const auto create_rinfo = + [&](const data::resource_type& rtype) -> + std::shared_ptr { + + switch(rtype) { + case data::resource_type::remote_resource: + return std::make_shared( + args.address(), args.in_nsid(), + args.resource_name(), args.buffers()); + case data::resource_type::local_posix_path: + case data::resource_type::shared_posix_path: + return std::make_shared( + args.out_nsid(), args.resource_name()); + default: + rv = urd_error::not_supported; + return {}; + } + }; + + auto src_rinfo = create_rinfo(data::resource_type::remote_resource); + auto dst_rinfo = create_rinfo(dst_rtype); + + if(m_is_paused) { + rv = urd_error::accept_paused; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), static_cast(rv)); + return; + } + + // TODO: actually retrieve and validate credentials, etc + + + src_backend = + std::make_shared(args.in_nsid()); + + { + boost::shared_lock lock(m_namespace_mgr_mutex); + dst_backend = m_namespace_mgr->find(args.out_nsid()); + } + + if(!dst_backend) { + rv = urd_error::no_such_namespace; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), static_cast(rv)); + return; + } + + LOGGER_DEBUG("nsid: {}, bptr: {}", args.in_nsid(), dst_backend); + + auto ctx = + std::make_shared>(std::move(req)); + + std::tie(rv, t) = + m_task_mgr->create_remote_initiated_task( + iotask_type::remote_transfer, auth, + ctx, src_backend, src_rinfo, + *dst_backend, dst_rinfo); + + if(rv == urd_error::success) { + // run the task + (*t)(); + + // rv = t.info().status() ?? + } + +} + + void urd::configure(const config::settings& settings) { m_settings = std::make_shared(settings); } @@ -687,7 +815,7 @@ void urd::init_event_handlers() { // create (but not start) the API listener // and register handlers for each request type try { - m_api_listener = std::make_unique(); + m_ipc_endpoint = std::make_unique(); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create the event listener. This should " @@ -716,7 +844,7 @@ void urd::init_event_handlers() { ::umask(S_IXUSR | S_IRWXG | S_IRWXO); // u=rw-, g=---, o=--- try { - m_api_listener->register_endpoint(m_settings->control_socket()); + m_ipc_endpoint->register_endpoint(m_settings->control_socket()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create control API socket: {}", e.what()); @@ -739,7 +867,7 @@ void urd::init_event_handlers() { ::umask(S_IXUSR | S_IXGRP | S_IXOTH); // u=rw-, g=rw-, o=rw- try { - m_api_listener->register_endpoint(m_settings->global_socket()); + m_ipc_endpoint->register_endpoint(m_settings->global_socket()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create user API socket: {}", e.what()); @@ -747,9 +875,31 @@ void urd::init_event_handlers() { exit(EXIT_FAILURE); } + // restore the umask + ::umask(old_mask); + + try { + + const std::string bind_address = + std::string("127.0.0.1:") + + std::to_string(m_settings->remote_port()); + + m_network_endpoint = + std::make_shared( + hermes::transport::ofi_tcp, + bind_address, + true); + } + catch(const std::exception& e) { + LOGGER_ERROR("Failed to create remote listener: {}", e.what()); + teardown(); + exit(EXIT_FAILURE); + } + +#if 0 // setup socket for remote connections try { - m_api_listener->register_endpoint(m_settings->remote_port()); + m_ipc_endpoint->register_endpoint(m_settings->remote_port()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create socket for remote connections: {}", @@ -757,74 +907,80 @@ void urd::init_event_handlers() { teardown(); exit(EXIT_FAILURE); } +#endif - // restore the umask - ::umask(old_mask); LOGGER_INFO(" * Installing message handlers..."); /* user-level functionalities */ - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::iotask_create, std::bind(&urd::iotask_create_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::iotask_status, std::bind(&urd::iotask_status_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::ping, std::bind(&urd::ping_handler, this, std::placeholders::_1)); /* admin-level functionalities */ - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::job_register, std::bind(&urd::job_register_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::job_update, std::bind(&urd::job_update_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::job_unregister, std::bind(&urd::job_remove_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::process_register, std::bind(&urd::process_add_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::process_unregister, std::bind(&urd::process_remove_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( - api::request_type::backend_register, - std::bind(&urd::namespace_register_handler, this, std::placeholders::_1)); + m_ipc_endpoint->register_callback( + api::request_type::backend_register, + std::bind(&urd::namespace_register_handler, this, + std::placeholders::_1)); -/* m_api_listener->register_callback( - api::request_type::backend_update, - std::bind(&urd::namespace_update_handler, this, std::placeholders::_1));*/ + /* m_ipc_endpoint->register_callback( + api::request_type::backend_update, + std::bind(&urd::namespace_update_handler, this, + std::placeholders::_1));*/ - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::backend_unregister, std::bind(&urd::namespace_remove_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::global_status, std::bind(&urd::global_status_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::command, std::bind(&urd::command_handler, this, std::placeholders::_1)); - m_api_listener->register_callback( + m_ipc_endpoint->register_callback( api::request_type::bad_request, std::bind(&urd::unknown_request_handler, this, std::placeholders::_1)); + /* remote event handlers */ + m_network_endpoint->register_handler( + std::bind(&urd::remote_transfer_handler, this, + std::placeholders::_1)); + // signal handlers must be installed AFTER daemonizing LOGGER_INFO(" * Installing signal handlers..."); - m_api_listener->set_signal_handler( + m_ipc_endpoint->set_signal_handler( std::bind(&urd::signal_handler, this, std::placeholders::_1), SIGHUP, SIGTERM, SIGINT); } @@ -874,9 +1030,10 @@ void urd::load_backend_plugins() { storage::backend_factory::get(). register_backend( backend_type::posix_filesystem, - [](bool track, const bfs::path& mount, uint32_t quota) { + [](const std::string& nsid, bool track, + const bfs::path& mount, uint32_t quota) { return std::shared_ptr( - new storage::posix_filesystem(track, mount, quota)); + new storage::posix_filesystem(nsid, track, mount, quota)); }); storage::backend_factory::get(). @@ -888,9 +1045,10 @@ void urd::load_backend_plugins() { storage::backend_factory::get(). register_backend( backend_type::nvml, - [](bool track, const bfs::path& mount, uint32_t quota) { + [](const std::string& nsid, bool track, + const bfs::path& mount, uint32_t quota) { return std::shared_ptr( - new storage::nvml_dax(track, mount, quota)); + new storage::nvml_dax(nsid, track, mount, quota)); }); storage::backend_factory::get(). @@ -900,9 +1058,10 @@ void urd::load_backend_plugins() { storage::backend_factory::get(). register_backend( backend_type::lustre, - [](bool track, const bfs::path& mount, uint32_t quota) { + [](const std::string& nsid, bool track, + const bfs::path& mount, uint32_t quota) { return std::shared_ptr( - new storage::lustre(track, mount, quota)); + new storage::lustre(nsid, track, mount, quota)); }); storage::backend_factory::get(). register_alias("Lustre", backend_type::lustre); @@ -925,6 +1084,7 @@ void urd::load_transfer_plugins() { const data::resource_type t2, std::shared_ptr&& trp) { +#if 0 // if(!m_transferor_registry->add(t1, t2, // std::forward>(trp))) { @@ -935,15 +1095,16 @@ void urd::load_transfer_plugins() { // return; // } +#else if(!m_task_mgr->register_transfer_plugin(t1, t2, std::forward>(trp))) { LOGGER_WARN(" Failed to load transfer plugin ({} to {}):\n" - " Another plugin was already registered for this " + " Another plugin was already registered for this\n" " combination (Plugin ignored)", utils::to_string(t1), utils::to_string(t2)); return; } - +#endif LOGGER_INFO(" Loaded transfer plugin ({} => {})", utils::to_string(t1), utils::to_string(t2)); @@ -974,10 +1135,17 @@ void urd::load_transfer_plugins() { data::resource_type::shared_posix_path, std::make_shared()); - // local path -> remote path + // local path -> remote resource load_plugin(data::resource_type::local_posix_path, - data::resource_type::remote_posix_path, - std::make_shared()); + data::resource_type::remote_resource, + std::make_shared( + m_network_endpoint)); + + // remote resource -> local path + load_plugin(data::resource_type::remote_resource, + data::resource_type::local_posix_path, + std::make_shared( + m_network_endpoint)); } void urd::load_default_namespaces() { @@ -1115,10 +1283,16 @@ int urd::run() { load_transfer_plugins(); load_default_namespaces(); + // start the listener for remote transfers + // N.B. This call returns immediately + m_network_endpoint->run(); + LOGGER_INFO(""); LOGGER_INFO("[[ Start up successful, awaiting requests... ]]"); - m_api_listener->run(); + // N.B. This call blocks here, which means that everything after it + // will only run when a shutdown command is received + m_ipc_endpoint->run(); print_farewell(); teardown(); @@ -1138,10 +1312,10 @@ void urd::teardown() { // //m_signal_listener.reset(); // } - if(m_api_listener) { + if(m_ipc_endpoint) { LOGGER_INFO("* Stopping API listener..."); - m_api_listener->stop(); - m_api_listener.reset(); + m_ipc_endpoint->stop(); + m_ipc_endpoint.reset(); } api_listener::cleanup(); @@ -1167,8 +1341,8 @@ void urd::teardown() { } void urd::shutdown() { - if(m_api_listener) { - m_api_listener->stop(); + if(m_ipc_endpoint) { + m_ipc_endpoint->stop(); } } diff --git a/src/urd.hpp b/src/urd.hpp index 8943dad..9bc01ba 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -39,15 +39,16 @@ #include "job.hpp" - - +namespace hermes { + class async_engine; + template class request; +} namespace norns { /*! Aliases for convenience */ using api_listener = api::listener>; -using api_listener_ptr = std::unique_ptr; using request_ptr = std::unique_ptr; using response_ptr = std::unique_ptr; @@ -70,6 +71,10 @@ namespace ns { struct namespace_manager; } +namespace rpc { + struct remote_transfer; +} + enum class urd_error; class urd { @@ -117,6 +122,8 @@ private: response_ptr command_handler(const request_ptr req); response_ptr unknown_request_handler(const request_ptr req); + void remote_transfer_handler(hermes::request&& req); + // TODO: add helpers for remove and update urd_error create_namespace(const config::namespace_def& nsdef); urd_error create_namespace(const std::string& nsid, backend_type type, @@ -134,7 +141,9 @@ private: std::shared_ptr m_settings; std::unique_ptr m_transferor_registry; - api_listener_ptr m_api_listener; + std::unique_ptr m_ipc_endpoint; + + std::shared_ptr m_network_endpoint; std::unique_ptr m_namespace_mgr; mutable boost::shared_mutex m_namespace_mgr_mutex; diff --git a/tests/Makefile.am b/tests/Makefile.am index 0f07a00..a3ae6d8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -41,6 +41,7 @@ api_CPPFLAGS = \ -I$(top_srcdir)/include \ -I$(top_srcdir)/rpc \ -I$(top_srcdir)/src \ + -I$(top_srcdir)/src/externals/hermes/include \ -D__NORNS_DEBUG__ \ $(END) @@ -133,6 +134,7 @@ core_CXXFLAGS = \ core_CPPFLAGS = \ -I$(top_srcdir)/include \ + -I$(top_srcdir)/src/externals/hermes/include \ -I$(top_srcdir)/rpc \ -I$(top_srcdir)/src \ $(END) diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp index 1c38ab9..270624a 100644 --- a/tests/api-copy-remote-data.cpp +++ b/tests/api-copy-remote-data.cpp @@ -37,11 +37,11 @@ SCENARIO("copy local POSIX file to remote POSIX file", "[api::norns_submit_remote_copy_local_posix_files]") { GIVEN("a running urd instance") { - test_env env(true); + test_env env(false); const char* nsid0 = "tmp0"; const char* nsid1 = "tmp1"; - const char* remote_host = "localhost"; + const char* remote_host = "127.0.0.1:42000"; bfs::path src_mnt, dst_mnt; // create namespaces @@ -84,8 +84,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname // create input data - env.add_to_namespace(nsid0, src_file_at_root, 4096); - env.add_to_namespace(nsid0, src_file_at_subdir, 8192); + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); env.add_to_namespace(nsid0, src_subdir0); env.add_to_namespace(nsid0, src_subdir1); env.add_to_namespace(nsid0, src_empty_dir); @@ -506,7 +506,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", /**********************************************************************/ /* tests for single files */ /**********************************************************************/ - // cp -r ns0://file0.txt -> ns1:// = ns1://file0.txt + // cp -r ns0://file0.txt + // -> ns1:// = ns1://file0.txt WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " "another NORNS_REMOTE_PATH at dst namespace's root " "(keeping the name)") { @@ -527,30 +528,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", // wait until the task completes rv = norns_wait(&task); -// THEN("NORNS_SUCCESS is returned") { -// REQUIRE(rv == NORNS_SUCCESS); -// -// THEN("Files are equal") { -// -// bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); -// bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); -// -// REQUIRE(compare_files(src, dst) == true); -// } -// } + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Files are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } } } -#if 0 - // cp -r ns0://file0.txt -> ns1://file1.txt = ns1://file1.txt + // cp -r ns0://file0.txt + // -> ns1://file1.txt = ns1://file1.txt WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " "another NORNS_LOCAL_PATH at dst namespace's root (changing the name)") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root1.c_str())); norns_error_t rv = norns_submit(&task); @@ -565,8 +569,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root1); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root1); REQUIRE(compare_files(src, dst) == true); } @@ -574,15 +580,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../d/file0.txt -> ns1://file0.txt = ns1://file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at dst namespace's root (keeping the name)") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file0.txt = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at dst namespace's root (keeping " + "the name)") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -597,8 +606,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_files(src, dst) == true); } @@ -606,15 +617,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../d/file0.txt -> ns1://file1.txt = ns1://file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at dst namespace's root (changing the name)") { + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at dst namespace's root (changing the " + "name)") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root1.c_str())); norns_error_t rv = norns_submit(&task); @@ -629,8 +643,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root1); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root1); REQUIRE(compare_files(src, dst) == true); } @@ -638,15 +654,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://file0.txt -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (keeping the name)") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + "another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(keeping the name)") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir0.c_str())); norns_error_t rv = norns_submit(&task); @@ -661,8 +680,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir0); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir0); REQUIRE(compare_files(src, dst) == true); } @@ -670,15 +691,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://file0.txt -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + "another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(changing the name)") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir1.c_str())); norns_error_t rv = norns_submit(&task); @@ -693,8 +717,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir1); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir1); REQUIRE(compare_files(src, dst) == true); } @@ -702,15 +728,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../file0.txt -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (keeping the name)") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(keeping the name)") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir0.c_str())); norns_error_t rv = norns_submit(&task); @@ -725,8 +754,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir0); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir0); REQUIRE(compare_files(src, dst) == true); } @@ -734,15 +765,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../file0.txt -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(changing the name)") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir1.c_str())); norns_error_t rv = norns_submit(&task); @@ -757,8 +791,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir1); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir1); REQUIRE(compare_files(src, dst) == true); } @@ -766,15 +802,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../file0.txt -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the parents names)") { + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(changing the parents names)") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir2.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir2.c_str())); norns_error_t rv = norns_submit(&task); @@ -789,8 +828,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir2); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir2); REQUIRE(compare_files(src, dst) == true); } @@ -798,15 +839,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - // cp -r ns0://a/b/c/.../file0.txt -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir (changing the name)") { + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " + "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + "(changing the name)") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_subdir3.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_subdir3.c_str())); norns_error_t rv = norns_submit(&task); @@ -821,8 +865,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_subdir3); + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_subdir3); REQUIRE(compare_files(src, dst) == true); } @@ -830,18 +876,20 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - /**************************************************************************************************************/ - /* tests for directories */ - /**************************************************************************************************************/ +#if 0 + /**********************************************************************/ + /* tests for directories */ + /**********************************************************************/ // cp -r /a/contents.* -> / = /contents.* - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " - "dst namespace's root") { + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " + "namespace's root to dst namespace's root") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_root.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_root.c_str())); norns_error_t rv = norns_submit(&task); @@ -856,7 +904,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir0); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); bfs::path dst = env.get_from_namespace(nsid1, dst_root); REQUIRE(compare_directories(src, dst) == true); diff --git a/tests/api-task-submit.cpp b/tests/api-task-submit.cpp index 0ef2e8f..f71d778 100644 --- a/tests/api-task-submit.cpp +++ b/tests/api-task-submit.cpp @@ -144,18 +144,20 @@ SCENARIO("submit request", "[api::norns_submit]") { } } - WHEN("submitting a request to copy from NORNS_MEMORY_REGION to NORNS_REMOTE_PATH") { + WHEN("submitting a request to copy from a NORNS_MEMORY_REGION to " + "a NORNS_REMOTE_PATH") { - norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(src_mem_addr, src_mem_size), - NORNS_REMOTE_PATH(nsid0, dst_host0, dst_file0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(src_mem_addr, src_mem_size), + NORNS_REMOTE_PATH(nsid0, + dst_host0, + dst_file0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); + THEN("NORNS_ENOTSUPPORTED is returned") { + REQUIRE(rv == NORNS_ENOTSUPPORTED); } } @@ -332,16 +334,17 @@ SCENARIO("submit request", "[api::norns_submit]") { WHEN("submitting a request to move from NORNS_MEMORY_REGION to NORNS_REMOTE_PATH") { - norns_op_t task_op = NORNS_IOTASK_MOVE; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(src_mem_addr, src_mem_size), - NORNS_REMOTE_PATH(nsid1, dst_host0, dst_file0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_MOVE, + NORNS_MEMORY_REGION(src_mem_addr, src_mem_size), + NORNS_REMOTE_PATH(nsid1, + dst_host0, + dst_file0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); + THEN("NORNS_ENOTSUPPORTED is returned") { + REQUIRE(rv == NORNS_ENOTSUPPORTED); } } -- GitLab From 072c1535df574520fee6ecdba7dfa9f65e8118b1 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Feb 2019 15:28:08 +0100 Subject: [PATCH 04/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9281846..0ab9992 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,25 @@ before_script: libyaml-cpp-dev libyaml-dev + - git clone https://github.com/mercury-hpc/mercury.git && + cd mercury && + git checkout 3d8ed01eeff6b504862702048d7577457c71227c && + mkdir build && + cd build && + cmake + -DCMAKE_BUILD_TYPE:STRING=Debug + -DBUILD_TESTING:BOOL=OFF + -DMERCURY_USE_SM_ROUTING:BOOL=OFF + -DMERCURY_USE_SELF_FORWARD:BOOL=OFF + -DMERCURY_USE_CHECKSUMS:BOOL=ON + -DMERCURY_USE_BOOST_PP:BOOL=ON + -DMERCURY_USE_EAGER_BULK:BOOL=ON + -DBUILD_SHARED_LIBS:BOOL=ON + -DNA_USE_OFI:BOOL=ON + .. && + make -j$(nproc) && + make install + ### GCC 5 build:gcc:5: -- GitLab From d6a869054f655164856d32d2c1bab44576385400 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Feb 2019 16:00:55 +0100 Subject: [PATCH 05/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 27 ++++++++++++++++++++++----- .gitmodules | 2 +- src/backends/backend-base.hpp | 2 +- src/externals/hermes | 2 +- src/urd.cpp | 2 ++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0ab9992..53ad74d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,6 +3,9 @@ # can use verions as well, like gcc:5.2 # see https://hub.docker.com/_/gcc/ +variables: + GIT_SUBMODULE_STRATEGY: normal + stages: - build - test @@ -22,10 +25,23 @@ before_script: protobuf-c-compiler libyaml-cpp-dev libyaml-dev + cmake + + - pushd . && + git clone https://github.com/ofiwg/libfabric.git && + cd libfabric && + ./autogen.sh && + mkdir build && + cd build && + ../configure && + make -j $(nproc) && + make install && + popd - - git clone https://github.com/mercury-hpc/mercury.git && + - pushd . && + git clone https://github.com/mercury-hpc/mercury.git && cd mercury && - git checkout 3d8ed01eeff6b504862702048d7577457c71227c && + git reset --hard 3d8ed01eeff6b504862702048d7577457c71227c && mkdir build && cd build && cmake @@ -33,14 +49,15 @@ before_script: -DBUILD_TESTING:BOOL=OFF -DMERCURY_USE_SM_ROUTING:BOOL=OFF -DMERCURY_USE_SELF_FORWARD:BOOL=OFF - -DMERCURY_USE_CHECKSUMS:BOOL=ON + -DMERCURY_USE_CHECKSUMS:BOOL=OFF -DMERCURY_USE_BOOST_PP:BOOL=ON -DMERCURY_USE_EAGER_BULK:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON -DNA_USE_OFI:BOOL=ON .. && - make -j$(nproc) && - make install + make -j $(nproc) && + make install && + popd ### GCC 5 diff --git a/.gitmodules b/.gitmodules index 0617326..f1ecde2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/externals/hermes"] path = src/externals/hermes - url = ssh://git@bscssrg.bsc.es:2222/hpc/hermes.git + url = ../hermes.git diff --git a/src/backends/backend-base.hpp b/src/backends/backend-base.hpp index 3154e14..807892f 100644 --- a/src/backends/backend-base.hpp +++ b/src/backends/backend-base.hpp @@ -104,7 +104,7 @@ public: try { return get().create(std::forward(args)...); } - catch(std::invalid_argument) { + catch(const std::invalid_argument&) { return std::shared_ptr(); } } diff --git a/src/externals/hermes b/src/externals/hermes index 57619ec..5d30d5a 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 57619ec9e554cca491b2d495f4a64aa411f3a564 +Subproject commit 5d30d5aaa329e4984ba31f6702000f87f7b7082e diff --git a/src/urd.cpp b/src/urd.cpp index ae871f3..bee6fc7 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -42,6 +42,8 @@ #include #include +#include +#include #include #include -- GitLab From 1e042f3ec8c976a16bc8837e05f1031fe67b6bd6 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 5 Feb 2019 11:11:48 +0100 Subject: [PATCH 06/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 66 ++++++++++++++++++++++++++++++++++++++------------ .gitmodules | 2 +- 2 files changed, 52 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 53ad74d..7a48b09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,10 +70,10 @@ build:gcc:5: - mkdir build && cd build - ../configure --enable-tests - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - cd tests - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api ### GCC 6 @@ -86,10 +86,10 @@ build:gcc:6: - mkdir build && cd build - ../configure --enable-tests - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" - cd tests - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core - - make -j4 CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" core + - make -j$(nproc) CPPFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0" api ### GCC 7 @@ -102,10 +102,10 @@ build:gcc:7: - mkdir build && cd build - ../configure --enable-tests - - make -j4 + - make -j$(nproc) - cd tests - - make -j4 - - make -j4 + - make -j$(nproc) + - make -j$(nproc) ### GCC 8 @@ -118,10 +118,10 @@ build:gcc:8: - mkdir build && cd build - ../configure --enable-tests - - make -j4 + - make -j$(nproc) - cd tests - - make -j4 - - make -j4 + - make -j$(nproc) + - make -j$(nproc) ################################################################################ @@ -136,6 +136,7 @@ test:ubuntu:latest: - apt-get update && apt-get upgrade -y && apt-get install -y + git build-essential autotools-dev automake @@ -155,6 +156,41 @@ test:ubuntu:latest: libyaml-dev libcap2-bin valgrind + cmake + + - pushd . && + git clone https://github.com/ofiwg/libfabric.git && + cd libfabric && + ./autogen.sh && + mkdir build && + cd build && + ../configure && + make -j $(nproc) && + make install && + popd + + - pushd . && + git clone https://github.com/mercury-hpc/mercury.git && + cd mercury && + git reset --hard 3d8ed01eeff6b504862702048d7577457c71227c && + mkdir build && + cd build && + cmake + -DCMAKE_BUILD_TYPE:STRING=Debug + -DBUILD_TESTING:BOOL=OFF + -DMERCURY_USE_SM_ROUTING:BOOL=OFF + -DMERCURY_USE_SELF_FORWARD:BOOL=OFF + -DMERCURY_USE_CHECKSUMS:BOOL=OFF + -DMERCURY_USE_BOOST_PP:BOOL=ON + -DMERCURY_USE_EAGER_BULK:BOOL=ON + -DBUILD_SHARED_LIBS:BOOL=ON + -DNA_USE_OFI:BOOL=ON + .. && + make -j $(nproc) && + make install && + popd + + - ldconfig # Build and test script: @@ -166,11 +202,11 @@ test:ubuntu:latest: # CXXFLAGS="-fsanitize=address" # LDFLAGS="-fsanitize=address" # CPPFLAGS="-D__LOGGER_ENABLE_DEBUG__" - - make -j4 + - make -j$(nproc) - cd tests - - make -j4 core + - make -j$(nproc) core - ./core -as - - make -j4 api + - make -j$(nproc) api # - NORNS_DEBUG_OUTPUT_TO_STDERR=1 NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" diff --git a/.gitmodules b/.gitmodules index f1ecde2..ea8eac9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/externals/hermes"] path = src/externals/hermes - url = ../hermes.git + url = https://gitlab+deploy-token-4:ByFiVuuKjbwSFfZPE68G@storage.bsc.es/gitlab/hpc/hermes.git -- GitLab From 3efdc0425611ef3e5c9c220d787ab73630a81326 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Feb 2019 14:52:00 +0100 Subject: [PATCH 07/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7a48b09..75dfc03 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -167,7 +167,8 @@ test:ubuntu:latest: ../configure && make -j $(nproc) && make install && - popd + popd && + ldconfig - pushd . && git clone https://github.com/mercury-hpc/mercury.git && @@ -188,9 +189,8 @@ test:ubuntu:latest: .. && make -j $(nproc) && make install && - popd - - - ldconfig + popd && + ldconfig # Build and test script: @@ -225,6 +225,8 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remote_copy_local_posix_files]" + after_script: - pwd - if [[ -e tests.log ]]; -- GitLab From e6afe9e71ffe725cc1c4e399552901825b80dd06 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Feb 2019 14:53:10 +0100 Subject: [PATCH 08/51] Additional debug messages --- .gitmodules | 2 +- src/externals/hermes | 2 +- .../local-path-to-remote-resource.cpp | 15 ++ src/rpcs.hpp | 132 +++++++++++++++++- src/urd.cpp | 18 +++ 5 files changed, 165 insertions(+), 4 deletions(-) diff --git a/.gitmodules b/.gitmodules index ea8eac9..f1ecde2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "src/externals/hermes"] path = src/externals/hermes - url = https://gitlab+deploy-token-4:ByFiVuuKjbwSFfZPE68G@storage.bsc.es/gitlab/hpc/hermes.git + url = ../hermes.git diff --git a/src/externals/hermes b/src/externals/hermes index 5d30d5a..160eafc 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 5d30d5aaa329e4984ba31f6702000f87f7b7082e +Subproject commit 160eafc2de0b6744f723a0f9332eefb070fddbbc diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index e0fa877..7818a9d 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -99,6 +99,21 @@ local_path_to_remote_resource_transferor::transfer( d_dst.name(), buffers); + LOGGER_DEBUG("rpc::in::args{{"); + LOGGER_DEBUG(" address: \"{}\",", m_remote_endpoint->self_address()); + LOGGER_DEBUG(" in_nsid: \"{}\",", d_src.parent()->nsid()); + LOGGER_DEBUG(" out_nsid: \"{}\",", d_dst.parent()->nsid()); + LOGGER_DEBUG(" btype: {} ({}),", + static_cast(backend_type::posix_filesystem), + utils::to_string(backend_type::posix_filesystem)); + LOGGER_DEBUG(" rtype: {} ({}),", + static_cast(data::resource_type::local_posix_path), + utils::to_string(data::resource_type::local_posix_path)); + LOGGER_DEBUG(" rname: \"{}\",", d_dst.name()); + LOGGER_DEBUG(" buffers: {{...}}"); + LOGGER_DEBUG("};"); + LOGGER_FLUSH(); + auto start = std::chrono::steady_clock::now(); auto rpc = diff --git a/src/rpcs.hpp b/src/rpcs.hpp index c2d9dd5..3ebe490 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -104,7 +104,90 @@ struct remote_transfer { m_backend_type(backend_type), m_resource_type(resource_type), m_resource_name(resource_name), - m_buffers(buffers) { } + m_buffers(buffers) { + +#ifdef HERMES_DEBUG_BUILD + this->print("this", __PRETTY_FUNCTION__); +#endif + + } + +#ifdef HERMES_DEBUG_BUILD + input(input&& rhs) : + m_address(std::move(rhs.m_address)), + m_in_nsid(std::move(rhs.m_in_nsid)), + m_out_nsid(std::move(rhs.m_out_nsid)), + m_backend_type(std::move(rhs.m_backend_type)), + m_resource_type(std::move(rhs.m_resource_type)), + m_resource_name(std::move(rhs.m_resource_name)), + m_buffers(std::move(rhs.m_buffers)) { + + rhs.m_backend_type = 0; + rhs.m_resource_type = 0; + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + } + + input(const input& other) : + m_address(other.m_address), + m_in_nsid(other.m_in_nsid), + m_out_nsid(other.m_out_nsid), + m_backend_type(other.m_backend_type), + m_resource_type(other.m_resource_type), + m_resource_name(other.m_resource_name), + m_buffers(other.m_buffers) { + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + } + + input& + operator=(input&& rhs) { + + if(this != &rhs) { + m_address = std::move(rhs.m_address); + m_in_nsid = std::move(rhs.m_in_nsid); + m_out_nsid = std::move(rhs.m_out_nsid); + m_backend_type = std::move(rhs.m_backend_type); + m_resource_type = std::move(rhs.m_resource_type); + m_resource_name = std::move(rhs.m_resource_name); + m_buffers = std::move(rhs.m_buffers); + + rhs.m_backend_type = 0; + rhs.m_resource_type = 0; + } + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + + return *this; + } + + input& + operator=(const input& other) { + + if(this != &other) { + m_address = other.m_address; + m_in_nsid = other.m_in_nsid; + m_out_nsid = other.m_out_nsid; + m_backend_type = other.m_backend_type; + m_resource_type = other.m_resource_type; + m_resource_name = other.m_resource_name; + m_buffers = other.m_buffers; + } + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + + return *this; + } +#else // HERMES_DEBUG_BUILD + input(input&& rhs) = default; + input(const input& other) = default; + input& operator=(input&& rhs) = default; + input& operator=(const input& other) = default; +#endif // ! HERMES_DEBUG_BUILD std::string address() const { @@ -141,6 +224,36 @@ struct remote_transfer { return m_buffers; } +#ifdef HERMES_DEBUG_BUILD + void + print(const std::string& id, + const std::string& caller = "") const { + + (void) id; + auto c = caller.empty() ? "unknown_caller" : caller; + + HERMES_DEBUG2("{}, {} ({}) = {{", caller, id, fmt::ptr(this)); + HERMES_DEBUG2(" m_address: \"{}\" ({} -> {}),", + m_address, fmt::ptr(&m_address), + fmt::ptr(m_address.c_str())); + HERMES_DEBUG2(" m_in_nsid: \"{}\" ({} -> {}),", + m_in_nsid, fmt::ptr(&m_in_nsid), + fmt::ptr(m_in_nsid.c_str())); + HERMES_DEBUG2(" m_out_nsid: \"{}\" ({} -> {}),", + m_out_nsid, fmt::ptr(&m_out_nsid), + fmt::ptr(m_out_nsid.c_str())); + HERMES_DEBUG2(" m_backend_type: {},", + m_backend_type); + HERMES_DEBUG2(" m_resource_type: {},", + m_resource_type); + HERMES_DEBUG2(" m_resource_name: \"{}\" ({} -> {}),", + m_resource_name, fmt::ptr(&m_resource_name), + fmt::ptr(m_resource_name.c_str())); + HERMES_DEBUG2(" m_buffers: {...},"); + HERMES_DEBUG2("}}"); + } +#endif // ! HERMES_DEBUG_BUILD + //TODO: make private explicit input(const hermes::detail::remote_transfer_in_t& other) : @@ -150,7 +263,22 @@ struct remote_transfer { m_backend_type(other.backend_type), m_resource_type(other.resource_type), m_resource_name(other.resource_name), - m_buffers(other.buffers) { } + m_buffers(other.buffers) { + + HERMES_DEBUG("input::input(const hermes::detail::remote_transfer_in_t&){{"); + HERMES_DEBUG(" m_address: {} ({}),", + m_address, fmt::ptr(&m_address)); + HERMES_DEBUG(" m_in_nsid: {} ({}),", + m_in_nsid, fmt::ptr(&m_in_nsid)); + HERMES_DEBUG(" m_out_nsid: {} ({}),", + m_out_nsid, fmt::ptr(&m_out_nsid)); + HERMES_DEBUG(" m_backend_type: {},", + m_backend_type); + HERMES_DEBUG(" m_resource_type: {},", + m_resource_type); + HERMES_DEBUG(" m_buffers: {...},"); + HERMES_DEBUG("}}"); + } explicit operator hermes::detail::remote_transfer_in_t() { diff --git a/src/urd.cpp b/src/urd.cpp index bee6fc7..bc963e1 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -680,6 +680,24 @@ urd::remote_transfer_handler(hermes::request&& req) { args.out_nsid(), args.resource_name()); + LOGGER_DEBUG("rpc::out::args{{"); + LOGGER_DEBUG(" address: \"{}\",", args.address()); + LOGGER_DEBUG(" in_nsid: \"{}\",", args.in_nsid()); + LOGGER_DEBUG(" out_nsid: \"{}\",", args.out_nsid()); + LOGGER_DEBUG(" btype: {} ({}),", + args.backend_type(), + utils::to_string( + static_cast(args.backend_type()))); + LOGGER_DEBUG(" rtype: {} ({}),", + args.resource_type(), + utils::to_string( + static_cast(args.resource_type()))); + LOGGER_DEBUG(" rname: \"{}\",", args.resource_name()); + LOGGER_DEBUG(" buffers: {{...}}"); + LOGGER_DEBUG("};"); + LOGGER_FLUSH(); + + urd_error rv = urd_error::success; boost::optional t; std::shared_ptr src_backend; -- GitLab From ff966c4aa03ab9969eb45f6fdcd697505f8df855 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Feb 2019 12:57:10 +0100 Subject: [PATCH 09/51] Update README.md --- README.md | 119 +++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 96 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 3588a22..e65be7d 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,104 @@ +# Norns [![pipeline status](https://storage.bsc.es/gitlab/hpc/norns/badges/master/pipeline.svg)](https://storage.bsc.es/gitlab/hpc/norns/commits/master) +Norns is an open-source data scheduling service that orchestrates asynchronous +data transfers between different storage backends in an HPC cluster. Through +its API, Norns provides three different functions. First, it allows system +administrators to expose the storage architecture of an HPC cluster by creating +**dataspaces** associated to different storage backends such as node-local NVMs +and POSIX file systems, or system-wide parallel file systems and object stores, +thus making them available to applications, services, and users. +Second, it provides a framework for submitting and monitoring the asynchronous +transfers of **data resources** between the (local and remote) dataspaces +available to a user, such as process buffers, POSIX files and directories or +objects. Third, it arbitrates requests by managing a queue of pending work and +evaluating requests to maximize dataspace throughput while minimizing the +interferences with normal application I/O. -Norns Data Scheduler -==================== +Norns has currently been tested only under GNU/Linux. -Build dependencies: +## Building and installing from source -- A c++11-conforming compiler -- libboost-system >= 1.53 -- libboost-filesystem >= 1.53 -- libboost-program-options >= 1.53 -- libboost-thread >= 1.53 -- libboost-regex >= 1.53 (only if self tests are also built) -- libprotobuf + protobuf compiler >= 2.5.0 -- libprotobuf-c + protobuf-c compiler >= 1.0.2 -- libyaml-cpp >= 0.5.1 -- libyaml >= 0.1.4 +Distribution tarballs are available from the [releases](releases) tab. If you +are building Norns from a developer Git clone, you must first run the +`bootstrap.sh` script, which will invoke the GNU Autotools to bootstrap Norns' +configuration and build mechanisms. If you are building Norns from an official +distribiution tarball, there is no need to run the `bootstrap.sh` script, since +all distribution tarballs are already boostrapped. +### Dependencies -- Installation in CentOS 7 -git clone git@git.ph.ed.ac.uk:nextgenio/norns.git && cd norns -./bootstrap.sh -mkdir -cd -./configure --prefix= --sysconfdir= -make -make install -cp /etc/norns.service /usr/lib/systemd/system/norns.service +Compiling and running Norns requires up-to-date versions of the following +software packages (note that, though it may compile and run, using excessively +old versions of these packages can cause indirect errors that are very +difficult to track down): -sudo setcap cap_sys_ptrace,cap_chown=+ep ./urd +- A standard **C++11** conforming compiler (the code is routinely tested + against GCC 4.9/Clang 3.3 and higher). +- Autotools (autoconf 2.69 or higher, automake 1.14.1 or higher, and libtool + 2.4.2 or higher) and CMake (3.10.0 or higher). +- The following Boost libraries (1.53 or higher): `system`, `filesystem`, + `program_options`, and `thread`. Optionally, the `regex` library may also be + required if self tests are enabled with the `--enable-tests` option. +- Google's Protocol Buffers for [C](https://github.com/protobuf-c/protobuf-c) + (1.0.2 or higher) and for [C++](https://github.com/protocolbuffers/protobuf) + (2.5.0 or higher). +- [LibYAML](https://github.com/yaml/libyaml) (0.1.4 or higher) and + [yaml-cpp](https://github.com/jbeder/yaml-cpp) (0.5.1 or higher). +- [Mercury](https://github.com/mercury-hpc/mercury) 1.0 or higher + (**IMPORTANT** Mercury may require additional dependencies such as libfabric + depending on the desired transport protocol). +- [Hermes](https://storage.bsc.es/gitlab/hpc/hermes) (our own C++ wrapper for + Mercury. It should be automatically downloaded when cloning with the + `--recursive` option). + +#### Installation in CentOS + +TODO + +#### Installation in Ubuntu + +```bash +# Installing dependencies avaiable through package manager +$ apt-get install -y libboost-system-dev libboost-filesystem-dev \ + libboost-program-options-dev libboost-thread-dev \ + libboost-regex-dev libprotobuf-dev protobuf-compiler \ + libprotobuf-c-dev protobuf-c-compiler \ + libyaml-cpp-dev libyaml-dev + +# Building and installing libfabric (required for Mercury's OFI/libfabric plugin) +$ git clone https://github.com/ofiwg/libfabric.git && +$ cd libfabric +$ ./autogen.sh +$ mkdir build && cd build +$ ../configure && make && make install + +# Building and installing Mercury with OFI/libfabric plugin +$ git clone https://github.com/mercury-hpc/mercury.git +$ cd mercury +$ mkdir build && cd build +$ cmake -DCMAKE_BUILD_TYPE:STRING=Debug -DBUILD_TESTING:BOOL=OFF \ + -DMERCURY_USE_SM_ROUTING:BOOL=OFF -DMERCURY_USE_SELF_FORWARD:BOOL=OFF \ + -DMERCURY_USE_CHECKSUMS:BOOL=OFF -DMERCURY_USE_BOOST_PP:BOOL=ON \ + -DMERCURY_USE_EAGER_BULK:BOOL=ON -DBUILD_SHARED_LIBS:BOOL=ON \ + -DNA_USE_OFI:BOOL=ON \ + .. +$ make && make install + +# Building, testing and installing Norns under '/usr/local/', with configuration +# files under '/etc/norns/' and temporary files under '/var/run/norns/' +$ git clone --recursive https://storage.bsc.es/gitlab/hpc/norns.git +$ cd norns +$ ./bootstrap.sh +$ mkdir build && cd build +$ ../configure \ + --enable-tests \ + --prefix=/usr/local \ + --sysconfdir=/etc/norns \ + --localstatedir=/var/run/norns +$ make && make check && make install + +# Optional: providing file system permission override capabilities to Norns +# control daemon +$ setcap cap_sys_ptrace,cap_chown=+ep /usr/local/bin/urd +``` -- GitLab From 9f1e8f551ce357ec76596659adcd6c3925b63f7a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Feb 2019 16:06:59 +0100 Subject: [PATCH 10/51] Add 'bind_address' to config file --- etc/norns.conf.in | 3 +++ src/Makefile.am | 1 + src/config/config-schema.hpp | 4 ++++ src/config/defaults.hpp | 1 + src/config/keywords.hpp | 1 + src/config/settings.cpp | 9 +++++++++ src/config/settings.hpp | 3 +++ src/urd.cpp | 2 +- tests/fake-daemon.cpp | 2 ++ 9 files changed, 25 insertions(+), 1 deletion(-) diff --git a/etc/norns.conf.in b/etc/norns.conf.in index 125ab3f..becbdf6 100644 --- a/etc/norns.conf.in +++ b/etc/norns.conf.in @@ -17,6 +17,9 @@ global_settings: [ # path to pidfile pidfile: "@localstatedir@/urd.pid", + + # address to bind to + bind_address: "127.0.0.1", # incoming port for remote connections remote_port: 42000, diff --git a/src/Makefile.am b/src/Makefile.am index 66ffdb8..4f025b7 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -247,6 +247,7 @@ config/defaults.cpp: Makefile \ echo " const char* global_socket = \"$(localstatedir)/global.socket.2\";"; \ echo " const char* control_socket = \"$(localstatedir)/control.socket.2\";"; \ + echo " const char* bind_address = \"127.0.0.1\";"; \ echo " const in_port_t remote_port = 42000;"; \ echo " const char* pidfile = \"$(localstatedir)/urd.pid\";"; \ \ diff --git a/src/config/config-schema.hpp b/src/config/config-schema.hpp index 81bfb68..611a6d4 100644 --- a/src/config/config-schema.hpp +++ b/src/config/config-schema.hpp @@ -84,6 +84,10 @@ const file_schema valid_options = declare_file({ opt_type::mandatory, converter(parsers::parse_path)), + declare_option( + keywords::bind_address, + opt_type::mandatory), + declare_option( keywords::remote_port, opt_type::mandatory, diff --git a/src/config/defaults.hpp b/src/config/defaults.hpp index 1a38ec6..cb33fe1 100644 --- a/src/config/defaults.hpp +++ b/src/config/defaults.hpp @@ -48,6 +48,7 @@ namespace defaults { extern const uint32_t dry_run_duration; extern const char* global_socket; extern const char* control_socket; + extern const char* bind_address; extern const in_port_t remote_port; extern const char* pidfile; extern const uint32_t workers_in_pool; diff --git a/src/config/keywords.hpp b/src/config/keywords.hpp index e5bf86b..0f73457 100644 --- a/src/config/keywords.hpp +++ b/src/config/keywords.hpp @@ -48,6 +48,7 @@ constexpr static const auto log_file_max_size = "log_file_max_size"; constexpr static const auto dry_run = "dry_run"; constexpr static const auto global_socket = "global_socket"; constexpr static const auto control_socket = "control_socket"; +constexpr static const auto bind_address = "bind_address"; constexpr static const auto remote_port = "remote_port"; constexpr static const auto pidfile = "pidfile"; constexpr static const auto workers = "workers"; diff --git a/src/config/settings.cpp b/src/config/settings.cpp index accbf0f..eb44067 100644 --- a/src/config/settings.cpp +++ b/src/config/settings.cpp @@ -56,6 +56,7 @@ settings::settings(const std::string& progname, uint32_t dry_run_duration, const bfs::path& global_socket, const bfs::path& control_socket, + const std::string& bind_address, uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, @@ -72,6 +73,7 @@ settings::settings(const std::string& progname, m_dry_run_duration(dry_run_duration), m_global_socket(global_socket), m_control_socket(control_socket), + m_bind_address(bind_address), m_remote_port(remote_port), m_daemon_pidfile(pidfile), m_workers_in_pool(workers), @@ -90,6 +92,7 @@ void settings::load_defaults() { m_dry_run_duration = defaults::dry_run_duration; m_global_socket = defaults::global_socket; m_control_socket = defaults::control_socket; + m_bind_address = defaults::bind_address; m_remote_port = defaults::remote_port; m_daemon_pidfile = defaults::pidfile; m_workers_in_pool = defaults::workers_in_pool; @@ -124,6 +127,7 @@ void settings::load_from_file(const bfs::path& filename) { m_dry_run_duration = defaults::dry_run_duration; m_global_socket = gsettings.get_as(keywords::global_socket); m_control_socket = gsettings.get_as(keywords::control_socket); + m_bind_address = gsettings.get_as(keywords::bind_address); m_remote_port = gsettings.get_as(keywords::remote_port); m_daemon_pidfile = gsettings.get_as(keywords::pidfile); m_workers_in_pool = gsettings.get_as(keywords::workers); @@ -156,6 +160,7 @@ std::string settings::to_string() const { " m_dry_run_duration: " + std::to_string(m_dry_run_duration) + ",\n" + " m_global_socket: " + m_global_socket.string() + ",\n" + " m_control_socket: " + m_control_socket.string() + ",\n" + + " m_bind_address: " + m_bind_address + ",\n" + " m_remote_port: " + std::to_string(m_remote_port) + ",\n" + " m_pidfile: " + m_daemon_pidfile.string() + ",\n" + " m_workers: " + std::to_string(m_workers_in_pool) + ",\n" + @@ -206,6 +211,10 @@ bfs::path& settings::control_socket() { return m_control_socket; } +std::string& settings::bind_address() { + return m_bind_address; +} + in_port_t& settings::remote_port() { return m_remote_port; } diff --git a/src/config/settings.hpp b/src/config/settings.hpp index 309a515..d53bedb 100644 --- a/src/config/settings.hpp +++ b/src/config/settings.hpp @@ -97,6 +97,7 @@ struct settings { uint32_t dry_run_duration, const bfs::path& global_socket, const bfs::path& control_socket, + const std::string& bind_address, uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, @@ -117,6 +118,7 @@ struct settings { uint32_t& dry_run_duration(); bfs::path& global_socket(); bfs::path& control_socket(); + std::string& bind_address(); in_port_t& remote_port(); bfs::path& pidfile(); uint32_t& workers_in_pool(); @@ -134,6 +136,7 @@ struct settings { uint32_t m_dry_run_duration = defaults::dry_run_duration; bfs::path m_global_socket = defaults::global_socket; bfs::path m_control_socket = defaults::control_socket; + std::string m_bind_address = defaults::bind_address; in_port_t m_remote_port = defaults::remote_port; bfs::path m_daemon_pidfile = defaults::pidfile; uint32_t m_workers_in_pool = defaults::workers_in_pool; diff --git a/src/urd.cpp b/src/urd.cpp index bc963e1..6bc11f2 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -901,7 +901,7 @@ void urd::init_event_handlers() { try { const std::string bind_address = - std::string("127.0.0.1:") + + m_settings->bind_address() + ":" + std::to_string(m_settings->remote_port()); m_network_endpoint = diff --git a/tests/fake-daemon.cpp b/tests/fake-daemon.cpp index 573ada2..2517804 100644 --- a/tests/fake-daemon.cpp +++ b/tests/fake-daemon.cpp @@ -45,6 +45,7 @@ norns::config::settings default_cfg( 100, /* dry run duration */ "./test_urd.global.socket", /* global_socket */ "./test_urd.control.socket", /* control_socket */ + "127.0.0.1", /* bind address */ 42002, /* remote port */ "./test_urd.pid", /* daemon_pidfile */ 2, /* api workers */ @@ -77,6 +78,7 @@ void fake_daemon::configure(const bfs::path& config_file, m_config.use_console() = default_cfg.use_console(); m_config.dry_run() = override_cfg.m_dry_run; m_config.dry_run_duration() = override_cfg.m_dry_run_duration; + m_config.bind_address() = default_cfg.bind_address(); m_config.remote_port() = default_cfg.remote_port(); m_config.workers_in_pool() = default_cfg.workers_in_pool(); m_config.config_file() = config_file; -- GitLab From b65139d705f292339a912fbe63ea6de2b2fc83ad Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Feb 2019 20:15:59 +0100 Subject: [PATCH 11/51] Propagate remote errors to task creator --- src/io/task-info.cpp | 11 ++++ src/io/task-info.hpp | 6 +++ src/io/task.hpp | 12 +++++ .../local-path-to-remote-resource.cpp | 6 ++- .../remote-resource-to-local-path.cpp | 16 +++++- src/rpcs.hpp | 52 +++++++++++++++---- src/urd.cpp | 22 ++++++-- 7 files changed, 105 insertions(+), 20 deletions(-) diff --git a/src/io/task-info.cpp b/src/io/task-info.cpp index 085bae5..cddfc95 100644 --- a/src/io/task-info.cpp +++ b/src/io/task-info.cpp @@ -138,6 +138,17 @@ task_info::update_status(const task_status st, const urd_error ec, m_sys_error = sc; } +urd_error +task_info::task_error() const { + boost::shared_lock lock(m_mutex); + return m_task_error; +} + +std::error_code +task_info::sys_error() const { + return m_sys_error; +} + std::size_t task_info::sent_bytes() const { boost::shared_lock lock(m_mutex); diff --git a/src/io/task-info.hpp b/src/io/task-info.hpp index 566d021..10c79dd 100644 --- a/src/io/task-info.hpp +++ b/src/io/task-info.hpp @@ -73,6 +73,12 @@ struct task_info { void update_status(const task_status st, const urd_error ec, const std::error_code& sc); + urd_error + task_error() const; + + std::error_code + sys_error() const; + std::size_t sent_bytes() const; std::size_t total_bytes() const; double bandwidth() const; diff --git a/src/io/task.hpp b/src/io/task.hpp index 95b1c67..c1769c5 100644 --- a/src/io/task.hpp +++ b/src/io/task.hpp @@ -132,6 +132,12 @@ struct generic_task { return boost::get(m_impl).m_task_info->id(); } + case iotask_type::remote_transfer: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info->id(); + } + default: return static_cast(0); } @@ -164,6 +170,12 @@ struct generic_task { return boost::get(m_impl).m_task_info; } + case iotask_type::remote_transfer: + { + using TaskType = io::task; + return boost::get(m_impl).m_task_info; + } + default: return {}; } diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 7818a9d..a8226b7 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -126,9 +126,11 @@ local_path_to_remote_resource_transferor::transfer( task_info->record_transfer(input_data.size(), usecs); - LOGGER_DEBUG("Remote request completed with retval {} " + LOGGER_DEBUG("Remote request completed with output " + "{{status: {}, task_error: {}, sys_errnum: {}}} " "({} bytes, {} usecs)", - resp.at(0).retval(), input_data.size(), usecs); + resp.at(0).status(), resp.at(0).task_error(), + resp.at(0).sys_errnum(), input_data.size(), usecs); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index dcb02c8..ba1a08e 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -30,6 +30,7 @@ #include "resources.hpp" #include "auth.hpp" #include "io/task-info.hpp" +#include "io/task-stats.hpp" #include "hermes.hpp" #include "rpcs.hpp" #include "remote-resource-to-local-path.hpp" @@ -156,7 +157,10 @@ remote_resource_to_local_path_transferor::transfer( if(ec) { if(req.requires_response()) { m_remote_endpoint->respond( - std::move(req), static_cast(urd_error::snafu));//XXX + std::move(req), + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value())); } return ec; } @@ -183,7 +187,10 @@ remote_resource_to_local_path_transferor::transfer( if(req.requires_response()) { m_remote_endpoint->respond( - std::move(req), static_cast(urd_error::success)); + std::move(req), + static_cast(task_status::finished), + static_cast(urd_error::success), + 0); } }; @@ -202,6 +209,11 @@ remote_resource_to_local_path_transferor::transfer( const std::shared_ptr& src_rinfo, const std::shared_ptr& dst_rinfo) const { + (void) auth; + (void) task_info; + (void) src_rinfo; + (void) dst_rinfo; + #if 0 (void) auth; const auto src_backend = task_info->src_backend(); diff --git a/src/rpcs.hpp b/src/rpcs.hpp index 3ebe490..ab02d8e 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -43,7 +43,9 @@ MERCURY_GEN_PROC(remote_transfer_in_t, ((hg_bulk_t) (buffers))) MERCURY_GEN_PROC(remote_transfer_out_t, - ((int32_t) (retval))) + ((uint32_t) (status)) + ((uint32_t) (task_error)) + ((uint32_t) (sys_errnum))) }} // namespace hermes::detail @@ -308,31 +310,59 @@ struct remote_transfer { friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); public: - output(int32_t retval) : - m_retval(retval) { } + output(uint32_t status, + uint32_t task_error, + uint32_t sys_errnum) : + m_status(status), + m_task_error(task_error), + m_sys_errnum(sys_errnum) {} - int32_t - retval() const { - return m_retval; + uint32_t + status() const { + return m_status; } void - set_retval(int32_t retval) { - m_retval = retval; + set_retval(uint32_t status) { + m_status = status; + } + + uint32_t + task_error() const { + return m_task_error; + } + + void + set_task_error(uint32_t task_error) { + m_task_error = task_error; + } + + uint32_t + sys_errnum() const { + return m_sys_errnum; + } + + void + set_sys_errnum(uint32_t errnum) { + m_sys_errnum = errnum; } explicit output(const hermes::detail::remote_transfer_out_t& out) { - m_retval = out.retval; + m_status = out.status; + m_task_error = out.task_error; + m_sys_errnum = out.sys_errnum; } explicit operator hermes::detail::remote_transfer_out_t() { - return {m_retval}; + return {m_status, m_task_error, m_sys_errnum}; } private: - int32_t m_retval; + uint32_t m_status; + uint32_t m_task_error; + uint32_t m_sys_errnum; }; }; diff --git a/src/urd.cpp b/src/urd.cpp index 6bc11f2..0a8aa21 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -730,7 +730,10 @@ urd::remote_transfer_handler(hermes::request&& req) { if(m_is_paused) { rv = urd_error::accept_paused; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), static_cast(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(rv), + 0); return; } @@ -748,7 +751,10 @@ urd::remote_transfer_handler(hermes::request&& req) { if(!dst_backend) { rv = urd_error::no_such_namespace; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), static_cast(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(rv), + 0); return; } @@ -764,12 +770,18 @@ urd::remote_transfer_handler(hermes::request&& req) { *dst_backend, dst_rinfo); if(rv == urd_error::success) { - // run the task + // run the task and check that it started correctly (*t)(); - // rv = t.info().status() ?? + if(t->info()->status() == io::task_status::finished_with_error) { + rv = urd_error::no_such_namespace; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(t->info()->task_error()), + static_cast(t->info()->sys_error().value())); + } } - } -- GitLab From 079054591776e3075618e5c241424e3e834c1f6c Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Feb 2019 21:16:50 +0100 Subject: [PATCH 12/51] Improve error reporting for remote tasks --- src/externals/hermes | 2 +- .../local-path-to-remote-resource.cpp | 26 +++++++++++------ .../local-path-to-remote-resource.hpp | 4 +-- .../remote-resource-to-local-path.cpp | 28 +++++++++---------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/externals/hermes b/src/externals/hermes index 160eafc..e49040c 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 160eafc2de0b6744f723a0f9332eefb070fddbbc +Subproject commit e49040c5cc2df53c4a89349bb3bf5ce59bcf0231 diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index a8226b7..4f579b5 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -45,8 +45,8 @@ namespace norns { namespace io { local_path_to_remote_resource_transferor::local_path_to_remote_resource_transferor( - std::shared_ptr remote_endpoint) : - m_remote_endpoint(remote_endpoint) { } + std::shared_ptr network_endpoint) : + m_network_endpoint(network_endpoint) { } bool local_path_to_remote_resource_transferor::validate( @@ -78,20 +78,28 @@ local_path_to_remote_resource_transferor::transfer( LOGGER_DEBUG("[{}] start_transfer: {} -> {}", task_info->id(), d_src.canonical_path(), d_dst.to_string()); - hermes::endpoint endp = m_remote_endpoint->lookup(d_dst.address()); + hermes::endpoint endp = m_network_endpoint->lookup(d_dst.address()); try { - hermes::mapped_buffer input_data(d_src.canonical_path().string()); + std::error_code ec; + hermes::mapped_buffer input_data(d_src.canonical_path().string(), + hermes::access_mode::read_only, + &ec); + + if(ec) { + LOGGER_ERROR("Failed mapping input data: {}", ec.value()); + return ec; + } std::vector bufvec{ - hermes::mutable_buffer{(void*) input_data.data(), input_data.size()} + hermes::mutable_buffer{input_data.data(), input_data.size()} }; auto buffers = - m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); + m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); norns::rpc::remote_transfer::input args( - m_remote_endpoint->self_address(), + m_network_endpoint->self_address(), d_src.parent()->nsid(), d_dst.parent()->nsid(), static_cast(backend_type::posix_filesystem), @@ -100,7 +108,7 @@ local_path_to_remote_resource_transferor::transfer( buffers); LOGGER_DEBUG("rpc::in::args{{"); - LOGGER_DEBUG(" address: \"{}\",", m_remote_endpoint->self_address()); + LOGGER_DEBUG(" address: \"{}\",", m_network_endpoint->self_address()); LOGGER_DEBUG(" in_nsid: \"{}\",", d_src.parent()->nsid()); LOGGER_DEBUG(" out_nsid: \"{}\",", d_dst.parent()->nsid()); LOGGER_DEBUG(" btype: {} ({}),", @@ -117,7 +125,7 @@ local_path_to_remote_resource_transferor::transfer( auto start = std::chrono::steady_clock::now(); auto rpc = - m_remote_endpoint->post(endp, args); + m_network_endpoint->post(endp, args); auto resp = rpc.get(); diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index 852e173..fe1c034 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -60,7 +60,7 @@ namespace io { struct local_path_to_remote_resource_transferor : public transferor { local_path_to_remote_resource_transferor( - std::shared_ptr remote_endpoint); + std::shared_ptr network_endpoint); bool validate(const std::shared_ptr& src_info, @@ -88,7 +88,7 @@ private: void do_accept(hermes::request&& req); - std::shared_ptr m_remote_endpoint; + std::shared_ptr m_network_endpoint; }; diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index ba1a08e..627bac8 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -41,11 +41,13 @@ std::tuple> create_file(const bfs::path& filename, std::size_t size) { + std::error_code ec; + int out_fd = ::open(filename.c_str(), O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); if(out_fd == -1) { - auto ec = std::make_error_code(static_cast(errno)); + ec = std::make_error_code(static_cast(errno)); return std::make_tuple(ec, nullptr); } @@ -54,11 +56,11 @@ create_file(const bfs::path& filename, // filesystem doesn't support fallocate(), fallback to truncate() if(errno == EOPNOTSUPP) { if(::ftruncate(out_fd, size) != 0) { - auto ec = std::make_error_code(static_cast(errno)); + ec = std::make_error_code(static_cast(errno)); return std::make_tuple(ec, nullptr); } } - auto ec = std::make_error_code(static_cast(errno)); + ec = std::make_error_code(static_cast(errno)); return std::make_tuple(ec, nullptr); } @@ -68,25 +70,21 @@ retry_close: goto retry_close; } - auto ec = std::make_error_code(static_cast(errno)); + ec = std::make_error_code(static_cast(errno)); return std::make_tuple(ec, nullptr); } - std::shared_ptr output_data; - - try { - output_data = - std::make_shared( - filename.string(), hermes::access_mode::write_only); - } - catch(const std::exception& ex) { - LOGGER_ERROR(ex.what()); + auto output_data = + std::make_shared( + filename.string(), + hermes::access_mode::write_only, + &ec); - auto ec = std::make_error_code(static_cast(-1)); + if(ec) { + LOGGER_ERROR("Failed mapping output data: {}", ec.value()); return std::make_tuple(ec, output_data); } - auto ec = std::make_error_code(static_cast(0)); return std::make_tuple(ec, output_data); } -- GitLab From a454f2ec2dcef4635422a76e3e8a8ccd79b97a6a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 19 Feb 2019 16:40:23 +0100 Subject: [PATCH 13/51] Initial support for transferring remote directories --- configure.ac | 8 +- src/Makefile.am | 5 + src/io/task-remote-transfer.hpp | 2 +- .../local-path-to-remote-resource.cpp | 66 ++- .../remote-resource-to-local-path.cpp | 94 ++++- src/io/transferors/tar-archive.cpp | 108 +++++ src/io/transferors/tar-archive.hpp | 68 +++ src/logger.hpp | 9 + .../detail/remote-resource-impl.cpp | 2 +- .../detail/remote-resource-info.cpp | 22 +- .../detail/remote-resource-info.hpp | 21 +- src/rpcs.hpp | 45 +- src/urd.cpp | 11 +- src/utils.cpp | 20 + src/utils.hpp | 3 + tests/Makefile.am | 2 + tests/api-copy-remote-data.cpp | 391 +++++++++++------- 17 files changed, 671 insertions(+), 206 deletions(-) create mode 100644 src/io/transferors/tar-archive.cpp create mode 100644 src/io/transferors/tar-archive.hpp diff --git a/configure.ac b/configure.ac index 8b6fbbe..01f1a80 100644 --- a/configure.ac +++ b/configure.ac @@ -144,7 +144,13 @@ AS_IF([test "x${PROTOC}" == "x"], AC_SEARCH_LIBS([yaml_parser_initialize], [yaml], [YAML_LIBS="-lyaml" AC_SUBST(YAML_LIBS)], - [AC_MSG_ERROR([This software required libyaml >= 0.1.4])]) + [AC_MSG_ERROR([This software requires libyaml >= 0.1.4])]) + +# check for libtar manually (since it doesn't provide a pkgconfig file) +AC_SEARCH_LIBS([tar_open], [tar], + [TAR_LIBS="-ltar" + AC_SUBST(TAR_LIBS)], + [AC_MSG_ERROR([This software requires libtar >= 1.2.0])]) # Checks for header files. diff --git a/src/Makefile.am b/src/Makefile.am index 4f025b7..08c4df3 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -90,6 +90,7 @@ liburd_resources_la_LDFLAGS = \ @BOOST_THREAD_LIB@ \ @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ + @TAR_LIBS@ \ -pthread @@ -167,6 +168,8 @@ liburd_aux_la_SOURCES = \ io/transferors/local-path-to-remote-resource.hpp \ io/transferors/remote-resource-to-local-path.cpp \ io/transferors/remote-resource-to-local-path.hpp \ + io/transferors/tar-archive.cpp \ + io/transferors/tar-archive.hpp \ io/transferors/memory-to-local-path.cpp \ io/transferors/memory-to-local-path.hpp \ io/transferors/memory-to-shared-path.cpp \ @@ -218,6 +221,7 @@ liburd_aux_la_LDFLAGS = \ @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ @YAMLCPP_LIBS@ \ + @TAR_LIBS@ \ liburd_resources.la \ -pthread @@ -293,6 +297,7 @@ urd_LDFLAGS = \ @BOOST_THREAD_LIB@ \ @MERCURY_LIBS@ \ @PROTOBUF_LIBS@ \ + @TAR_LIBS@ \ liburd_aux.la # we also need to include it as an additional dependency, since automake diff --git a/src/io/task-remote-transfer.hpp b/src/io/task-remote-transfer.hpp index 1f9ba96..b758d3e 100644 --- a/src/io/task-remote-transfer.hpp +++ b/src/io/task-remote-transfer.hpp @@ -51,7 +51,7 @@ task::operator()() { const auto log_error = [&] (const std::string& msg) { m_task_info->update_status(task_status::finished_with_error, - urd_error::system_error, ec); + urd_error::system_error, ec); std::string r_msg = "[{}] " + msg + ": {}"; LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 4f579b5..b5a07a0 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include "utils.hpp" #include "logger.hpp" @@ -38,6 +39,7 @@ #include "backends/posix-fs.hpp" #include "hermes.hpp" #include "rpcs.hpp" +#include "tar-archive.hpp" #include "local-path-to-remote-resource.hpp" @@ -68,6 +70,8 @@ local_path_to_remote_resource_transferor::transfer( const std::shared_ptr& src, const std::shared_ptr& dst) const { + using utils::tar; + (void) auth; const auto& d_src = @@ -75,6 +79,40 @@ local_path_to_remote_resource_transferor::transfer( const auto& d_dst = reinterpret_cast(*dst); + std::string input_path = d_src.canonical_path().string(); + + if(src->is_collection()) { + LOGGER_DEBUG("[{}] Creating archive for local directory", + task_info->id()); + + std::error_code ec; + + bfs::path ar_path = + "/tmp" / bfs::unique_path("norns-archive-%%%%-%%%%-%%%%.tar"); + + tar ar(ar_path, tar::create, ec); + + if(ec) { + LOGGER_ERROR("Failed to create archive: {}", + logger::errno_message(ec.value())); + return ec; + } + + LOGGER_INFO("Archive created in {}", ar.path()); + + ar.add_directory(d_src.canonical_path(), + d_dst.name(), + ec); + + if(ec) { + LOGGER_ERROR("Failed to add directory to archive: {}", + logger::errno_message(ec.value())); + return ec; + } + + input_path = ar.path().string(); + } + LOGGER_DEBUG("[{}] start_transfer: {} -> {}", task_info->id(), d_src.canonical_path(), d_dst.to_string()); @@ -82,7 +120,7 @@ local_path_to_remote_resource_transferor::transfer( try { std::error_code ec; - hermes::mapped_buffer input_data(d_src.canonical_path().string(), + hermes::mapped_buffer input_data(input_path, hermes::access_mode::read_only, &ec); @@ -104,6 +142,7 @@ local_path_to_remote_resource_transferor::transfer( d_dst.parent()->nsid(), static_cast(backend_type::posix_filesystem), static_cast(data::resource_type::local_posix_path), + d_src.is_collection(), d_dst.name(), buffers); @@ -122,23 +161,34 @@ local_path_to_remote_resource_transferor::transfer( LOGGER_DEBUG("};"); LOGGER_FLUSH(); - auto start = std::chrono::steady_clock::now(); - auto rpc = m_network_endpoint->post(endp, args); auto resp = rpc.get(); - double usecs = std::chrono::duration( - std::chrono::steady_clock::now() - start).count(); - - task_info->record_transfer(input_data.size(), usecs); + task_info->record_transfer(input_data.size(), + resp.at(0).elapsed_time()); LOGGER_DEBUG("Remote request completed with output " "{{status: {}, task_error: {}, sys_errnum: {}}} " "({} bytes, {} usecs)", resp.at(0).status(), resp.at(0).task_error(), - resp.at(0).sys_errnum(), input_data.size(), usecs); + resp.at(0).sys_errnum(), input_data.size(), + resp.at(0).elapsed_time()); + + if(src->is_collection()) { + boost::system::error_code bec; + + bfs::remove(input_path, bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + input_path, logger::errno_message(ec.value())); + //TODO + return std::make_error_code( + static_cast(bec.value())); + } + } return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 627bac8..d33087b 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -33,6 +33,7 @@ #include "io/task-stats.hpp" #include "hermes.hpp" #include "rpcs.hpp" +#include "tar-archive.hpp" #include "remote-resource-to-local-path.hpp" namespace { @@ -121,6 +122,8 @@ remote_resource_to_local_path_transferor::transfer( (void) src; (void) dst; + using utils::tar; + const auto& d_src = reinterpret_cast(*src); const auto& d_dst = @@ -144,22 +147,21 @@ remote_resource_to_local_path_transferor::transfer( assert(remote_buffers.count() == 1); - LOGGER_DEBUG("creating local resource: {}", d_dst.canonical_path()); + bool is_collection = d_src.is_collection(); + + bfs::path output_path = + is_collection ? "/tmp/test.tar" : d_dst.canonical_path(); + + LOGGER_DEBUG("creating local resource: {}", output_path); std::error_code ec; std::shared_ptr output_data; std::tie(ec, output_data) = - ::create_file(d_dst.canonical_path(), remote_buffers.size()); + ::create_file(output_path, remote_buffers.size()); if(ec) { - if(req.requires_response()) { - m_remote_endpoint->respond( - std::move(req), - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(ec.value())); - } + *ctx = std::move(req); return ec; } @@ -172,23 +174,89 @@ remote_resource_to_local_path_transferor::transfer( hermes::exposed_memory local_buffers = m_remote_endpoint->expose(bufseq, hermes::access_mode::write_only); - LOGGER_DEBUG("pulling remote data into {}", d_dst.canonical_path()); + LOGGER_DEBUG("pulling remote data into {}", output_path); + + auto start = std::chrono::steady_clock::now(); // N.B. IMPORTANT: we NEED to capture output_data by value here so that // the mapped_buffer doesn't get released before completion_callback() // is called. - const auto completion_callback = [this, output_data]( + const auto completion_callback = + [this, is_collection, output_path, d_dst, output_data, start]( hermes::request&& req) { + uint32_t usecs = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + + // default response + rpc::remote_transfer::output out( + static_cast(task_status::finished), + static_cast(urd_error::success), + 0, + usecs); + //TODO: hermes offers no way to check for an error yet - LOGGER_DEBUG("pull succeeded"); + LOGGER_DEBUG("Transfer completed ({} usecs)", usecs); + + if(is_collection) { + std::error_code ec; + boost::system::error_code bec; + tar ar(output_path, tar::open, ec); + + if(ec) { + LOGGER_ERROR("Failed to open archive {}: {}", + output_path, logger::errno_message(ec.value())); + + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + goto respond; + } + + ar.extract(d_dst.parent()->mount(), ec); + + if(ec) { + LOGGER_ERROR("Failed to extact archive into {}: {}", + ar.path(), d_dst.parent()->mount(), + logger::errno_message(ec.value())); + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + goto respond; + } + + LOGGER_DEBUG("Archive {} extracted into {}", + ar.path(), d_dst.parent()->mount()); + + bfs::remove(ar.path(), bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + ar.path(), logger::errno_message(ec.value())); + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + } + } +respond: if(req.requires_response()) { m_remote_endpoint->respond( std::move(req), static_cast(task_status::finished), static_cast(urd_error::success), - 0); + 0, + usecs); } }; diff --git a/src/io/transferors/tar-archive.cpp b/src/io/transferors/tar-archive.cpp new file mode 100644 index 0000000..54bc38a --- /dev/null +++ b/src/io/transferors/tar-archive.cpp @@ -0,0 +1,108 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "utils.hpp" +#include "logger.hpp" +#include "tar-archive.hpp" + +namespace norns { +namespace utils { + +tar::tar(const bfs::path& filename, + openmode op, + std::error_code& ec) : + m_path(filename) { + + constexpr const std::array flags = { + {O_WRONLY | O_CREAT| O_EXCL, O_RDONLY} + }; + + constexpr const std::array modes = { + {S_IRUSR | S_IWUSR, 0} + }; + + if(tar_open(&m_tar, m_path.c_str(), NULL, + flags[static_cast(op)], + modes[static_cast(op)], TAR_GNU) != 0) { + ec = std::make_error_code(static_cast(errno)); + LOGGER_ERROR("Failed to open archive for writing: {}", + logger::errno_message(ec.value())); + return; + } +} + +void +tar::add_directory(const bfs::path& real_dir, + const bfs::path& archive_dir, + std::error_code& ec) { + + const bfs::path rd = + norns::utils::remove_trailing_separator(real_dir); + const bfs::path ad = + norns::utils::remove_trailing_separator(archive_dir); + + if(tar_append_tree(m_tar, const_cast(rd.c_str()), + const_cast(ad.c_str())) != 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } + + ec = std::make_error_code(static_cast(0)); +} + +void +tar::extract(const bfs::path& parent_dir, + std::error_code& ec) { + + if(m_tar == nullptr) { + ec = std::make_error_code(static_cast(EINVAL)); + return; + } + + if(tar_extract_all(m_tar, const_cast(parent_dir.c_str())) != 0) { + ec = std::make_error_code(static_cast(errno)); + } +} + + +bfs::path +tar::path() const { + return m_path; +} + +tar::~tar() { + + if(m_tar != nullptr) { + if(tar_close(m_tar) != 0) { + LOGGER_ERROR("Failed to close TAR archive: {}", + logger::errno_message(errno)); + } + } +} + +} // namespace utils +} // namespace norns diff --git a/src/io/transferors/tar-archive.hpp b/src/io/transferors/tar-archive.hpp new file mode 100644 index 0000000..d736c87 --- /dev/null +++ b/src/io/transferors/tar-archive.hpp @@ -0,0 +1,68 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include +#include + +namespace bfs = boost::filesystem; + +namespace norns { +namespace utils { + +struct tar { + + enum class openmode : int { + create = 0, + open = 1, + }; + + constexpr static openmode create = openmode::create; + constexpr static openmode open = openmode::open; + + tar(const bfs::path& filename, openmode op, std::error_code& ec); + + void + add_directory(const bfs::path& real_dir, + const bfs::path& archive_dir, + std::error_code& ec); + + void + extract(const bfs::path& parent_dir, + std::error_code& ec); + + bfs::path + path() const; + + ~tar(); + + TAR* m_tar = nullptr; + bfs::path m_path; +}; + +} // namespace utils +} // namespace norns diff --git a/src/logger.hpp b/src/logger.hpp index 9a6100f..7a04567 100644 --- a/src/logger.hpp +++ b/src/logger.hpp @@ -221,6 +221,15 @@ do { \ m_internal_logger->error(fmt, args...); } + static inline std::string + errno_message(int errno_value) { + // 1024 should be more than enough for most locales + constexpr const std::size_t MAX_ERROR_MSG = 1024; + std::array errstr; + char* msg = strerror_r(errno_value, errstr.data(), MAX_ERROR_MSG); + return std::string{msg}; + } + template inline void error_errno(const char* fmt, const Args&... args) { diff --git a/src/resources/remote_resource/detail/remote-resource-impl.cpp b/src/resources/remote_resource/detail/remote-resource-impl.cpp index 5e93041..aaa4f3e 100644 --- a/src/resources/remote_resource/detail/remote-resource-impl.cpp +++ b/src/resources/remote_resource/detail/remote-resource-impl.cpp @@ -49,7 +49,7 @@ remote_resource::resource_impl( m_address(rinfo->m_address), m_nsid(rinfo->m_nsid), m_buffers(rinfo->m_buffers), - m_is_collection(rinfo->m_buffers.count() > 1), + m_is_collection(rinfo->m_is_collection), m_parent(std::static_pointer_cast(std::move(parent))) { } std::string diff --git a/src/resources/remote_resource/detail/remote-resource-info.cpp b/src/resources/remote_resource/detail/remote-resource-info.cpp index d85662d..5f1f023 100644 --- a/src/resources/remote_resource/detail/remote-resource-info.cpp +++ b/src/resources/remote_resource/detail/remote-resource-info.cpp @@ -35,20 +35,23 @@ namespace detail { /*! Remote path data */ remote_resource_info::remote_resource_info( - std::string address, - std::string nsid, - std::string name) : + const std::string& address, + const std::string& nsid, + const std::string& name) : m_address(address), m_nsid(nsid), + m_is_collection(false), m_name(name) { } remote_resource_info::remote_resource_info( - std::string address, - std::string nsid, - std::string name, - hermes::exposed_memory buffers) : + const std::string& address, + const std::string& nsid, + bool is_collection, + const std::string& name, + const hermes::exposed_memory& buffers) : m_address(address), m_nsid(nsid), + m_is_collection(is_collection), m_name(name), m_buffers(buffers) { } @@ -84,6 +87,11 @@ remote_resource_info::name() const { return m_name; } +bool +remote_resource_info::is_collection() const { + return m_is_collection; +} + bool remote_resource_info::has_buffers() const { return m_buffers.count() != 0; diff --git a/src/resources/remote_resource/detail/remote-resource-info.hpp b/src/resources/remote_resource/detail/remote-resource-info.hpp index 78359f4..87f5f85 100644 --- a/src/resources/remote_resource/detail/remote-resource-info.hpp +++ b/src/resources/remote_resource/detail/remote-resource-info.hpp @@ -43,14 +43,15 @@ namespace detail { /*! Local filesystem path data */ struct remote_resource_info : public resource_info { - remote_resource_info(std::string address, - std::string nsid, - std::string name); - - remote_resource_info(std::string address, - std::string nsid, - std::string name, - hermes::exposed_memory buffers); + remote_resource_info(const std::string& address, + const std::string& nsid, + const std::string& name); + + remote_resource_info(const std::string& address, + const std::string& nsid, + bool is_collection, + const std::string& name, + const hermes::exposed_memory& buffers); ~remote_resource_info(); resource_type type() const override final; std::string nsid() const override final; @@ -63,6 +64,9 @@ struct remote_resource_info : public resource_info { std::string name() const; + bool + is_collection() const; + bool has_buffers() const; @@ -71,6 +75,7 @@ struct remote_resource_info : public resource_info { std::string m_address; std::string m_nsid; + bool m_is_collection; std::string m_name; hermes::exposed_memory m_buffers; }; diff --git a/src/rpcs.hpp b/src/rpcs.hpp index ab02d8e..bbf6a3a 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -37,15 +37,17 @@ MERCURY_GEN_PROC(remote_transfer_in_t, ((hg_const_string_t) (address)) ((hg_const_string_t) (in_nsid)) ((hg_const_string_t) (out_nsid)) - ((uint32_t) (backend_type)) - ((uint32_t) (resource_type)) + ((uint32_t) (backend_type)) + ((uint32_t) (resource_type)) + ((hg_bool_t) (is_collection)) ((hg_const_string_t) (resource_name)) - ((hg_bulk_t) (buffers))) + ((hg_bulk_t) (buffers))) MERCURY_GEN_PROC(remote_transfer_out_t, ((uint32_t) (status)) ((uint32_t) (task_error)) - ((uint32_t) (sys_errnum))) + ((uint32_t) (sys_errnum)) + ((uint32_t) (elapsed_time))) }} // namespace hermes::detail @@ -98,6 +100,7 @@ struct remote_transfer { const std::string& out_nsid, uint32_t backend_type, uint32_t resource_type, + uint32_t is_collection, const std::string& resource_name, const hermes::exposed_memory& buffers) : m_address(address), @@ -105,6 +108,7 @@ struct remote_transfer { m_out_nsid(out_nsid), m_backend_type(backend_type), m_resource_type(resource_type), + m_is_collection(is_collection), m_resource_name(resource_name), m_buffers(buffers) { @@ -121,11 +125,13 @@ struct remote_transfer { m_out_nsid(std::move(rhs.m_out_nsid)), m_backend_type(std::move(rhs.m_backend_type)), m_resource_type(std::move(rhs.m_resource_type)), + m_is_collection(std::move(rhs.m_is_collection)), m_resource_name(std::move(rhs.m_resource_name)), m_buffers(std::move(rhs.m_buffers)) { rhs.m_backend_type = 0; rhs.m_resource_type = 0; + rhs.m_is_collection = false; this->print("this", __PRETTY_FUNCTION__); rhs.print("rhs", __PRETTY_FUNCTION__); @@ -137,6 +143,7 @@ struct remote_transfer { m_out_nsid(other.m_out_nsid), m_backend_type(other.m_backend_type), m_resource_type(other.m_resource_type), + m_is_collection(other.m_is_collection), m_resource_name(other.m_resource_name), m_buffers(other.m_buffers) { @@ -153,11 +160,13 @@ struct remote_transfer { m_out_nsid = std::move(rhs.m_out_nsid); m_backend_type = std::move(rhs.m_backend_type); m_resource_type = std::move(rhs.m_resource_type); + m_is_collection = std::move(rhs.m_is_collection); m_resource_name = std::move(rhs.m_resource_name); m_buffers = std::move(rhs.m_buffers); rhs.m_backend_type = 0; rhs.m_resource_type = 0; + rhs.m_is_collection = false; } this->print("this", __PRETTY_FUNCTION__); @@ -175,6 +184,7 @@ struct remote_transfer { m_out_nsid = other.m_out_nsid; m_backend_type = other.m_backend_type; m_resource_type = other.m_resource_type; + m_is_collection = other.m_is_collection; m_resource_name = other.m_resource_name; m_buffers = other.m_buffers; } @@ -216,6 +226,11 @@ struct remote_transfer { return m_resource_type; } + bool + is_collection() const { + return m_is_collection; + } + std::string resource_name() const { return m_resource_name; @@ -248,6 +263,8 @@ struct remote_transfer { m_backend_type); HERMES_DEBUG2(" m_resource_type: {},", m_resource_type); + HERMES_DEBUG2(" m_is_collection: {},", + m_is_collection); HERMES_DEBUG2(" m_resource_name: \"{}\" ({} -> {}),", m_resource_name, fmt::ptr(&m_resource_name), fmt::ptr(m_resource_name.c_str())); @@ -264,6 +281,7 @@ struct remote_transfer { m_out_nsid(other.out_nsid), m_backend_type(other.backend_type), m_resource_type(other.resource_type), + m_is_collection(other.is_collection), m_resource_name(other.resource_name), m_buffers(other.buffers) { @@ -278,6 +296,8 @@ struct remote_transfer { m_backend_type); HERMES_DEBUG(" m_resource_type: {},", m_resource_type); + HERMES_DEBUG(" m_is_collection: {},", + m_is_collection); HERMES_DEBUG(" m_buffers: {...},"); HERMES_DEBUG("}}"); } @@ -289,6 +309,7 @@ struct remote_transfer { m_out_nsid.c_str(), m_backend_type, m_resource_type, + m_is_collection, m_resource_name.c_str(), hg_bulk_t(m_buffers)}; } @@ -300,6 +321,7 @@ struct remote_transfer { std::string m_out_nsid; uint32_t m_backend_type; uint32_t m_resource_type; + bool m_is_collection; std::string m_resource_name; hermes::exposed_memory m_buffers; }; @@ -312,10 +334,12 @@ struct remote_transfer { public: output(uint32_t status, uint32_t task_error, - uint32_t sys_errnum) : + uint32_t sys_errnum, + uint32_t elapsed_time) : m_status(status), m_task_error(task_error), - m_sys_errnum(sys_errnum) {} + m_sys_errnum(sys_errnum), + m_elapsed_time(elapsed_time) {} uint32_t status() const { @@ -342,6 +366,11 @@ struct remote_transfer { return m_sys_errnum; } + uint32_t + elapsed_time() const { + return m_elapsed_time; + } + void set_sys_errnum(uint32_t errnum) { m_sys_errnum = errnum; @@ -352,17 +381,19 @@ struct remote_transfer { m_status = out.status; m_task_error = out.task_error; m_sys_errnum = out.sys_errnum; + m_elapsed_time = out.elapsed_time; } explicit operator hermes::detail::remote_transfer_out_t() { - return {m_status, m_task_error, m_sys_errnum}; + return {m_status, m_task_error, m_sys_errnum, m_elapsed_time}; } private: uint32_t m_status; uint32_t m_task_error; uint32_t m_sys_errnum; + uint32_t m_elapsed_time; }; }; diff --git a/src/urd.cpp b/src/urd.cpp index 0a8aa21..7f209b9 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -692,12 +692,12 @@ urd::remote_transfer_handler(hermes::request&& req) { args.resource_type(), utils::to_string( static_cast(args.resource_type()))); + LOGGER_DEBUG(" is_collection: {}", args.is_collection()); LOGGER_DEBUG(" rname: \"{}\",", args.resource_name()); LOGGER_DEBUG(" buffers: {{...}}"); LOGGER_DEBUG("};"); LOGGER_FLUSH(); - urd_error rv = urd_error::success; boost::optional t; std::shared_ptr src_backend; @@ -712,7 +712,7 @@ urd::remote_transfer_handler(hermes::request&& req) { switch(rtype) { case data::resource_type::remote_resource: return std::make_shared( - args.address(), args.in_nsid(), + args.address(), args.in_nsid(), args.is_collection(), args.resource_name(), args.buffers()); case data::resource_type::local_posix_path: case data::resource_type::shared_posix_path: @@ -733,6 +733,7 @@ urd::remote_transfer_handler(hermes::request&& req) { m_network_endpoint->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), + 0, 0); return; } @@ -754,6 +755,7 @@ urd::remote_transfer_handler(hermes::request&& req) { m_network_endpoint->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), + 0, 0); return; } @@ -773,13 +775,16 @@ urd::remote_transfer_handler(hermes::request&& req) { // run the task and check that it started correctly (*t)(); + auto req = std::move(*ctx); + if(t->info()->status() == io::task_status::finished_with_error) { rv = urd_error::no_such_namespace; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); m_network_endpoint->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(t->info()->task_error()), - static_cast(t->info()->sys_error().value())); + static_cast(t->info()->sys_error().value()), + 0); } } } diff --git a/src/utils.cpp b/src/utils.cpp index b8f5bb7..a42f9f8 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -152,6 +152,26 @@ boost::filesystem::path lexical_normalize(const boost::filesystem::path& pathnam } +// remove a trailing separator +// (adapted from boost::filesystem::path::remove_trailing_separator()) +boost::filesystem::path +remove_trailing_separator(const boost::filesystem::path& pathname) { + + using boost::filesystem::path; + + std::string str{pathname.generic_string()}; + + auto is_directory_separator = [](path::value_type c) { + return c == '/'; + }; + + if(!str.empty() && is_directory_separator(str[str.size() - 1])) { + str.erase(str.size() - 1); + } + + return path{str}; +} + } // namespace utils } // namespace norns diff --git a/src/utils.hpp b/src/utils.hpp index 94f6c73..bc79a1c 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -54,6 +54,9 @@ std::string n2hexstr(T i, bool zero_pad=false) { boost::filesystem::path lexical_normalize(const boost::filesystem::path& pathname, bool as_directory=false); +boost::filesystem::path +remove_trailing_separator(const boost::filesystem::path& pathname); + } // namespace utils } // namespace norns diff --git a/tests/Makefile.am b/tests/Makefile.am index a3ae6d8..6cebe39 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -82,6 +82,7 @@ api_LDFLAGS = \ @BOOST_THREAD_LIB@ \ @BOOST_REGEX_LIB@ \ @PROTOBUF_LIBS@ \ + -no-install \ -Wl,-rpath,$(top_builddir)/lib/.libs \ $(top_builddir)/src/liburd_aux.la \ $(top_builddir)/lib/libnorns_debug.la \ @@ -146,6 +147,7 @@ core_SOURCES = \ $(END) core_LDFLAGS = \ + -no-install \ @BOOST_ASIO_LIB@ \ @BOOST_LDFLAGS@ \ @BOOST_FILESYSTEM_LIB@ \ diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp index 270624a..85e79be 100644 --- a/tests/api-copy-remote-data.cpp +++ b/tests/api-copy-remote-data.cpp @@ -138,18 +138,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", // create required output directories env.add_to_namespace(nsid1, dst_subdir1); -#if 0 - /**************************************************************************************************************/ - /* tests for error conditions */ - /**************************************************************************************************************/ + /**********************************************************************/ + /* tests for error conditions */ + /**********************************************************************/ // - trying to copy a non-existing file WHEN("copying a non-existing NORNS_LOCAL_PATH file") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_invalid_file.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_invalid_file.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -179,11 +179,12 @@ SCENARIO("copy local POSIX file to remote POSIX file", // - trying to copy a non-existing directory WHEN("copying a non-existing NORNS_LOCAL_PATH directory") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_invalid_dir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_invalid_dir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -213,11 +214,12 @@ SCENARIO("copy local POSIX file to remote POSIX file", // - trying to copy an empty directory WHEN("copying an empty NORNS_LOCAL_PATH directory") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_empty_dir.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_empty_dir.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -250,13 +252,15 @@ SCENARIO("copy local POSIX file to remote POSIX file", #ifdef __SETCAP_TESTS__ // - trying to copy a file from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH file from \"/\" without appropriate permissions to access it") { + WHEN("copying a NORNS_LOCAL_PATH file from \"/\" without appropriate " + "permissions to access it") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_file0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -270,7 +274,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("NORNS_SUCCESS is returned") { REQUIRE(rv == NORNS_SUCCESS); - THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { norns_stat_t stats; rv = norns_status(&task, &stats); @@ -286,13 +291,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", } // - trying to copy a file from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access it") { + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without " + "appropriate permissions to access it") { norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -322,13 +331,15 @@ SCENARIO("copy local POSIX file to remote POSIX file", } // - trying to copy a file from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access a parent") { + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without " + "appropriate permissions to access a parent") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_file2.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -342,7 +353,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("NORNS_SUCCESS is returned") { REQUIRE(rv == NORNS_SUCCESS); - THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { norns_stat_t stats; rv = norns_status(&task, &stats); @@ -358,13 +370,16 @@ SCENARIO("copy local POSIX file to remote POSIX file", } // - trying to copy a subdir from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH subdir from \"/\" without appropriate permissions to access it") { + WHEN("copying a NORNS_LOCAL_PATH subdir from \"/\" without " + "appropriate permissions to access it") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_subdir0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -378,7 +393,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("NORNS_SUCCESS is returned") { REQUIRE(rv == NORNS_SUCCESS); - THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { norns_stat_t stats; rv = norns_status(&task, &stats); @@ -394,13 +410,16 @@ SCENARIO("copy local POSIX file to remote POSIX file", } // - trying to copy a subdir from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access it") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir " + "without appropriate permissions to access it") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_subdir1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -430,13 +449,16 @@ SCENARIO("copy local POSIX file to remote POSIX file", } // - trying to copy a subdir from namespace root with invalid access permissions - WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access a parent") { + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without " + "appropriate permissions to access a parent") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_noperms_subdir2.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -450,7 +472,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("NORNS_SUCCESS is returned") { REQUIRE(rv == NORNS_SUCCESS); - THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { norns_stat_t stats; rv = norns_status(&task, &stats); @@ -464,17 +487,18 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } } -#endif // symlink leading out of namespace WHEN("copying a NORNS_LOCAL_PATH through a symbolic link that leads " "out of the src namespace") { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, out_symlink.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + out_symlink.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -547,7 +571,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://file0.txt // -> ns1://file1.txt = ns1://file1.txt WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at dst namespace's root (changing the name)") { + "another NORNS_LOCAL_PATH at dst namespace's root " + "(changing the name)") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, @@ -620,8 +645,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../d/file0.txt // -> ns1://file1.txt = ns1://file1.txt WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at dst namespace's root (changing the " - "name)") { + "to another NORNS_LOCAL_PATH at dst namespace's root (changing " + "the name)") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, @@ -876,13 +901,13 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } -#if 0 /**********************************************************************/ /* tests for directories */ /**********************************************************************/ // cp -r /a/contents.* -> / = /contents.* WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to dst namespace's root") { + "namespace's root to dst namespace's root\n" + " cp -r /a/contents.* -> / = /contents.* ") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, @@ -916,13 +941,15 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> / = /contents.* WHEN("copying the contents of a NORNS_LOCAL_PATH arbitrary subdir to " - "dst namespace's root") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + "dst namespace's root\n" + " cp -r /a/b/c/.../contents.* -> / = /contents.*") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_root.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_root.c_str())); norns_error_t rv = norns_submit(&task); @@ -937,8 +964,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = env.get_from_namespace(nsid1, dst_root); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_root); REQUIRE(compare_directories(src, dst) == true); } @@ -948,14 +977,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/contents.* -> /c = /c/contents.* // (c did not exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " - "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " + "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + "namespace's root while changing its name\n" + " cp -r /a/contents.* -> /c = /c/contents.*") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_subdir0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir0.c_str())); norns_error_t rv = norns_submit(&task); @@ -970,8 +1002,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir0); - bfs::path dst = env.get_from_namespace(nsid1, dst_subdir0); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); REQUIRE(compare_directories(src, dst) == true); } @@ -981,14 +1015,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/contents.* -> /c = /c/contents.* // (c did exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " - "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " + "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + "namespace's root while changing its name\n" + " cp -r /a/contents.* -> /c / /c/contents.*") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_subdir1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir1.c_str())); norns_error_t rv = norns_submit(&task); @@ -1003,8 +1040,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir0); - bfs::path dst = env.get_from_namespace(nsid1, dst_subdir1); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); REQUIRE(compare_directories(src, dst) == true); } @@ -1014,14 +1053,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* // (c did not exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " - "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " + "namespace's root to a NORNS_REMOTE_PATH subdir at dst " + "namespace's root while changing its name\n" + "cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_subdir0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1036,8 +1078,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = env.get_from_namespace(nsid1, dst_subdir0); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); REQUIRE(compare_directories(src, dst) == true); } @@ -1047,14 +1091,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* // (c did exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src namespace's root to " - "another NORNS_LOCAL_PATH subdir at dst namespace's root while changing its name") { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " + "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + "namespace's root while changing its name:" + " cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_subdir1.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir1.c_str())); norns_error_t rv = norns_submit(&task); @@ -1069,8 +1116,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", REQUIRE(rv == NORNS_SUCCESS); THEN("Copied files are identical to original") { - bfs::path src = env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = env.get_from_namespace(nsid1, dst_subdir1); + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); REQUIRE(compare_directories(src, dst) == true); } @@ -1084,11 +1133,13 @@ SCENARIO("copy local POSIX file to remote POSIX file", WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " "through a symlink also located at '/'" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_root0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1104,8 +1155,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root0); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_files(src, dst) == true); } @@ -1113,14 +1166,16 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - WHEN("copying a single NORNS_LOCAL_PATH subdir from src namespace's '/' " - "through a symlink also located at '/'" ) { - - norns_op_t task_op = NORNS_IOTASK_COPY; + WHEN("copying a single NORNS_LOCAL_PATH subdir from src " + "namespace's '/' through a symlink also located at '/'" ) { - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_root1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1136,8 +1191,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Directories are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root1); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_directories(src, dst) == true); } @@ -1150,9 +1207,13 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_root2.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1168,8 +1229,10 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Directories are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_root2); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_directories(src, dst) == true); } @@ -1180,11 +1243,13 @@ SCENARIO("copy local POSIX file to remote POSIX file", WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " "through a symlink located in a subdir" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1200,8 +1265,11 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Files are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir0); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_files(src, dst) == true); } @@ -1209,14 +1277,16 @@ SCENARIO("copy local POSIX file to remote POSIX file", } } - WHEN("copying a single NORNS_LOCAL_PATH subdir from src namespace's '/' " - "through a symlink also located at subdir" ) { + WHEN("copying a single NORNS_LOCAL_PATH subdir from src " + "namespace's '/' through a symlink also located at subdir" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir1.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1232,8 +1302,11 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Directories are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir1); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_directories(src, dst) == true); } @@ -1244,11 +1317,13 @@ SCENARIO("copy local POSIX file to remote POSIX file", WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" "through a symlink also located at a subdir" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir2.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); @@ -1264,15 +1339,17 @@ SCENARIO("copy local POSIX file to remote POSIX file", THEN("Directories are equal") { - bfs::path src = env.get_from_namespace(nsid0, src_symlink_at_subdir2); - bfs::path dst = env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); REQUIRE(compare_directories(src, dst) == true); } } } } -#endif env.notify_success(); } -- GitLab From 8f9a17a23eef2183029010906695d870ac54de55 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 19 Feb 2019 18:38:19 +0100 Subject: [PATCH 14/51] Add libtar-dev dependency to CI --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75dfc03..316cf88 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,6 +25,7 @@ before_script: protobuf-c-compiler libyaml-cpp-dev libyaml-dev + libtar-dev cmake - pushd . && @@ -154,6 +155,7 @@ test:ubuntu:latest: protobuf-c-compiler libyaml-cpp-dev libyaml-dev + libtar-dev libcap2-bin valgrind cmake -- GitLab From 45fdee94890873c3fa1c846df749b6fb84ba0529 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 19 Feb 2019 18:38:40 +0100 Subject: [PATCH 15/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e65be7d..f6a60b1 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ $ apt-get install -y libboost-system-dev libboost-filesystem-dev \ libboost-program-options-dev libboost-thread-dev \ libboost-regex-dev libprotobuf-dev protobuf-compiler \ libprotobuf-c-dev protobuf-c-compiler \ - libyaml-cpp-dev libyaml-dev + libyaml-cpp-dev libyaml-dev libtar-dev # Building and installing libfabric (required for Mercury's OFI/libfabric plugin) $ git clone https://github.com/ofiwg/libfabric.git && -- GitLab From 4f43a21a7e9193ce51296a6876fe2054afaea96b Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 22 Feb 2019 09:29:18 +0100 Subject: [PATCH 16/51] Add RPC to remotely stat() resources --- src/Makefile.am | 6 +- .../local-path-to-remote-resource.cpp | 1 - .../remote-resource-to-local-path.cpp | 1 - .../detail/local-path-impl.cpp | 10 + .../detail/local-path-impl.hpp | 1 + .../detail/memory-region-impl.cpp | 5 + .../detail/memory-region-impl.hpp | 1 + .../detail/remote-path-impl.cpp | 5 + .../detail/remote-path-impl.hpp | 1 + .../detail/remote-resource-impl.cpp | 5 + .../detail/remote-resource-impl.hpp | 1 + src/resources/resource.hpp | 1 + .../detail/shared-path-impl.cpp | 5 + .../detail/shared-path-impl.hpp | 1 + src/rpcs.cpp | 1 + src/rpcs.hpp | 246 ++++++++++++ src/urd.cpp | 72 +++- src/urd.hpp | 2 + src/utils.hpp | 6 + src/{io/transferors => utils}/tar-archive.cpp | 110 +++++- src/{io/transferors => utils}/tar-archive.hpp | 15 + tests/Makefile.am | 4 + tests/utils-tar.cpp | 349 ++++++++++++++++++ 23 files changed, 841 insertions(+), 8 deletions(-) rename src/{io/transferors => utils}/tar-archive.cpp (56%) rename src/{io/transferors => utils}/tar-archive.hpp (87%) create mode 100644 tests/utils-tar.cpp diff --git a/src/Makefile.am b/src/Makefile.am index 08c4df3..8aad795 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -168,8 +168,6 @@ liburd_aux_la_SOURCES = \ io/transferors/local-path-to-remote-resource.hpp \ io/transferors/remote-resource-to-local-path.cpp \ io/transferors/remote-resource-to-local-path.hpp \ - io/transferors/tar-archive.cpp \ - io/transferors/tar-archive.hpp \ io/transferors/memory-to-local-path.cpp \ io/transferors/memory-to-local-path.hpp \ io/transferors/memory-to-shared-path.cpp \ @@ -190,7 +188,9 @@ liburd_aux_la_SOURCES = \ urd.cpp \ urd.hpp \ utils.cpp \ - utils.hpp + utils.hpp \ + utils/tar-archive.cpp \ + utils/tar-archive.hpp nodist_liburd_aux_la_SOURCES = \ config/defaults.cpp \ diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index b5a07a0..2ba50ce 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -39,7 +39,6 @@ #include "backends/posix-fs.hpp" #include "hermes.hpp" #include "rpcs.hpp" -#include "tar-archive.hpp" #include "local-path-to-remote-resource.hpp" diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index d33087b..15109fe 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -33,7 +33,6 @@ #include "io/task-stats.hpp" #include "hermes.hpp" #include "rpcs.hpp" -#include "tar-archive.hpp" #include "remote-resource-to-local-path.hpp" namespace { diff --git a/src/resources/local_posix_path/detail/local-path-impl.cpp b/src/resources/local_posix_path/detail/local-path-impl.cpp index eadf4b3..6ba8ebf 100644 --- a/src/resources/local_posix_path/detail/local-path-impl.cpp +++ b/src/resources/local_posix_path/detail/local-path-impl.cpp @@ -36,6 +36,7 @@ namespace bfs = boost::filesystem; #include "local-path-info.hpp" #include "local-path-impl.hpp" #include "backends/posix-fs.hpp" +#include "utils.hpp" #include "logger.hpp" namespace norns { @@ -67,6 +68,15 @@ local_path_resource::is_collection() const { return m_is_collection; } +std::size_t +local_path_resource::packed_size() const { + std::error_code ec; + std::size_t sz = + utils::tar::estimate_size_once_packed(m_canonical_path, ec); + + return ec ? 0 : sz; +} + const std::shared_ptr local_path_resource::parent() const { return std::static_pointer_cast(m_parent); diff --git a/src/resources/local_posix_path/detail/local-path-impl.hpp b/src/resources/local_posix_path/detail/local-path-impl.hpp index df2e9b4..ddb4ddd 100644 --- a/src/resources/local_posix_path/detail/local-path-impl.hpp +++ b/src/resources/local_posix_path/detail/local-path-impl.hpp @@ -58,6 +58,7 @@ struct resource_impl : public resource { std::string name() const override final; resource_type type() const override final; bool is_collection() const override final; + std::size_t packed_size() const override final; const std::shared_ptr parent() const override final; std::string to_string() const override final; diff --git a/src/resources/memory_buffer/detail/memory-region-impl.cpp b/src/resources/memory_buffer/detail/memory-region-impl.cpp index 45d6e80..89f3417 100644 --- a/src/resources/memory_buffer/detail/memory-region-impl.cpp +++ b/src/resources/memory_buffer/detail/memory-region-impl.cpp @@ -58,6 +58,11 @@ bool memory_region_resource::is_collection() const { return false; } +std::size_t +memory_region_resource::packed_size() const { + return m_size; +} + const std::shared_ptr memory_region_resource::parent() const { return std::static_pointer_cast(m_parent); diff --git a/src/resources/memory_buffer/detail/memory-region-impl.hpp b/src/resources/memory_buffer/detail/memory-region-impl.hpp index 22b632a..1e08713 100644 --- a/src/resources/memory_buffer/detail/memory-region-impl.hpp +++ b/src/resources/memory_buffer/detail/memory-region-impl.hpp @@ -52,6 +52,7 @@ struct resource_impl : public resource { std::string name() const override final; resource_type type() const override final; bool is_collection() const override final; + std::size_t packed_size() const override final; const std::shared_ptr parent() const override final; std::string to_string() const override final; diff --git a/src/resources/remote_posix_path/detail/remote-path-impl.cpp b/src/resources/remote_posix_path/detail/remote-path-impl.cpp index 67b251c..e41949f 100644 --- a/src/resources/remote_posix_path/detail/remote-path-impl.cpp +++ b/src/resources/remote_posix_path/detail/remote-path-impl.cpp @@ -60,6 +60,11 @@ bool remote_path_resource::is_collection() const { return false; } +std::size_t +remote_path_resource::packed_size() const { + return 0; +} + const std::shared_ptr remote_path_resource::parent() const { return std::static_pointer_cast(m_parent); diff --git a/src/resources/remote_posix_path/detail/remote-path-impl.hpp b/src/resources/remote_posix_path/detail/remote-path-impl.hpp index 8865e02..b775f48 100644 --- a/src/resources/remote_posix_path/detail/remote-path-impl.hpp +++ b/src/resources/remote_posix_path/detail/remote-path-impl.hpp @@ -52,6 +52,7 @@ struct resource_impl : public resource { std::string name() const override final; resource_type type() const override final; bool is_collection() const override final; + std::size_t packed_size() const override final; const std::shared_ptr parent() const override final; std::string to_string() const override final; diff --git a/src/resources/remote_resource/detail/remote-resource-impl.cpp b/src/resources/remote_resource/detail/remote-resource-impl.cpp index aaa4f3e..510e5c8 100644 --- a/src/resources/remote_resource/detail/remote-resource-impl.cpp +++ b/src/resources/remote_resource/detail/remote-resource-impl.cpp @@ -67,6 +67,11 @@ remote_resource::is_collection() const { return m_is_collection; } +std::size_t +remote_resource::packed_size() const { + return 0; +} + const std::shared_ptr remote_resource::parent() const { return std::static_pointer_cast(m_parent); diff --git a/src/resources/remote_resource/detail/remote-resource-impl.hpp b/src/resources/remote_resource/detail/remote-resource-impl.hpp index 86201f1..a41fd57 100644 --- a/src/resources/remote_resource/detail/remote-resource-impl.hpp +++ b/src/resources/remote_resource/detail/remote-resource-impl.hpp @@ -58,6 +58,7 @@ struct resource_impl : public resource { std::string name() const override final; resource_type type() const override final; bool is_collection() const override final; + std::size_t packed_size() const override final; const std::shared_ptr parent() const override final; std::string to_string() const override final; diff --git a/src/resources/resource.hpp b/src/resources/resource.hpp index 19d5161..8a3c553 100644 --- a/src/resources/resource.hpp +++ b/src/resources/resource.hpp @@ -49,6 +49,7 @@ struct resource : public std::enable_shared_from_this { virtual resource_type type() const = 0; virtual std::string name() const = 0; virtual bool is_collection() const = 0; + virtual std::size_t packed_size() const = 0; virtual const std::shared_ptr parent() const = 0; virtual std::string to_string() const = 0; }; diff --git a/src/resources/shared_posix_path/detail/shared-path-impl.cpp b/src/resources/shared_posix_path/detail/shared-path-impl.cpp index 6834c39..f5fca7a 100644 --- a/src/resources/shared_posix_path/detail/shared-path-impl.cpp +++ b/src/resources/shared_posix_path/detail/shared-path-impl.cpp @@ -62,6 +62,11 @@ bool shared_path_resource::is_collection() const { return m_is_collection; } +std::size_t +shared_path_resource::packed_size() const { + return 0; +} + const std::shared_ptr shared_path_resource::parent() const { return std::static_pointer_cast(m_parent); diff --git a/src/resources/shared_posix_path/detail/shared-path-impl.hpp b/src/resources/shared_posix_path/detail/shared-path-impl.hpp index c55251a..9c4d4ed 100644 --- a/src/resources/shared_posix_path/detail/shared-path-impl.hpp +++ b/src/resources/shared_posix_path/detail/shared-path-impl.hpp @@ -58,6 +58,7 @@ struct resource_impl : public resource { std::string name() const override final; resource_type type() const override final; bool is_collection() const override final; + std::size_t packed_size() const override final; const std::shared_ptr parent() const override final; std::string to_string() const override final; diff --git a/src/rpcs.cpp b/src/rpcs.cpp index 664a7c4..f53ca92 100644 --- a/src/rpcs.cpp +++ b/src/rpcs.cpp @@ -10,6 +10,7 @@ namespace hermes { namespace detail { void register_user_request_types() { (void) registered_requests().add(); + (void) registered_requests().add(); } }} // namespace hermes::detail diff --git a/src/rpcs.hpp b/src/rpcs.hpp index bbf6a3a..77fb482 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -49,6 +49,16 @@ MERCURY_GEN_PROC(remote_transfer_out_t, ((uint32_t) (sys_errnum)) ((uint32_t) (elapsed_time))) +MERCURY_GEN_PROC(resource_stat_in_t, + ((hg_const_string_t) (address)) + ((hg_const_string_t) (nsid)) + ((uint32_t) (resource_type)) + ((hg_const_string_t) (resource_name))) + +MERCURY_GEN_PROC(resource_stat_out_t, + ((uint32_t) (return_value)) + ((uint64_t) (packed_size))) + }} // namespace hermes::detail @@ -397,6 +407,242 @@ struct remote_transfer { }; }; +struct resource_stat { + // forward declarations of public input/output types for this RPC + class input; + class output; + + // traits used so that the engine knows what to do with the RPC + using self_type = resource_stat; + using handle_type = hermes::rpc_handle; + using input_type = input; + using output_type = output; + using mercury_input_type = hermes::detail::resource_stat_in_t; + using mercury_output_type = hermes::detail::resource_stat_out_t; + + // RPC public identifier + constexpr static const uint16_t public_id = 44; + + // RPC internal Mercury identifier + constexpr static const uint16_t mercury_id = public_id; + + // RPC name + constexpr static const auto name = "resource_stat"; + + // requires response? + constexpr static const auto requires_response = true; + + // Mercury callback to serialize input arguments + constexpr static const auto mercury_in_proc_cb = + HG_GEN_PROC_NAME(resource_stat_in_t); + + // Mercury callback to serialize output arguments + constexpr static const auto mercury_out_proc_cb = + HG_GEN_PROC_NAME(resource_stat_out_t); + + class input { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + input(const std::string& address, + const std::string& nsid, + uint32_t resource_type, + const std::string& resource_name) : + m_address(address), + m_nsid(nsid), + m_resource_type(resource_type), + m_resource_name(resource_name) { + +#ifdef HERMES_DEBUG_BUILD + this->print("this", __PRETTY_FUNCTION__); +#endif + + } + +#ifdef HERMES_DEBUG_BUILD + input(input&& rhs) : + m_address(std::move(rhs.m_address)), + m_nsid(std::move(rhs.m_nsid)), + m_resource_type(std::move(rhs.m_resource_type)), + m_resource_name(std::move(rhs.m_resource_name)) { + + rhs.m_resource_type = 0; + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + } + + input(const input& other) : + m_address(other.m_address), + m_nsid(other.m_nsid), + m_resource_type(other.m_resource_type), + m_resource_name(other.m_resource_name) { + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + } + + input& + operator=(input&& rhs) { + + if(this != &rhs) { + m_address = std::move(rhs.m_address); + m_nsid = std::move(rhs.m_nsid); + m_resource_type = std::move(rhs.m_resource_type); + m_resource_name = std::move(rhs.m_resource_name); + + rhs.m_resource_type = 0; + } + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + + return *this; + } + + input& + operator=(const input& other) { + + if(this != &other) { + m_address = other.m_address; + m_nsid = other.m_nsid; + m_resource_type = other.m_resource_type; + m_resource_name = other.m_resource_name; + } + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + + return *this; + } +#else // HERMES_DEBUG_BUILD + input(input&& rhs) = default; + input(const input& other) = default; + input& operator=(input&& rhs) = default; + input& operator=(const input& other) = default; +#endif // ! HERMES_DEBUG_BUILD + + std::string + address() const { + return m_address; + } + + std::string + nsid() const { + return m_nsid; + } + + uint32_t + resource_type() const { + return m_resource_type; + } + + std::string + resource_name() const { + return m_resource_name; + } + +#ifdef HERMES_DEBUG_BUILD + void + print(const std::string& id, + const std::string& caller = "") const { + + (void) id; + auto c = caller.empty() ? "unknown_caller" : caller; + + HERMES_DEBUG2("{}, {} ({}) = {{", caller, id, fmt::ptr(this)); + HERMES_DEBUG2(" m_address: \"{}\" ({} -> {}),", + m_address, fmt::ptr(&m_address), + fmt::ptr(m_address.c_str())); + HERMES_DEBUG2(" m_nsid: \"{}\" ({} -> {}),", + m_nsid, fmt::ptr(&m_nsid), + fmt::ptr(m_nsid.c_str())); + HERMES_DEBUG2(" m_resource_type: {},", + m_resource_type); + HERMES_DEBUG2(" m_resource_name: \"{}\" ({} -> {}),", + m_resource_name, fmt::ptr(&m_resource_name), + fmt::ptr(m_resource_name.c_str())); + HERMES_DEBUG2("}}"); + } +#endif // ! HERMES_DEBUG_BUILD + +//TODO: make private + explicit + input(const hermes::detail::resource_stat_in_t& other) : + m_address(other.address), + m_nsid(other.nsid), + m_resource_type(other.resource_type), + m_resource_name(other.resource_name) { + + HERMES_DEBUG("input::input(const hermes::detail::resource_stat_in_t&){{"); + HERMES_DEBUG(" m_address: {} ({}),", + m_address, fmt::ptr(&m_address)); + HERMES_DEBUG(" m_nsid: {} ({}),", + m_nsid, fmt::ptr(&m_nsid)); + HERMES_DEBUG(" m_resource_type: {},", + m_resource_type); + HERMES_DEBUG(" m_resource_name: \"{}\" ({} -> {}),", + m_resource_name, fmt::ptr(&m_resource_name), + fmt::ptr(m_resource_name.c_str())); + HERMES_DEBUG("}}"); + } + + explicit + operator hermes::detail::resource_stat_in_t() { + return {m_address.c_str(), + m_nsid.c_str(), + m_resource_type, + m_resource_name.c_str()}; + } + + + private: + std::string m_address; + std::string m_nsid; + uint32_t m_resource_type; + std::string m_resource_name; + }; + + class output { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + output(uint32_t return_value, + uint64_t packed_size) : + m_return_value(return_value), + m_packed_size(packed_size) {} + + uint32_t + return_value() const { + return m_return_value; + } + + uint64_t + packed_size() const { + return m_packed_size; + } + + explicit + output(const hermes::detail::resource_stat_out_t& out) { + m_return_value = out.return_value; + m_packed_size = out.packed_size; + } + + explicit + operator hermes::detail::resource_stat_out_t() { + return {m_return_value, m_packed_size}; + } + + private: + uint32_t m_return_value; + uint64_t m_packed_size; + }; +}; + } // namespace rpc } // namespace norns diff --git a/src/urd.cpp b/src/urd.cpp index 7f209b9..7e5eb9d 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -778,7 +778,7 @@ urd::remote_transfer_handler(hermes::request&& req) { auto req = std::move(*ctx); if(t->info()->status() == io::task_status::finished_with_error) { - rv = urd_error::no_such_namespace; + rv = t->info()->task_error(); LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); m_network_endpoint->respond(std::move(req), static_cast(io::task_status::finished_with_error), @@ -789,6 +789,71 @@ urd::remote_transfer_handler(hermes::request&& req) { } } +void +urd::resource_stat_handler(hermes::request&& req) { + + const auto args = req.args(); + + LOGGER_WARN("incoming rpc::resource_stat(\"{}:{}\")", + args.nsid(), + args.resource_name()); + + urd_error rv = urd_error::success; + boost::optional> dst_backend; + + { + boost::shared_lock lock(m_namespace_mgr_mutex); + dst_backend = m_namespace_mgr->find(args.nsid()); + } + + if(!dst_backend) { + rv = urd_error::no_such_namespace; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(rv), + 0); + return; + } + + auto rtype = static_cast(args.resource_type()); + + const auto create_rinfo = + [&](const data::resource_type& rtype) -> + std::shared_ptr { + + switch(rtype) { + case data::resource_type::local_posix_path: + case data::resource_type::shared_posix_path: + return std::make_shared( + args.nsid(), args.resource_name()); + default: + rv = urd_error::not_supported; + return {}; + } + }; + + auto rinfo = create_rinfo(rtype); + + std::error_code ec; + auto rsrc = (*dst_backend)->get_resource(rinfo, ec); + + if(ec) { + LOGGER_ERROR("Failed to access resource {}", rinfo->to_string()); + rv = urd_error::snafu; + + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(rv), + 0); + return; + } + + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(urd_error::success), + rsrc->packed_size()); +} + void urd::configure(const config::settings& settings) { m_settings = std::make_shared(settings); @@ -1014,6 +1079,11 @@ void urd::init_event_handlers() { std::bind(&urd::remote_transfer_handler, this, std::placeholders::_1)); + m_network_endpoint->register_handler( + std::bind(&urd::resource_stat_handler, this, + std::placeholders::_1)); + + // signal handlers must be installed AFTER daemonizing LOGGER_INFO(" * Installing signal handlers..."); diff --git a/src/urd.hpp b/src/urd.hpp index 9bc01ba..fa0c484 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -73,6 +73,7 @@ namespace ns { namespace rpc { struct remote_transfer; + struct resource_stat; } enum class urd_error; @@ -123,6 +124,7 @@ private: response_ptr unknown_request_handler(const request_ptr req); void remote_transfer_handler(hermes::request&& req); + void resource_stat_handler(hermes::request&& req); // TODO: add helpers for remove and update urd_error create_namespace(const config::namespace_def& nsdef); diff --git a/src/utils.hpp b/src/utils.hpp index bc79a1c..c0f56d7 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -25,6 +25,9 @@ * . * *************************************************************************/ +#ifndef NORNS_UTILS_HPP +#define NORNS_UTILS_HPP + #include #include #include @@ -33,6 +36,7 @@ #include #include "common.hpp" +#include "utils/tar-archive.hpp" namespace norns { namespace utils { @@ -71,3 +75,5 @@ path relative(path from_path, path to_path); }} // namespace boost::filesystem #endif + +#endif // NORNS_UTILS_HPP diff --git a/src/io/transferors/tar-archive.cpp b/src/utils/tar-archive.cpp similarity index 56% rename from src/io/transferors/tar-archive.cpp rename to src/utils/tar-archive.cpp index 54bc38a..73282c8 100644 --- a/src/io/transferors/tar-archive.cpp +++ b/src/utils/tar-archive.cpp @@ -29,13 +29,34 @@ #include "logger.hpp" #include "tar-archive.hpp" +namespace { + +constexpr std::size_t +align(std::size_t n, std::size_t block_size) { + return n & ~(block_size - 1); +} + +constexpr std::size_t +xalign(std::size_t n, std::size_t block_size) { + return (n - align(n, block_size)) != 0 ? + align(n, block_size) + block_size : + n; +} + +} // anonymous namespace + namespace norns { namespace utils { tar::tar(const bfs::path& filename, openmode op, std::error_code& ec) : - m_path(filename) { + m_path(filename), + m_openmode(op) { + + if(filename.empty()) { + return; + } constexpr const std::array flags = { {O_WRONLY | O_CREAT| O_EXCL, O_RDONLY} @@ -55,18 +76,42 @@ tar::tar(const bfs::path& filename, } } +void +tar::add_file(const bfs::path& real_name, + const bfs::path& archive_name, + std::error_code& ec) { + + if(m_tar == nullptr) { + ec = std::make_error_code(static_cast(EINVAL)); + return; + } + + if(tar_append_file(m_tar, const_cast(real_name.c_str()), + const_cast(archive_name.c_str())) != 0) { + ec = std::make_error_code(static_cast(errno)); + return; + } + + ec = std::make_error_code(static_cast(0)); +} + void tar::add_directory(const bfs::path& real_dir, const bfs::path& archive_dir, std::error_code& ec) { + if(m_tar == nullptr) { + ec = std::make_error_code(static_cast(EINVAL)); + return; + } + const bfs::path rd = norns::utils::remove_trailing_separator(real_dir); const bfs::path ad = norns::utils::remove_trailing_separator(archive_dir); if(tar_append_tree(m_tar, const_cast(rd.c_str()), - const_cast(ad.c_str())) != 0) { + const_cast(ad.c_str())) != 0) { ec = std::make_error_code(static_cast(errno)); return; } @@ -94,9 +139,70 @@ tar::path() const { return m_path; } +std::size_t +tar::estimate_size_once_packed(const bfs::path& path, + std::error_code& ec) { + + std::size_t sz = 0; + boost::system::error_code error; + + if(bfs::is_directory(path)) { + for(bfs::recursive_directory_iterator it(path, error); + it != bfs::recursive_directory_iterator(); + ++it) { + + if(error) { + LOGGER_ERROR("Failed to traverse path {}", path); + ec = std::make_error_code( + static_cast(error.value())); + return 0; + } + + if(bfs::is_directory(*it)) { + sz += T_BLOCKSIZE; + } + else if(bfs::is_regular(*it) || bfs::is_symlink(*it)) { + sz += T_BLOCKSIZE + + ::xalign(bfs::file_size(*it, error), T_BLOCKSIZE); + + if(error) { + LOGGER_ERROR("Failed to determine size for {}", *it); + ec = std::make_error_code( + static_cast(error.value())); + return 0; + } + } + else { + // not a critical error, report it and go on + LOGGER_ERROR("Unhandled file type at {}", *it); + } + } + + // we need to take into account the header for the basedir 'path' + // since recursive_directory_iterator skips it + sz += T_BLOCKSIZE; + } + else { + sz += T_BLOCKSIZE + ::xalign(bfs::file_size(path), T_BLOCKSIZE); + } + + // EOF + sz += 2*T_BLOCKSIZE; + + return sz; +} + tar::~tar() { if(m_tar != nullptr) { + + if(m_openmode == tar::create) { + if(tar_append_eof(m_tar)) { + LOGGER_ERROR("Failed to append EOF to TAR archive: {}", + logger::errno_message(errno)); + } + } + if(tar_close(m_tar) != 0) { LOGGER_ERROR("Failed to close TAR archive: {}", logger::errno_message(errno)); diff --git a/src/io/transferors/tar-archive.hpp b/src/utils/tar-archive.hpp similarity index 87% rename from src/io/transferors/tar-archive.hpp rename to src/utils/tar-archive.hpp index d736c87..4054ebe 100644 --- a/src/io/transferors/tar-archive.hpp +++ b/src/utils/tar-archive.hpp @@ -25,6 +25,9 @@ * . * *************************************************************************/ +#ifndef NORNS_UTILS_TAR_ARCHIVE_HPP +#define NORNS_UTILS_TAR_ARCHIVE_HPP + #include #include #include @@ -46,6 +49,11 @@ struct tar { tar(const bfs::path& filename, openmode op, std::error_code& ec); + void + add_file(const bfs::path& real_name, + const bfs::path& archive_name, + std::error_code& ec); + void add_directory(const bfs::path& real_dir, const bfs::path& archive_dir, @@ -58,11 +66,18 @@ struct tar { bfs::path path() const; + static std::size_t + estimate_size_once_packed(const bfs::path& path, + std::error_code& ec); + ~tar(); TAR* m_tar = nullptr; bfs::path m_path; + openmode m_openmode; }; } // namespace utils } // namespace norns + +#endif // NORNS_UTILS_TAR_ARCHIVE_HPP diff --git a/tests/Makefile.am b/tests/Makefile.am index 6cebe39..8de929b 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -138,12 +138,16 @@ core_CPPFLAGS = \ -I$(top_srcdir)/src/externals/hermes/include \ -I$(top_srcdir)/rpc \ -I$(top_srcdir)/src \ + -DUSE_REAL_DAEMON \ $(END) core_SOURCES = \ catch.hpp \ api-main.cpp \ utils-path-normalize.cpp \ + utils-tar.cpp \ + test-env.cpp \ + test-env.hpp \ $(END) core_LDFLAGS = \ diff --git a/tests/utils-tar.cpp b/tests/utils-tar.cpp new file mode 100644 index 0000000..2c0b95f --- /dev/null +++ b/tests/utils-tar.cpp @@ -0,0 +1,349 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include +#include +#include "utils.hpp" +#include "test-env.hpp" +#include "catch.hpp" + +namespace { + +static constexpr std::size_t tar_header_size = T_BLOCKSIZE; +static constexpr std::size_t tar_eof_size = 2*T_BLOCKSIZE; + +constexpr std::size_t +tar_aligned_size(std::size_t sz) { + return sz % T_BLOCKSIZE != 0 ? + ((sz & ~(T_BLOCKSIZE - 1)) + T_BLOCKSIZE) : + sz; +} + + +using rng = boost::mt19937; +using distribution = boost::uniform_int<>; +using size_generator = + boost::variate_generator; + +bfs::path +create_hierarchy(test_env& env, + size_generator& size_gen, + const bfs::path& subdir, + const bfs::path& parent, + std::size_t subdirs_per_level, + std::size_t files_per_subdir, + std::size_t levels) { + + if(levels == 0) { + return {}; + } + + bfs::path self = env.create_directory(subdir, parent); + + for(std::size_t i = 0; i < files_per_subdir; ++i) { + std::string fname = "/regular_file" + std::to_string(i); + auto p = env.create_file(fname, self, size_gen()); + } + + if(levels == 1) { + return self; + } + + for(std::size_t i = 0; i < subdirs_per_level; ++i) { + std::string dname = "/subdir" + std::to_string(i); + auto child = env.create_directory(dname, self); + + (void) create_hierarchy(env, size_gen, dname, self, subdirs_per_level, + files_per_subdir, levels - 1); + } + + return self; +} + +} + +SCENARIO("tar size estimation", "[utils::tar::estimate_packed_size()]") { + + GIVEN("a path to a regular file of 0 bytes") { + + test_env env; + using norns::utils::tar; + + bfs::path file = env.create_file("/regular_file", env.basedir(), 0); + + THEN("estimated size follows formula") { + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(file, ec); + + REQUIRE(psize == tar_header_size + + tar_eof_size); + + AND_THEN("size of generated TAR archive matches estimated one") { + + bfs::path file_tar = file; + file_tar.replace_extension(".tar"); + + { + tar t(file_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_file(file, bfs::relative(file, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(file_tar)); + } + } + + env.notify_success(); + } + + GIVEN("a path to a regular file of T_BLOCKSIZE bytes") { + + test_env env; + using norns::utils::tar; + + bfs::path file = + env.create_file("/regular_file", env.basedir(), tar_header_size); + + THEN("estimated size follows formula") { + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(file, ec); + + REQUIRE(psize == tar_header_size + + tar_header_size + + tar_eof_size); + + AND_THEN("size of generated TAR archive matches estimated one") { + + bfs::path file_tar = file; + file_tar.replace_extension(".tar"); + + { + tar t(file_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_file(file, bfs::relative(file, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(file_tar)); + } + } + + env.notify_success(); + } + + GIVEN("a path to a regular file of 2*T_BLOCKSIZE bytes") { + + test_env env; + using norns::utils::tar; + + bfs::path file = + env.create_file("/regular_file", env.basedir(), 2*tar_header_size); + + THEN("estimated size follows formula") { + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(file, ec); + + REQUIRE(psize == tar_header_size + + 2*tar_header_size + + tar_eof_size); + + AND_THEN("size of generated TAR archive matches estimated one") { + + bfs::path file_tar = file; + file_tar.replace_extension(".tar"); + + { + tar t(file_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_file(file, bfs::relative(file, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(file_tar)); + } + } + + env.notify_success(); + } + + GIVEN("a path to a regular file of 10000 bytes") { + + test_env env; + using norns::utils::tar; + + bfs::path file = + env.create_file("/regular_file", env.basedir(), 10000); + + THEN("estimated size follows formula") { + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(file, ec); + + REQUIRE(psize == tar_header_size + + tar_aligned_size(10000) + + tar_eof_size); + + AND_THEN("size of generated TAR archive matches estimated one") { + + bfs::path file_tar = file; + file_tar.replace_extension(".tar"); + + { + tar t(file_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_directory(file, + bfs::relative(file, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(file_tar)); + } + } + + env.notify_success(); + } + + + GIVEN("a path to an empty directory") { + + test_env env; + using norns::utils::tar; + + bfs::path subdir = env.create_directory("/subdir", env.basedir()); + + THEN("estimated size follows formula") { + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(subdir, ec); + + REQUIRE(psize == tar_header_size + + tar_eof_size); + + AND_THEN("size of generated TAR archive matches estimated one") { + + bfs::path subdir_tar = subdir; + subdir_tar.replace_extension(".tar"); + + { + tar t(subdir_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_file(subdir, bfs::relative(subdir, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(subdir_tar)); + } + } + + env.notify_success(); + } + + GIVEN("a path to a directory hierarchy containing empty files") { + + test_env env; + using norns::utils::tar; + + size_generator gen(rng{42}, distribution{0, 0}); + + bfs::path subdir = + create_hierarchy(env, gen, "/subdir", env.basedir(), 5, 5, 5); + + THEN("size of generated TAR archive matches estimated one") { + + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(subdir, ec); + + bfs::path subdir_tar = subdir; + subdir_tar.replace_extension(".tar"); + + { + tar t(subdir_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_directory(subdir, + bfs::relative(subdir, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(subdir_tar)); + } + + env.notify_success(); + } + + GIVEN("a path to a directory hierarchy containing arbitrary files") { + + test_env env; + using norns::utils::tar; + + size_generator gen(rng{42}, distribution{100, 42*1024}); + + bfs::path subdir = + create_hierarchy(env, gen, "/subdir", env.basedir(), 5, 5, 5); + + THEN("size of generated TAR archive matches estimated one") { + + std::error_code ec; + std::size_t psize = tar::estimate_size_once_packed(subdir, ec); + + bfs::path subdir_tar = subdir; + subdir_tar.replace_extension(".tar"); + + { + tar t(subdir_tar, tar::create, ec); + REQUIRE(!ec); + + t.add_directory(subdir, + bfs::relative(subdir, env.basedir()), ec); + REQUIRE(!ec); + } + + // this check needs to be done after tar::~tar() has been + // called, since it appends the EOF blocks + REQUIRE(psize == bfs::file_size(subdir_tar)); + } + + env.notify_success(); + } + +} -- GitLab From 213a0011845eb3d72b084dbbcbd7001a6f55ac75 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 22 Feb 2019 15:30:42 +0100 Subject: [PATCH 17/51] Remove deprecated functions --- .../transferors/local-path-to-local-path.cpp | 53 ----------- .../transferors/local-path-to-local-path.hpp | 15 +-- .../local-path-to-remote-resource.cpp | 15 --- .../local-path-to-remote-resource.hpp | 14 +-- .../transferors/local-path-to-shared-path.cpp | 17 ---- .../transferors/local-path-to-shared-path.hpp | 20 ++-- src/io/transferors/memory-to-local-path.cpp | 47 --------- src/io/transferors/memory-to-local-path.hpp | 20 ++-- src/io/transferors/memory-to-remote-path.cpp | 17 ---- src/io/transferors/memory-to-remote-path.hpp | 20 ++-- src/io/transferors/memory-to-shared-path.cpp | 17 ---- src/io/transferors/memory-to-shared-path.hpp | 20 ++-- .../remote-resource-to-local-path.cpp | 95 ------------------- .../remote-resource-to-local-path.hpp | 11 +-- src/io/transferors/transferor.hpp | 19 ++-- 15 files changed, 58 insertions(+), 342 deletions(-) diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index bb61540..851b00d 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -231,59 +231,6 @@ local_path_to_local_path_transferor::transfer( d_dst.canonical_path()); } - -std::error_code -local_path_to_local_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) auth; - const auto src_backend = task_info->src_backend(); - const auto dst_backend = task_info->dst_backend(); - - std::error_code ec; - auto src = src_backend->get_resource(src_rinfo, ec); - - if(ec) { - LOGGER_ERROR("[{}] Could not access input resource '{}': {}", - task_info->id(), - src_rinfo->to_string(), - ec.message()); - return ec; - } - - auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); - - if(ec) { - LOGGER_ERROR("[{}] Could not create output resource '{}': {}", - task_info->id(), - src_rinfo->to_string(), - ec.message()); - return ec; - } - - const auto& d_src = - reinterpret_cast(*src); - const auto& d_dst = - reinterpret_cast(*dst); - - LOGGER_DEBUG("[{}] transfer: {} -> {}", - task_info->id(), - d_src.canonical_path(), - d_dst.canonical_path()); - - if(bfs::is_directory(d_src.canonical_path())) { - return ::copy_directory(task_info, d_src.canonical_path(), - d_dst.canonical_path()); - } - - return ::copy_file(task_info, - d_src.canonical_path(), - d_dst.canonical_path()); -} - std::string local_path_to_local_path_transferor::to_string() const { return "transferor[local_path => local_path]"; diff --git a/src/io/transferors/local-path-to-local-path.hpp b/src/io/transferors/local-path-to-local-path.hpp index 98f1e4d..202c009 100644 --- a/src/io/transferors/local-path-to-local-path.hpp +++ b/src/io/transferors/local-path-to-local-path.hpp @@ -51,20 +51,15 @@ struct local_path_to_local_path_transferor : public transferor { bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) const override final; - __attribute__((deprecated)) - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; - std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; - std::string to_string() const override final; + std::string + to_string() const override final; }; diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 2ba50ce..1aa7d9c 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -197,21 +197,6 @@ local_path_to_remote_resource_transferor::transfer( } } -std::error_code -local_path_to_remote_resource_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) auth; - (void) task_info; - (void) src_rinfo; - (void) dst_rinfo; - - return std::make_error_code(static_cast(0)); -} - std::string local_path_to_remote_resource_transferor::to_string() const { return "transferor[local_path => remote_resource]"; diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index fe1c034..9101149 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -65,23 +65,17 @@ struct local_path_to_remote_resource_transferor : public transferor { bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) - const override final; + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) - const override final; + const override final; - std::error_code - transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; - - std::string to_string() const override final; + std::string + to_string() const override final; private: diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index 4ac9cbb..f3f1a6f 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -71,23 +71,6 @@ local_path_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } -std::error_code -local_path_to_shared_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const { - - (void) auth; - (void) task_info; - (void) src; - (void) dst; - - LOGGER_WARN("Transfer not implemented"); - - return std::make_error_code(static_cast(0)); -} - std::string local_path_to_shared_path_transferor::to_string() const { return "transferor[local_path => shared_path]"; diff --git a/src/io/transferors/local-path-to-shared-path.hpp b/src/io/transferors/local-path-to-shared-path.hpp index 18329b8..750a346 100644 --- a/src/io/transferors/local-path-to-shared-path.hpp +++ b/src/io/transferors/local-path-to-shared-path.hpp @@ -47,21 +47,21 @@ struct resource; namespace io { struct local_path_to_shared_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; - std::string to_string() const override final; + std::string + to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index f42b189..5c4edf3 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -183,53 +183,6 @@ memory_region_to_local_path_transferor::transfer( d_src.size(), d_dst.canonical_path()); } -std::error_code -memory_region_to_local_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) task_info; - std::error_code ec; - const auto src_backend = task_info->src_backend(); - const auto dst_backend = task_info->dst_backend(); - - auto src = src_backend->get_resource(src_rinfo, ec); - - if(ec) { - LOGGER_ERROR("[{}] Could not access input resource '{}': {}", - task_info->id(), - src_rinfo->to_string(), - ec.message()); - return ec; - } - - auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); - - if(ec) { - LOGGER_ERROR("[{}] Could not create output resource '{}': {}", - task_info->id(), - src_rinfo->to_string(), - ec.message()); - return ec; - } - - const auto& d_src = reinterpret_cast(*src); - const auto& d_dst = reinterpret_cast(*dst); - - LOGGER_DEBUG("[{}] transfer: [{} {}+{}] -> {}", - task_info->id(), - auth.pid(), - utils::n2hexstr(d_src.address()), - d_src.size(), - d_dst.canonical_path()); - - return ::copy_memory_region(task_info, auth.pid(), - reinterpret_cast(d_src.address()), - d_src.size(), d_dst.canonical_path()); -} - std::string memory_region_to_local_path_transferor::to_string() const { return "transferor[memory_region => local_path]"; diff --git a/src/io/transferors/memory-to-local-path.hpp b/src/io/transferors/memory-to-local-path.hpp index c82799a..bb849a1 100644 --- a/src/io/transferors/memory-to-local-path.hpp +++ b/src/io/transferors/memory-to-local-path.hpp @@ -47,21 +47,21 @@ struct resource; namespace io { struct memory_region_to_local_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; - std::string to_string() const override final; + std::string + to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-remote-path.cpp b/src/io/transferors/memory-to-remote-path.cpp index da815f3..5135c07 100644 --- a/src/io/transferors/memory-to-remote-path.cpp +++ b/src/io/transferors/memory-to-remote-path.cpp @@ -70,23 +70,6 @@ memory_region_to_remote_path_transferor::transfer( return std::make_error_code(static_cast(0)); } -std::error_code -memory_region_to_remote_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) auth; - (void) task_info; - (void) src_rinfo; - (void) dst_rinfo; - - LOGGER_WARN("Transfer not implemented"); - - return std::make_error_code(static_cast(0)); -} - std::string memory_region_to_remote_path_transferor::to_string() const { return "transferor[memory_region => remote_path]"; diff --git a/src/io/transferors/memory-to-remote-path.hpp b/src/io/transferors/memory-to-remote-path.hpp index e35efe6..479c723 100644 --- a/src/io/transferors/memory-to-remote-path.hpp +++ b/src/io/transferors/memory-to-remote-path.hpp @@ -47,21 +47,21 @@ struct resource; namespace io { struct memory_region_to_remote_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; - std::string to_string() const override final; + std::string + to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index 43502c2..c573bb2 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -70,23 +70,6 @@ memory_region_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } -std::error_code -memory_region_to_shared_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) auth; - (void) task_info; - (void) src_rinfo; - (void) dst_rinfo; - - LOGGER_WARN("Transfer not implemented"); - - return std::make_error_code(static_cast(0)); -} - std::string memory_region_to_shared_path_transferor::to_string() const { return "transferor[memory_region => shared_path]"; diff --git a/src/io/transferors/memory-to-shared-path.hpp b/src/io/transferors/memory-to-shared-path.hpp index d39cb9e..dc95e24 100644 --- a/src/io/transferors/memory-to-shared-path.hpp +++ b/src/io/transferors/memory-to-shared-path.hpp @@ -47,21 +47,21 @@ struct resource; namespace io { struct memory_region_to_shared_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; - std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const override final; + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; - std::string to_string() const override final; + std::string + to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 15109fe..33b4b6b 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -267,101 +267,6 @@ respond: return std::make_error_code(static_cast(0)); } -std::error_code -remote_resource_to_local_path_transferor::transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src_rinfo, - const std::shared_ptr& dst_rinfo) const { - - (void) auth; - (void) task_info; - (void) src_rinfo; - (void) dst_rinfo; - -#if 0 - (void) auth; - const auto src_backend = task_info->src_backend(); - const auto dst_backend = task_info->dst_backend(); - - std::error_code ec; - auto src = src_backend->get_resource(src_rinfo, ec); - - if(ec) { - LOGGER_ERROR("[{}] Could not access input resource '{}': {}", - task_info->id(), - src_rinfo->to_string(), - ec.message()); - return ec; - } - - const auto& d_src = - reinterpret_cast(*src); - - const auto& d_dst = - reinterpret_cast(*dst_rinfo); - - LOGGER_DEBUG("[{}] transfer: {} -> {}://{}@{}", - task_info->id(), - d_src.canonical_path(), - d_dst.nsid(), - d_dst.datapath(), - d_dst.hostname()); - - LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", - dst_rinfo->to_string(), - "xxx"); - - hermes::endpoint endp = m_remote_endpoint->lookup(d_dst.hostname()); - - hermes::mapped_buffer b(d_src.canonical_path().string()); - - std::vector bufvec{ - hermes::mutable_buffer{b.data(), b.size()} - }; - - auto buffers = - m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); - - rpc::remote_transfer::input - args(d_dst.nsid(), d_dst.datapath(), buffers); - - auto rpc = - m_remote_endpoint->post(endp, args); -#endif - -#if 0 - const auto& d_src = - reinterpret_cast(*src); - const auto& d_dst = - reinterpret_cast(*dst); - - LOGGER_DEBUG("[{}] transfer: {} -> {}", task_info->id(), - d_src.canonical_path(), d_dst.to_string()); - - LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); - - hermes::endpoint endp = m_remote_endpoint->lookup("127.0.0.1:42000"); - - char* data = new char[42]; - - std::vector bufvec{ - hermes::mutable_buffer{(void*) data, 42} - }; - - auto buffers = - m_remote_endpoint->expose(bufvec, hermes::access_mode::read_only); - - norns::rpc::remote_transfer::input args("/tmp/foo", buffers); - - auto rpc = - m_remote_endpoint->post(endp, args); - - -#endif - return std::make_error_code(static_cast(0)); -} - std::string remote_resource_to_local_path_transferor::to_string() const { return "transferor[remote_resource => local_path]"; diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index 34c2d6f..924efb2 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -65,21 +65,14 @@ struct remote_resource_to_local_path_transferor : public transferor { bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) - const override final; + const override final; std::error_code transfer(const auth::credentials& auth, const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) - const override final; - - std::error_code - transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) - const override final; + const override final; std::string to_string() const override final; diff --git a/src/io/transferors/transferor.hpp b/src/io/transferors/transferor.hpp index 3d11fed..ba9c2e5 100644 --- a/src/io/transferors/transferor.hpp +++ b/src/io/transferors/transferor.hpp @@ -49,20 +49,15 @@ struct task_info; struct transferor { - virtual bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const = 0; - - virtual std::error_code transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const = 0; + virtual bool + validate(const std::shared_ptr &src_info, + const std::shared_ptr &dst_info) const = 0; virtual std::error_code - transfer(const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const = 0; - + transfer(const auth::credentials &auth, + const std::shared_ptr &task_info, + const std::shared_ptr &src, + const std::shared_ptr &dst) const = 0; virtual std::string to_string() const = 0; }; -- GitLab From 74d6baa55a78cea594e69dc7c0e1cdfe55f88125 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 22 Feb 2019 16:42:43 +0100 Subject: [PATCH 18/51] Add accept_transfer() to transferors API --- .../transferors/local-path-to-local-path.cpp | 15 +++++++++++++++ .../transferors/local-path-to-local-path.hpp | 7 +++++++ .../local-path-to-remote-resource.cpp | 16 ++++++++++++++++ .../local-path-to-remote-resource.hpp | 7 +++++++ .../transferors/local-path-to-shared-path.cpp | 19 ++++++++++++++++++- .../transferors/local-path-to-shared-path.hpp | 7 +++++++ src/io/transferors/memory-to-local-path.cpp | 15 +++++++++++++++ src/io/transferors/memory-to-local-path.hpp | 7 +++++++ src/io/transferors/memory-to-remote-path.cpp | 17 ++++++++++++++++- src/io/transferors/memory-to-remote-path.hpp | 7 +++++++ src/io/transferors/memory-to-shared-path.cpp | 17 ++++++++++++++++- src/io/transferors/memory-to-shared-path.hpp | 7 +++++++ .../remote-resource-to-local-path.cpp | 18 +++++++++++++++++- .../remote-resource-to-local-path.hpp | 7 +++++++ src/io/transferors/transferor.hpp | 9 ++++++++- 15 files changed, 170 insertions(+), 5 deletions(-) diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 851b00d..9aee089 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -231,6 +231,21 @@ local_path_to_local_path_transferor::transfer( d_dst.canonical_path()); } +std::error_code +local_path_to_local_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + return std::make_error_code(static_cast(0)); +} + std::string local_path_to_local_path_transferor::to_string() const { return "transferor[local_path => local_path]"; diff --git a/src/io/transferors/local-path-to-local-path.hpp b/src/io/transferors/local-path-to-local-path.hpp index 202c009..9fbaf11 100644 --- a/src/io/transferors/local-path-to-local-path.hpp +++ b/src/io/transferors/local-path-to-local-path.hpp @@ -58,6 +58,13 @@ struct local_path_to_local_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 1aa7d9c..524b9d1 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -197,6 +197,22 @@ local_path_to_remote_resource_transferor::transfer( } } + +std::error_code +local_path_to_remote_resource_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + return std::make_error_code(static_cast(0)); +} + std::string local_path_to_remote_resource_transferor::to_string() const { return "transferor[local_path => remote_resource]"; diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index 9101149..f7d09c4 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -74,6 +74,13 @@ struct local_path_to_remote_resource_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index f3f1a6f..7edc2e9 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -66,7 +66,24 @@ local_path_to_shared_path_transferor::transfer( (void) src; (void) dst; - LOGGER_WARN("Transfer not implemented"); + LOGGER_WARN("transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + +std::error_code +local_path_to_shared_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + LOGGER_WARN("transfer not implemented"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/local-path-to-shared-path.hpp b/src/io/transferors/local-path-to-shared-path.hpp index 750a346..49e8921 100644 --- a/src/io/transferors/local-path-to-shared-path.hpp +++ b/src/io/transferors/local-path-to-shared-path.hpp @@ -60,6 +60,13 @@ struct local_path_to_shared_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index 5c4edf3..ad2a3df 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -183,6 +183,21 @@ memory_region_to_local_path_transferor::transfer( d_src.size(), d_dst.canonical_path()); } +std::error_code +memory_region_to_local_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + return std::make_error_code(static_cast(0)); +} + std::string memory_region_to_local_path_transferor::to_string() const { return "transferor[memory_region => local_path]"; diff --git a/src/io/transferors/memory-to-local-path.hpp b/src/io/transferors/memory-to-local-path.hpp index bb849a1..720f1e8 100644 --- a/src/io/transferors/memory-to-local-path.hpp +++ b/src/io/transferors/memory-to-local-path.hpp @@ -60,6 +60,13 @@ struct memory_region_to_local_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-remote-path.cpp b/src/io/transferors/memory-to-remote-path.cpp index 5135c07..b19f30c 100644 --- a/src/io/transferors/memory-to-remote-path.cpp +++ b/src/io/transferors/memory-to-remote-path.cpp @@ -65,7 +65,22 @@ memory_region_to_remote_path_transferor::transfer( (void) d_src; (void) d_dst; - LOGGER_WARN("Transfer not implemented"); + LOGGER_WARN("transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + +std::error_code +memory_region_to_remote_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/memory-to-remote-path.hpp b/src/io/transferors/memory-to-remote-path.hpp index 479c723..15f2bbd 100644 --- a/src/io/transferors/memory-to-remote-path.hpp +++ b/src/io/transferors/memory-to-remote-path.hpp @@ -60,6 +60,13 @@ struct memory_region_to_remote_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index c573bb2..a874634 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -65,7 +65,22 @@ memory_region_to_shared_path_transferor::transfer( (void) d_src; (void) d_dst; - LOGGER_WARN("Transfer not implemented"); + LOGGER_WARN("transfer not implemented"); + + return std::make_error_code(static_cast(0)); +} + +std::error_code +memory_region_to_shared_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/memory-to-shared-path.hpp b/src/io/transferors/memory-to-shared-path.hpp index dc95e24..b646d21 100644 --- a/src/io/transferors/memory-to-shared-path.hpp +++ b/src/io/transferors/memory-to-shared-path.hpp @@ -60,6 +60,13 @@ struct memory_region_to_shared_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 33b4b6b..963e356 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -196,7 +196,7 @@ remote_resource_to_local_path_transferor::transfer( usecs); //TODO: hermes offers no way to check for an error yet - LOGGER_DEBUG("Transfer completed ({} usecs)", usecs); + LOGGER_DEBUG("Pull completed ({} usecs)", usecs); if(is_collection) { std::error_code ec; @@ -267,6 +267,22 @@ respond: return std::make_error_code(static_cast(0)); } + +std::error_code +remote_resource_to_local_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + return std::make_error_code(static_cast(0)); +} + std::string remote_resource_to_local_path_transferor::to_string() const { return "transferor[remote_resource => local_path]"; diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index 924efb2..a8bdc0b 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -74,6 +74,13 @@ struct remote_resource_to_local_path_transferor : public transferor { const std::shared_ptr& dst) const override final; + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + std::string to_string() const override final; diff --git a/src/io/transferors/transferor.hpp b/src/io/transferors/transferor.hpp index ba9c2e5..186efc3 100644 --- a/src/io/transferors/transferor.hpp +++ b/src/io/transferors/transferor.hpp @@ -59,7 +59,14 @@ struct transferor { const std::shared_ptr &src, const std::shared_ptr &dst) const = 0; - virtual std::string to_string() const = 0; + virtual std::error_code + accept_transfer(const auth::credentials &auth, + const std::shared_ptr &task_info, + const std::shared_ptr &src, + const std::shared_ptr &dst) const = 0; + + virtual std::string + to_string() const = 0; }; } // namespace io -- GitLab From 124d73fc5c4462ae7dd1b7f7b71906b1da7ab2f4 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Sat, 23 Feb 2019 00:08:12 +0100 Subject: [PATCH 19/51] Improve internal organization of transfer plugins --- src/io/task-info.cpp | 10 +- src/io/task-info.hpp | 74 +++++-- src/io/task-manager.cpp | 13 +- src/io/task-remote-transfer.hpp | 8 +- .../transferors/local-path-to-local-path.cpp | 1 + .../local-path-to-remote-resource.cpp | 200 +++++++++++++++++- .../transferors/local-path-to-shared-path.cpp | 3 +- src/io/transferors/memory-to-local-path.cpp | 1 + src/io/transferors/memory-to-remote-path.cpp | 1 + src/io/transferors/memory-to-shared-path.cpp | 1 + .../remote-resource-to-local-path.cpp | 5 + src/urd.cpp | 9 +- 12 files changed, 294 insertions(+), 32 deletions(-) diff --git a/src/io/task-info.cpp b/src/io/task-info.cpp index cddfc95..0d394ec 100644 --- a/src/io/task-info.cpp +++ b/src/io/task-info.cpp @@ -34,7 +34,9 @@ namespace norns { namespace io { -task_info::task_info(const iotask_id tid, const iotask_type type, +task_info::task_info(const iotask_id tid, + const iotask_type type, + const bool is_remote, const auth::credentials& auth, const backend_ptr src_backend, const resource_info_ptr src_rinfo, @@ -43,6 +45,7 @@ task_info::task_info(const iotask_id tid, const iotask_type type, const boost::any& ctx) : m_id(tid), m_type(type), + m_is_remote(is_remote), m_auth(auth), m_src_backend(src_backend), m_src_rinfo(src_rinfo), @@ -81,6 +84,11 @@ task_info::type() const { return m_type; } +bool +task_info::is_remote() const { + return m_is_remote; +} + auth::credentials task_info::auth() const { return m_auth; diff --git a/src/io/task-info.hpp b/src/io/task-info.hpp index 10c79dd..0f260b8 100644 --- a/src/io/task-info.hpp +++ b/src/io/task-info.hpp @@ -48,6 +48,7 @@ struct task_info { task_info(const iotask_id tid, const iotask_type type, + const bool is_remote, const auth::credentials& creds, const backend_ptr src_backend, const resource_info_ptr src_rinfo, @@ -57,21 +58,45 @@ struct task_info { ~task_info(); - iotask_id id() const; - iotask_type type() const; - auth::credentials auth() const ; - backend_ptr src_backend() const; - resource_info_ptr src_rinfo() const; - backend_ptr dst_backend() const; - resource_info_ptr dst_rinfo() const; + iotask_id + id() const; - boost::any context() const; - void set_context(const boost::any& ctx); + iotask_type + type() const; - task_status status() const; - void update_status(const task_status st); - void update_status(const task_status st, const urd_error ec, - const std::error_code& sc); + bool + is_remote() const; + + auth::credentials + auth() const ; + + backend_ptr + src_backend() const; + + resource_info_ptr + src_rinfo() const; + + backend_ptr + dst_backend() const; + + resource_info_ptr + dst_rinfo() const; + + boost::any + context() const; + + void + set_context(const boost::any& ctx); + + task_status + status() const; + + void + update_status(const task_status st); + + void + update_status(const task_status st, const urd_error ec, + const std::error_code& sc); urd_error task_error() const; @@ -79,13 +104,23 @@ struct task_info { std::error_code sys_error() const; - std::size_t sent_bytes() const; - std::size_t total_bytes() const; - double bandwidth() const; - void update_bandwidth(std::size_t bytes, double usecs); - void record_transfer(std::size_t bytes, double usecs); + std::size_t + sent_bytes() const; + + std::size_t + total_bytes() const; + + double + bandwidth() const; + + void + update_bandwidth(std::size_t bytes, double usecs); + + void + record_transfer(std::size_t bytes, double usecs); - task_stats stats() const; + task_stats + stats() const; boost::shared_lock lock_shared() const; @@ -98,6 +133,7 @@ struct task_info { // task id and type const iotask_id m_id; const iotask_type m_type; + const bool m_is_remote; // user credentials const auth::credentials m_auth; diff --git a/src/io/task-manager.cpp b/src/io/task-manager.cpp index 03cadde..2dabe1f 100644 --- a/src/io/task-manager.cpp +++ b/src/io/task-manager.cpp @@ -169,7 +169,7 @@ task_manager::create_task(iotask_type type, auto it = m_task_info.end(); std::tie(it, std::ignore) = m_task_info.emplace(tid, - std::make_shared(tid, type, auth, + std::make_shared(tid, type, false, auth, src_backend, src_rinfo, dst_backend, dst_rinfo)); return it->second; @@ -326,7 +326,7 @@ task_manager::create_local_initiated_task(iotask_type type, auto it = m_task_info.end(); std::tie(it, std::ignore) = m_task_info.emplace(tid, - std::make_shared(tid, type, auth, + std::make_shared(tid, type, false, auth, src_backend, src_rinfo, dst_backend, dst_rinfo)); return it->second; @@ -431,15 +431,20 @@ task_manager::create_remote_initiated_task(iotask_type task_type, auto it = m_task_info.end(); std::tie(it, std::ignore) = m_task_info.emplace(tid, - std::make_shared(tid, task_type, auth, + std::make_shared(tid, task_type, true, auth, src_backend, src_rinfo, dst_backend, dst_rinfo, ctx)); return it->second; }(); + //auto tx_ptr = + // m_transferor_registry.get(src_rinfo->type(), dst_rinfo->type()); + + // XXX for a remote-initiated task the order of the types + // is swapped auto tx_ptr = - m_transferor_registry.get(src_rinfo->type(), dst_rinfo->type()); + m_transferor_registry.get(dst_rinfo->type(), src_rinfo->type()); if(!tx_ptr) { return std::make_tuple(urd_error::not_supported, boost::none); diff --git a/src/io/task-remote-transfer.hpp b/src/io/task-remote-transfer.hpp index b758d3e..28ad633 100644 --- a/src/io/task-remote-transfer.hpp +++ b/src/io/task-remote-transfer.hpp @@ -80,7 +80,13 @@ task::operator()() { return; } - ec = m_transferor->transfer(auth, m_task_info, src, dst); + if(m_task_info->is_remote()) { + ec = m_transferor->accept_transfer(auth, m_task_info, src, dst); + } + else { + ec = m_transferor->transfer(auth, m_task_info, src, dst); + } + if(ec) { log_error("Transfer failed"); diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 9aee089..2e7fe5a 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -243,6 +243,7 @@ local_path_to_local_path_transferor::accept_transfer( (void) src; (void) dst; + LOGGER_ERROR("This function should never be called for this transfer type"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 524b9d1..3dbdf2d 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -36,11 +36,65 @@ #include "resources.hpp" #include "auth.hpp" #include "io/task-info.hpp" -#include "backends/posix-fs.hpp" +#include "io/task-stats.hpp" #include "hermes.hpp" #include "rpcs.hpp" #include "local-path-to-remote-resource.hpp" +namespace { + +std::tuple> +create_file(const bfs::path& filename, + std::size_t size) { + + std::error_code ec; + + int out_fd = + ::open(filename.c_str(), O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); + + if(out_fd == -1) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + + // preallocate output file + if(::fallocate(out_fd, 0, 0, size) == -1) { + // filesystem doesn't support fallocate(), fallback to truncate() + if(errno == EOPNOTSUPP) { + if(::ftruncate(out_fd, size) != 0) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + } + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + +retry_close: + if(close(out_fd) == -1) { + if(errno == EINTR) { + goto retry_close; + } + + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } + + auto output_data = + std::make_shared( + filename.string(), + hermes::access_mode::write_only, + &ec); + + if(ec) { + LOGGER_ERROR("Failed mapping output data: {}", ec.value()); + return std::make_tuple(ec, output_data); + } + + return std::make_tuple(ec, output_data); +} + +} // anonymous namespace namespace norns { namespace io { @@ -206,10 +260,152 @@ local_path_to_remote_resource_transferor::accept_transfer( const std::shared_ptr& dst) const { (void) auth; - (void) task_info; (void) src; (void) dst; + using utils::tar; + + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + const auto ctx = boost::any_cast< + std::shared_ptr< + hermes::request>>(task_info->context()); + auto req = std::move(*ctx); + + LOGGER_DEBUG("[{}] accept_transfer: {} -> {}", task_info->id(), + d_src.to_string(), d_dst.canonical_path()); + + LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); + + hermes::exposed_memory remote_buffers = d_src.buffers(); + + LOGGER_DEBUG("remote_buffers{{count={}, total_size={}}}", + remote_buffers.count(), + remote_buffers.size()); + + assert(remote_buffers.count() == 1); + + bool is_collection = d_src.is_collection(); + + bfs::path output_path = + is_collection ? "/tmp/test.tar" : d_dst.canonical_path(); + + LOGGER_DEBUG("creating local resource: {}", output_path); + + std::error_code ec; + std::shared_ptr output_data; + + std::tie(ec, output_data) = + ::create_file(output_path, remote_buffers.size()); + + if(ec) { + *ctx = std::move(req); + return ec; + } + + // let's prepare some local buffers + + std::vector bufseq{ + hermes::mutable_buffer{output_data->data(), output_data->size()} + }; + + hermes::exposed_memory local_buffers = + m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); + + LOGGER_DEBUG("pulling remote data into {}", output_path); + + auto start = std::chrono::steady_clock::now(); + + // N.B. IMPORTANT: we NEED to capture output_data by value here so that + // the mapped_buffer doesn't get released before completion_callback() + // is called. + const auto completion_callback = + [this, is_collection, output_path, d_dst, output_data, start]( + hermes::request&& req) { + + uint32_t usecs = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + + // default response + rpc::remote_transfer::output out( + static_cast(task_status::finished), + static_cast(urd_error::success), + 0, + usecs); + + //TODO: hermes offers no way to check for an error yet + LOGGER_DEBUG("Pull completed ({} usecs)", usecs); + + if(is_collection) { + std::error_code ec; + boost::system::error_code bec; + tar ar(output_path, tar::open, ec); + + if(ec) { + LOGGER_ERROR("Failed to open archive {}: {}", + output_path, logger::errno_message(ec.value())); + + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + goto respond; + } + + ar.extract(d_dst.parent()->mount(), ec); + + if(ec) { + LOGGER_ERROR("Failed to extact archive into {}: {}", + ar.path(), d_dst.parent()->mount(), + logger::errno_message(ec.value())); + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + goto respond; + } + + LOGGER_DEBUG("Archive {} extracted into {}", + ar.path(), d_dst.parent()->mount()); + + bfs::remove(ar.path(), bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + ar.path(), logger::errno_message(ec.value())); + out = std::move(rpc::remote_transfer::output{ + static_cast(task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0 + }); + } + } + +respond: + if(req.requires_response()) { + m_network_endpoint->respond( + std::move(req), + static_cast(task_status::finished), + static_cast(urd_error::success), + 0, + usecs); + } + }; + + m_network_endpoint->async_pull(remote_buffers, + local_buffers, + std::move(req), + completion_callback); + return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index 7edc2e9..6833885 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -83,8 +83,7 @@ local_path_to_shared_path_transferor::accept_transfer( (void) src; (void) dst; - LOGGER_WARN("transfer not implemented"); - + LOGGER_ERROR("This function should never be called for this transfer type"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index ad2a3df..a6c80ac 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -195,6 +195,7 @@ memory_region_to_local_path_transferor::accept_transfer( (void) src; (void) dst; + LOGGER_ERROR("This function should never be called for this transfer type"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/memory-to-remote-path.cpp b/src/io/transferors/memory-to-remote-path.cpp index b19f30c..24503c6 100644 --- a/src/io/transferors/memory-to-remote-path.cpp +++ b/src/io/transferors/memory-to-remote-path.cpp @@ -82,6 +82,7 @@ memory_region_to_remote_path_transferor::accept_transfer( (void) src; (void) dst; + LOGGER_ERROR("This function should never be called for this transfer type"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index a874634..9f3aac7 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -82,6 +82,7 @@ memory_region_to_shared_path_transferor::accept_transfer( (void) src; (void) dst; + LOGGER_ERROR("This function should never be called for this transfer type"); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 963e356..67f5b88 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -35,6 +35,7 @@ #include "rpcs.hpp" #include "remote-resource-to-local-path.hpp" +#if 0 namespace { std::tuple> @@ -89,6 +90,7 @@ retry_close: } } // anonymous namespace +#endif namespace norns { namespace io { @@ -120,7 +122,9 @@ remote_resource_to_local_path_transferor::transfer( (void) auth; (void) src; (void) dst; + (void) task_info; +#if 0 using utils::tar; const auto& d_src = @@ -264,6 +268,7 @@ respond: std::move(req), completion_callback); +#endif return std::make_error_code(static_cast(0)); } diff --git a/src/urd.cpp b/src/urd.cpp index 7e5eb9d..1d87682 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -724,9 +724,6 @@ urd::remote_transfer_handler(hermes::request&& req) { } }; - auto src_rinfo = create_rinfo(data::resource_type::remote_resource); - auto dst_rinfo = create_rinfo(dst_rtype); - if(m_is_paused) { rv = urd_error::accept_paused; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); @@ -738,6 +735,12 @@ urd::remote_transfer_handler(hermes::request&& req) { return; } + auto src_rinfo = create_rinfo(data::resource_type::remote_resource); + auto dst_rinfo = create_rinfo(dst_rtype); + + //TODO: check src_rinfo and dst_rinfo + + // TODO: actually retrieve and validate credentials, etc -- GitLab From e2a5c0fb604af1664ec215e539f9a8352bc46ab0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 26 Feb 2019 14:14:28 +0100 Subject: [PATCH 20/51] Update hermes --- src/externals/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/externals/hermes b/src/externals/hermes index e49040c..37da27f 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit e49040c5cc2df53c4a89349bb3bf5ce59bcf0231 +Subproject commit 37da27f9b6d6ed279a96f2fae01e92b1355ae8c5 -- GitLab From 23bca485c99efbe2dbcb5131a0569119556041bc Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Tue, 26 Feb 2019 14:14:48 +0100 Subject: [PATCH 21/51] First working implementation of remote pushes --- include/norns/norns_error.h | 4 + lib/errors.c | 4 + src/common/types.cpp | 4 + src/common/types.hpp | 4 + src/io/task-remote-transfer.hpp | 4 +- .../local-path-to-remote-resource.cpp | 54 +- .../remote-resource-to-local-path.cpp | 309 +- .../remote-resource-to-local-path.hpp | 4 +- .../detail/local-path-impl.cpp | 9 +- src/rpcs.cpp | 1 + src/rpcs.hpp | 367 +- src/urd.cpp | 153 +- src/urd.hpp | 2 + tests/api-copy-remote-data.cpp | 3075 +++++++++++++++-- tests/api-task-submit.cpp | 40 +- 15 files changed, 3556 insertions(+), 478 deletions(-) diff --git a/include/norns/norns_error.h b/include/norns/norns_error.h index bbb23df..23e7a73 100644 --- a/include/norns/norns_error.h +++ b/include/norns/norns_error.h @@ -77,6 +77,10 @@ extern "C" { #define NORNS_EFINISHED -102 #define NORNS_EFINISHEDWERROR -103 +/* errors resources */ +#define NORNS_ERESOURCEEXISTS -110 +#define NORNS_ENOSUCHRESOURCE -111 + /* misc errors */ #define NORNS_ENOTSUPPORTED -200 #define NORNS_ESYSTEMERROR -201 diff --git a/lib/errors.c b/lib/errors.c index 8c1b274..c312e28 100644 --- a/lib/errors.c +++ b/lib/errors.c @@ -61,6 +61,10 @@ const char* const norns_errlist[NORNS_ERRMAX + 1] = { [ERR_REMAP(NORNS_ETOOMANYTASKS)] = "Too many pending tasks", [ERR_REMAP(NORNS_ETASKSPENDING)] = "There are still pending tasks", + /* resource errors */ + [ERR_REMAP(NORNS_ERESOURCEEXISTS)] = "Resource already exists", + [ERR_REMAP(NORNS_ENOSUCHRESOURCE)] = "Resource does not exist", + /* misc errors */ [ERR_REMAP(NORNS_ENOTSUPPORTED)] = "Not supported", [ERR_REMAP(NORNS_ESYSTEMERROR)] = "Operating system error", diff --git a/src/common/types.cpp b/src/common/types.cpp index f838144..2c2e5e9 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -109,6 +109,10 @@ std::string to_string(urd_error ecode) { return "NORNS_ETASKSPENDING"; case urd_error::accept_paused: return "NORNS_EACCEPTPAUSED"; + case urd_error::resource_exists: + return "NORNS_ERESOURCEEXISTS"; + case urd_error::no_such_resource: + return "NORNS_ENOSUCHRESOURCE"; default: return "UNKNOWN_ERROR"; } diff --git a/src/common/types.hpp b/src/common/types.hpp index aeffde8..aa3fad5 100644 --- a/src/common/types.hpp +++ b/src/common/types.hpp @@ -107,6 +107,10 @@ enum class urd_error : norns_error_t { no_such_task = NORNS_ENOSUCHTASK, too_many_tasks = NORNS_ETOOMANYTASKS, tasks_pending = NORNS_ETASKSPENDING, + + /* errors about resources */ + resource_exists = NORNS_ERESOURCEEXISTS, + no_such_resource = NORNS_ENOSUCHRESOURCE, }; namespace utils { diff --git a/src/io/task-remote-transfer.hpp b/src/io/task-remote-transfer.hpp index 28ad633..321612a 100644 --- a/src/io/task-remote-transfer.hpp +++ b/src/io/task-remote-transfer.hpp @@ -61,8 +61,8 @@ task::operator()() { LOGGER_WARN("[{}] Starting I/O task", tid); LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); - LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); - LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); + LOGGER_WARN("[{}] FROM: {}", tid, src_rinfo->to_string());//src_backend->to_string()); + LOGGER_WARN("[{}] TO: {}", tid, dst_rinfo->to_string());//dst_backend->to_string()); m_task_info->update_status(task_status::running); diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 3dbdf2d..a5faded 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -134,7 +134,7 @@ local_path_to_remote_resource_transferor::transfer( std::string input_path = d_src.canonical_path().string(); - if(src->is_collection()) { + if(d_src.is_collection()) { LOGGER_DEBUG("[{}] Creating archive for local directory", task_info->id()); @@ -146,8 +146,7 @@ local_path_to_remote_resource_transferor::transfer( tar ar(ar_path, tar::create, ec); if(ec) { - LOGGER_ERROR("Failed to create archive: {}", - logger::errno_message(ec.value())); + LOGGER_ERROR("Failed to create archive: {}", ec.message()); return ec; } @@ -159,7 +158,7 @@ local_path_to_remote_resource_transferor::transfer( if(ec) { LOGGER_ERROR("Failed to add directory to archive: {}", - logger::errno_message(ec.value())); + ec.message()); return ec; } @@ -189,7 +188,7 @@ local_path_to_remote_resource_transferor::transfer( auto buffers = m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); - norns::rpc::remote_transfer::input args( + rpc::remote_transfer::input args( m_network_endpoint->self_address(), d_src.parent()->nsid(), d_dst.parent()->nsid(), @@ -215,10 +214,36 @@ local_path_to_remote_resource_transferor::transfer( LOGGER_FLUSH(); auto rpc = - m_network_endpoint->post(endp, args); + m_network_endpoint->post(endp, args); auto resp = rpc.get(); + if(static_cast(resp.at(0).status()) == + task_status::finished_with_error) { + + // XXX it would probably be worth it to define + // a temporary_file class that cleans itself when + // destroyed to ease removal and avoid code duplication + // below + if(src->is_collection()) { + boost::system::error_code bec; + + bfs::remove(input_path, bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + input_path, ec.message()); + //TODO + return std::make_error_code( + static_cast(bec.value())); + } + } + + // XXX error interface should be improved + return std::make_error_code( + static_cast(resp.at(0).sys_errnum())); + } + task_info->record_transfer(input_data.size(), resp.at(0).elapsed_time()); @@ -236,7 +261,7 @@ local_path_to_remote_resource_transferor::transfer( if(bec) { LOGGER_ERROR("Failed to remove archive {}: {}", - input_path, logger::errno_message(ec.value())); + input_path, ec.message()); //TODO return std::make_error_code( static_cast(bec.value())); @@ -347,7 +372,7 @@ local_path_to_remote_resource_transferor::accept_transfer( if(ec) { LOGGER_ERROR("Failed to open archive {}: {}", - output_path, logger::errno_message(ec.value())); + output_path, ec.message()); out = std::move(rpc::remote_transfer::output{ static_cast(task_status::finished_with_error), @@ -361,9 +386,9 @@ local_path_to_remote_resource_transferor::accept_transfer( ar.extract(d_dst.parent()->mount(), ec); if(ec) { - LOGGER_ERROR("Failed to extact archive into {}: {}", + LOGGER_ERROR("Failed to extract archive into {}: {}", ar.path(), d_dst.parent()->mount(), - logger::errno_message(ec.value())); + ec.message()); out = std::move(rpc::remote_transfer::output{ static_cast(task_status::finished_with_error), static_cast(urd_error::system_error), @@ -380,11 +405,11 @@ local_path_to_remote_resource_transferor::accept_transfer( if(bec) { LOGGER_ERROR("Failed to remove archive {}: {}", - ar.path(), logger::errno_message(ec.value())); + ar.path(), ec.message()); out = std::move(rpc::remote_transfer::output{ static_cast(task_status::finished_with_error), static_cast(urd_error::system_error), - static_cast(ec.value()), + static_cast(bec.value()), 0 }); } @@ -394,10 +419,7 @@ respond: if(req.requires_response()) { m_network_endpoint->respond( std::move(req), - static_cast(task_status::finished), - static_cast(urd_error::success), - 0, - usecs); + out); } }; diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 67f5b88..6111600 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -35,7 +35,6 @@ #include "rpcs.hpp" #include "remote-resource-to-local-path.hpp" -#if 0 namespace { std::tuple> @@ -90,14 +89,13 @@ retry_close: } } // anonymous namespace -#endif namespace norns { namespace io { remote_resource_to_local_path_transferor::remote_resource_to_local_path_transferor( - std::shared_ptr remote_endpoint) : - m_remote_endpoint(remote_endpoint) { } + std::shared_ptr network_endpoint) : + m_network_endpoint(network_endpoint) { } bool remote_resource_to_local_path_transferor::validate( @@ -124,166 +122,259 @@ remote_resource_to_local_path_transferor::transfer( (void) dst; (void) task_info; -#if 0 - using utils::tar; - const auto& d_src = reinterpret_cast(*src); const auto& d_dst = reinterpret_cast(*dst); - const auto ctx = boost::any_cast< - std::shared_ptr< - hermes::request>>(task_info->context()); - auto req = std::move(*ctx); + LOGGER_DEBUG("[{}] request_transfer: {} -> {}", + task_info->id(), d_src.to_string(), d_dst.canonical_path()); - LOGGER_DEBUG("[{}] accept_transfer: {} -> {}", task_info->id(), - d_src.to_string(), d_dst.canonical_path()); + hermes::endpoint endp = m_network_endpoint->lookup(d_src.address()); - LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); + rpc::resource_stat::input args( + m_network_endpoint->self_address(), + d_src.parent()->nsid(), + static_cast(data::resource_type::local_posix_path), + d_src.name()); - hermes::exposed_memory remote_buffers = d_src.buffers(); - - LOGGER_DEBUG("remote_buffers{{count={}, total_size={}}}", - remote_buffers.count(), - remote_buffers.size()); + auto resp = + m_network_endpoint->post(endp, args).get(); - assert(remote_buffers.count() == 1); + LOGGER_DEBUG("remote_stat returned [task_error: {}, sys_errnum: {}, " + "is_collection: {}, packed_size: {}]", + resp.at(0).task_error(), + resp.at(0).sys_errnum(), + resp.at(0).is_collection(), + resp.at(0).packed_size()); - bool is_collection = d_src.is_collection(); + if(static_cast(resp.at(0).task_error()) != + urd_error::success) { + return std::make_error_code( + static_cast(resp.at(0).sys_errnum())); + } bfs::path output_path = - is_collection ? "/tmp/test.tar" : d_dst.canonical_path(); + resp.at(0).is_collection() ? "/tmp/test.tar" : d_dst.canonical_path(); LOGGER_DEBUG("creating local resource: {}", output_path); std::error_code ec; - std::shared_ptr output_data; + std::shared_ptr output_buffer; + + std::tie(ec, output_buffer) = + ::create_file(output_path, resp.at(0).packed_size()); + + // let's prepare some local buffers + std::vector bufseq{ + hermes::mutable_buffer{output_buffer->data(), output_buffer->size()} + }; + + hermes::exposed_memory local_buffers = + m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); + + rpc::pull_resource::input args2( + d_src.parent()->nsid(), + d_src.name(), + // XXX this resource_type should not be needed, but we cannot + // XXX (easily) find it out right now in the server, for now we + // XXX propagate it, but we should implement a lookup()/stat() + // XXX function in backends to retrieve this information from the + // XXX resource id + static_cast(data::resource_type::local_posix_path), + m_network_endpoint->self_address(), + d_dst.parent()->nsid(), + d_dst.name(), + local_buffers); + + auto resp2 = + m_network_endpoint->post(endp, args2).get(); + + LOGGER_DEBUG("Remote push request completed with output " + "{{status: {}, task_error: {}, sys_errnum: {}}} " + "({} bytes, {} usecs)", + resp2.at(0).status(), resp2.at(0).task_error(), + resp2.at(0).sys_errnum(), output_buffer->size(), + resp2.at(0).elapsed_time()); + + if(static_cast(resp2.at(0).status()) == + task_status::finished_with_error) { + // XXX error interface should be improved + return std::make_error_code( + static_cast(resp2.at(0).sys_errnum())); + } + + if(resp.at(0).is_collection()) { + using utils::tar; + + std::error_code ec; + boost::system::error_code bec; + tar ar(output_path, tar::open, ec); + + if(ec) { + LOGGER_ERROR("Failed to open archive {}: {}", + output_path, logger::errno_message(ec.value())); + + return std::make_error_code(static_cast(ec.value())); + } + + ar.extract(d_dst.parent()->mount(), ec); + + if(ec) { + LOGGER_ERROR("Failed to extact archive into {}: {}", + ar.path(), d_dst.parent()->mount(), ec.message()); + return std::make_error_code(static_cast(ec.value())); + } + + LOGGER_DEBUG("Archive {} extracted into {}", + ar.path(), d_dst.parent()->mount()); + + bfs::remove(ar.path(), bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + ar.path(), ec.message()); + return std::make_error_code(static_cast(ec.value())); + } + } + + return std::make_error_code(static_cast(0)); +} + + +std::error_code +remote_resource_to_local_path_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + std::error_code ec; + + std::string input_path = d_src.canonical_path().string(); + bool is_collection = d_src.is_collection(); - std::tie(ec, output_data) = - ::create_file(output_path, remote_buffers.size()); + if(is_collection) { + using utils::tar; + + LOGGER_DEBUG("[{}] Creating archive from local directory", + task_info->id()); + + bfs::path ar_path = + "/tmp" / bfs::unique_path("norns-archive-%%%%-%%%%-%%%%.tar"); + + tar ar(ar_path, tar::create, ec); + + if(ec) { + LOGGER_ERROR("Failed to create archive: {}", + logger::errno_message(ec.value())); + return ec; + } + + LOGGER_INFO("Archive created in {}", ar.path()); + + ar.add_directory(d_src.canonical_path(), + d_dst.name(), + ec); + + if(ec) { + LOGGER_ERROR("Failed to add directory to archive: {}", + logger::errno_message(ec.value())); + return ec; + } + + input_path = ar.path().string(); + } + + // retrieve task context + const auto ctx = boost::any_cast< + std::shared_ptr< + hermes::request>>(task_info->context()); + auto req = std::move(*ctx); + + LOGGER_DEBUG("[{}] accept_pull: {} -> {}", task_info->id(), + d_src.canonical_path(), d_dst.to_string()); + + // create local buffers from local input data + auto input_buffer = + std::make_shared(input_path, + hermes::access_mode::read_only, + &ec); if(ec) { - *ctx = std::move(req); + LOGGER_ERROR("Failed mapping input data: {}", ec.message()); return ec; } - // let's prepare some local buffers - - std::vector bufseq{ - hermes::mutable_buffer{output_data->data(), output_data->size()} + std::vector bufvec{ + hermes::mutable_buffer{input_buffer->data(), input_buffer->size()} }; - hermes::exposed_memory local_buffers = - m_remote_endpoint->expose(bufseq, hermes::access_mode::write_only); + auto local_buffers = + m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); - LOGGER_DEBUG("pulling remote data into {}", output_path); + // retrieve remote buffers descriptor + hermes::exposed_memory remote_buffers = d_dst.buffers(); - auto start = std::chrono::steady_clock::now(); + LOGGER_DEBUG("remote_buffers{{count={}, total_size={}}}", + remote_buffers.count(), + remote_buffers.size()); - // N.B. IMPORTANT: we NEED to capture output_data by value here so that + // N.B. IMPORTANT: we NEED to capture input_buffer by value here so that // the mapped_buffer doesn't get released before completion_callback() // is called. - const auto completion_callback = - [this, is_collection, output_path, d_dst, output_data, start]( - hermes::request&& req) { - - uint32_t usecs = - std::chrono::duration_cast( - std::chrono::steady_clock::now() - start).count(); + const auto completion_callback = + [this, is_collection, input_path, input_buffer]( + hermes::request&& req) { // default response - rpc::remote_transfer::output out( + rpc::pull_resource::output out( static_cast(task_status::finished), static_cast(urd_error::success), 0, - usecs); + 0); //TODO: hermes offers no way to check for an error yet - LOGGER_DEBUG("Pull completed ({} usecs)", usecs); + LOGGER_DEBUG("Push completed"); if(is_collection) { - std::error_code ec; boost::system::error_code bec; - tar ar(output_path, tar::open, ec); - - if(ec) { - LOGGER_ERROR("Failed to open archive {}: {}", - output_path, logger::errno_message(ec.value())); - - out = std::move(rpc::remote_transfer::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(ec.value()), - 0 - }); - goto respond; - } - - ar.extract(d_dst.parent()->mount(), ec); - - if(ec) { - LOGGER_ERROR("Failed to extact archive into {}: {}", - ar.path(), d_dst.parent()->mount(), - logger::errno_message(ec.value())); - out = std::move(rpc::remote_transfer::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(ec.value()), - 0 - }); - goto respond; - } - - LOGGER_DEBUG("Archive {} extracted into {}", - ar.path(), d_dst.parent()->mount()); - - bfs::remove(ar.path(), bec); + bfs::remove(input_path, bec); if(bec) { LOGGER_ERROR("Failed to remove archive {}: {}", - ar.path(), logger::errno_message(ec.value())); - out = std::move(rpc::remote_transfer::output{ + input_path, bec.message()); + out = std::move(rpc::pull_resource::output{ static_cast(task_status::finished_with_error), static_cast(urd_error::system_error), - static_cast(ec.value()), + static_cast(bec.value()), 0 }); + goto respond; } } respond: if(req.requires_response()) { - m_remote_endpoint->respond( + m_network_endpoint->respond( std::move(req), - static_cast(task_status::finished), - static_cast(urd_error::success), - 0, - usecs); + out); } }; - m_remote_endpoint->async_pull(remote_buffers, - local_buffers, - std::move(req), - completion_callback); - -#endif - return std::make_error_code(static_cast(0)); -} - - -std::error_code -remote_resource_to_local_path_transferor::accept_transfer( - const auth::credentials& auth, - const std::shared_ptr& task_info, - const std::shared_ptr& src, - const std::shared_ptr& dst) const { - - (void) auth; - (void) task_info; - (void) src; - (void) dst; + m_network_endpoint->async_push(local_buffers, + remote_buffers, + std::move(req), + completion_callback); return std::make_error_code(static_cast(0)); } diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index a8bdc0b..a83688f 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -60,7 +60,7 @@ namespace io { struct remote_resource_to_local_path_transferor : public transferor { remote_resource_to_local_path_transferor( - std::shared_ptr remote_endpoint); + std::shared_ptr network_endpoint); bool validate(const std::shared_ptr& src_info, @@ -89,7 +89,7 @@ private: void do_accept(hermes::request&& req); - std::shared_ptr m_remote_endpoint; + std::shared_ptr m_network_endpoint; }; } // namespace io diff --git a/src/resources/local_posix_path/detail/local-path-impl.cpp b/src/resources/local_posix_path/detail/local-path-impl.cpp index 6ba8ebf..6568583 100644 --- a/src/resources/local_posix_path/detail/local-path-impl.cpp +++ b/src/resources/local_posix_path/detail/local-path-impl.cpp @@ -71,10 +71,13 @@ local_path_resource::is_collection() const { std::size_t local_path_resource::packed_size() const { std::error_code ec; - std::size_t sz = - utils::tar::estimate_size_once_packed(m_canonical_path, ec); + boost::system::error_code error; - return ec ? 0 : sz; + std::size_t sz = m_is_collection ? + utils::tar::estimate_size_once_packed(m_canonical_path, ec) : + bfs::file_size(m_canonical_path, error); + + return (ec || error) ? 0 : sz; } const std::shared_ptr diff --git a/src/rpcs.cpp b/src/rpcs.cpp index f53ca92..05df83d 100644 --- a/src/rpcs.cpp +++ b/src/rpcs.cpp @@ -10,6 +10,7 @@ namespace hermes { namespace detail { void register_user_request_types() { (void) registered_requests().add(); + (void) registered_requests().add(); (void) registered_requests().add(); } diff --git a/src/rpcs.hpp b/src/rpcs.hpp index 77fb482..f63b798 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -49,6 +49,21 @@ MERCURY_GEN_PROC(remote_transfer_out_t, ((uint32_t) (sys_errnum)) ((uint32_t) (elapsed_time))) +MERCURY_GEN_PROC(pull_resource_in_t, + ((hg_const_string_t) (in_nsid)) + ((hg_const_string_t) (in_resource_name)) + ((uint32_t) (in_resource_type)) + ((hg_const_string_t) (out_address)) + ((hg_const_string_t) (out_nsid)) + ((hg_const_string_t) (out_resource_name)) + ((hg_bulk_t) (out_buffers))) + +MERCURY_GEN_PROC(pull_resource_out_t, + ((uint32_t) (status)) + ((uint32_t) (task_error)) + ((uint32_t) (sys_errnum)) + ((uint32_t) (elapsed_time))) + MERCURY_GEN_PROC(resource_stat_in_t, ((hg_const_string_t) (address)) ((hg_const_string_t) (nsid)) @@ -56,8 +71,10 @@ MERCURY_GEN_PROC(resource_stat_in_t, ((hg_const_string_t) (resource_name))) MERCURY_GEN_PROC(resource_stat_out_t, - ((uint32_t) (return_value)) - ((uint64_t) (packed_size))) + ((uint32_t) (task_error)) + ((uint32_t) (sys_errnum)) + ((hg_bool_t) (is_collection)) + ((uint64_t) (packed_size))) }} // namespace hermes::detail @@ -407,6 +424,317 @@ struct remote_transfer { }; }; +struct pull_resource { + + // forward declarations of public input/output types for this RPC + class input; + class output; + + // traits used so that the engine knows what to do with the RPC + using self_type = pull_resource; + using handle_type = hermes::rpc_handle; + using input_type = input; + using output_type = output; + using mercury_input_type = hermes::detail::pull_resource_in_t; + using mercury_output_type = hermes::detail::pull_resource_out_t; + + // RPC public identifier + constexpr static const uint16_t public_id = 44; + + // RPC internal Mercury identifier + constexpr static const uint16_t mercury_id = public_id; + + // RPC name + constexpr static const auto name = "pull_resource"; + + // requires response? + constexpr static const auto requires_response = true; + + // Mercury callback to serialize input arguments + constexpr static const auto mercury_in_proc_cb = + HG_GEN_PROC_NAME(pull_resource_in_t); + + // Mercury callback to serialize output arguments + constexpr static const auto mercury_out_proc_cb = + HG_GEN_PROC_NAME(pull_resource_out_t); + + class input { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + input(const std::string& in_nsid, + const std::string& in_resource_name, + uint32_t in_resource_type, + const std::string& out_address, + const std::string& out_nsid, + const std::string& out_resource_name, + const hermes::exposed_memory& out_buffers) : + m_in_nsid(in_nsid), + m_in_resource_name(in_resource_name), + m_in_resource_type(in_resource_type), + m_out_address(out_address), + m_out_nsid(out_nsid), + m_out_resource_name(out_resource_name), + m_out_buffers(out_buffers) { + +#ifdef HERMES_DEBUG_BUILD + this->print("this", __PRETTY_FUNCTION__); +#endif + + } + +#ifdef HERMES_DEBUG_BUILD + input(input&& rhs) : + m_in_nsid(std::move(rhs.m_in_nsid)), + m_out_address(std::move(rhs.m_out_address)), + m_out_nsid(std::move(rhs.m_out_nsid)), + m_in_resource_type(std::move(rhs.m_in_resource_type)), + m_resource_name(std::move(rhs.m_resource_name)), + m_buffers(std::move(rhs.m_buffers)) { + + rhs.m_in_resource_type = 0; + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + } + + input(const input& other) : + m_in_nsid(other.m_in_nsid), + m_out_address(other.m_out_address), + m_out_nsid(other.m_out_nsid), + m_in_resource_type(other.m_in_resource_type), + m_resource_name(other.m_resource_name), + m_buffers(other.m_buffers) { + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + } + + input& + operator=(input&& rhs) { + + if(this != &rhs) { + m_in_nsid = std::move(rhs.m_in_nsid); + m_out_address = std::move(rhs.m_out_address); + m_out_nsid = std::move(rhs.m_out_nsid); + m_in_resource_type = std::move(rhs.m_in_resource_type); + m_resource_name = std::move(rhs.m_resource_name); + m_buffers = std::move(rhs.m_buffers); + + rhs.m_in_resource_type = 0; + rhs.m_is_collection = false; + } + + this->print("this", __PRETTY_FUNCTION__); + rhs.print("rhs", __PRETTY_FUNCTION__); + + return *this; + } + + input& + operator=(const input& other) { + + if(this != &other) { + m_in_nsid = other.m_in_nsid; + m_out_address = other.m_out_address; + m_out_nsid = other.m_out_nsid; + m_in_resource_type = other.m_in_resource_type; + m_resource_name = other.m_resource_name; + m_buffers = other.m_buffers; + } + + this->print("this", __PRETTY_FUNCTION__); + other.print("other", __PRETTY_FUNCTION__); + + return *this; + } +#else // HERMES_DEBUG_BUILD + input(input&& rhs) = default; + input(const input& other) = default; + input& operator=(input&& rhs) = default; + input& operator=(const input& other) = default; +#endif // ! HERMES_DEBUG_BUILD + + std::string + in_nsid() const { + return m_in_nsid; + } + + uint32_t + in_resource_type() const { + return m_in_resource_type; + } + + std::string + in_resource_name() const { + return m_in_resource_name; + } + + std::string + out_address() const { + return m_out_address; + } + + std::string + out_nsid() const { + return m_out_nsid; + } + + std::string + out_resource_name() const { + return m_out_resource_name; + } + + hermes::exposed_memory + out_buffers() const { + return m_out_buffers; + } + +#ifdef HERMES_DEBUG_BUILD + void + print(const std::string& id, + const std::string& caller = "") const { + + (void) id; + auto c = caller.empty() ? "unknown_caller" : caller; + + HERMES_DEBUG2("{}, {} ({}) = {{", caller, id, fmt::ptr(this)); + HERMES_DEBUG2(" m_in_nsid: \"{}\" ({} -> {}),", + m_in_nsid, fmt::ptr(&m_in_nsid), + fmt::ptr(m_in_nsid.c_str())); + HERMES_DEBUG2(" m_in_resource_name: \"{}\" ({} -> {}),", + m_in_resource_name, fmt::ptr(&m_in_resource_name), + fmt::ptr(m_in_resource_name.c_str())); + HERMES_DEBUG2(" m_in_resource_type: {},", + m_in_resource_type); + HERMES_DEBUG2(" m_out_address: \"{}\" ({} -> {}),", + m_out_address, fmt::ptr(&m_out_address), + fmt::ptr(m_out_address.c_str())); + HERMES_DEBUG2(" m_out_nsid: \"{}\" ({} -> {}),", + m_out_nsid, fmt::ptr(&m_out_nsid), + fmt::ptr(m_out_nsid.c_str())); + HERMES_DEBUG2(" m_out_resource_name: \"{}\" ({} -> {}),", + m_out_resource_name, fmt::ptr(&m_out_resource_name), + fmt::ptr(m_out_resource_name.c_str())); + HERMES_DEBUG2(" m_backend_type: {},", + m_backend_type); + HERMES_DEBUG2(" m_out_buffers: {...},"); + HERMES_DEBUG2("}}"); + } +#endif // ! HERMES_DEBUG_BUILD + +//TODO: make private + explicit + input(const hermes::detail::pull_resource_in_t& other) : + m_in_nsid(other.in_nsid), + m_in_resource_name(other.in_resource_name), + m_in_resource_type(other.in_resource_type), + m_out_address(other.out_address), + m_out_nsid(other.out_nsid), + m_out_resource_name(other.out_resource_name), + m_out_buffers(other.out_buffers) { + +#ifdef HERMES_DEBUG_BUILD + this->print("this", __PRETTY_FUNCTION__); + rhs.print("other", __PRETTY_FUNCTION__); +#endif + } + + explicit + operator hermes::detail::pull_resource_in_t() { + return {m_in_nsid.c_str(), + m_in_resource_name.c_str(), + m_in_resource_type, + m_out_address.c_str(), + m_out_nsid.c_str(), + m_out_resource_name.c_str(), + hg_bulk_t(m_out_buffers)}; + } + + + private: + std::string m_in_nsid; + std::string m_in_resource_name; + uint32_t m_in_resource_type; + std::string m_out_address; + std::string m_out_nsid; + std::string m_out_resource_name; + hermes::exposed_memory m_out_buffers; + }; + + class output { + + template + friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); + + public: + output(uint32_t status, + uint32_t task_error, + uint32_t sys_errnum, + uint32_t elapsed_time) : + m_status(status), + m_task_error(task_error), + m_sys_errnum(sys_errnum), + m_elapsed_time(elapsed_time) {} + + uint32_t + status() const { + return m_status; + } + + void + set_retval(uint32_t status) { + m_status = status; + } + + uint32_t + task_error() const { + return m_task_error; + } + + void + set_task_error(uint32_t task_error) { + m_task_error = task_error; + } + + uint32_t + sys_errnum() const { + return m_sys_errnum; + } + + uint32_t + elapsed_time() const { + return m_elapsed_time; + } + + void + set_sys_errnum(uint32_t errnum) { + m_sys_errnum = errnum; + } + + explicit + output(const hermes::detail::pull_resource_out_t& out) { + m_status = out.status; + m_task_error = out.task_error; + m_sys_errnum = out.sys_errnum; + m_elapsed_time = out.elapsed_time; + } + + explicit + operator hermes::detail::pull_resource_out_t() { + return {m_status, m_task_error, m_sys_errnum, m_elapsed_time}; + } + + private: + uint32_t m_status; + uint32_t m_task_error; + uint32_t m_sys_errnum; + uint32_t m_elapsed_time; + }; +}; + struct resource_stat { // forward declarations of public input/output types for this RPC class input; @@ -421,7 +749,7 @@ struct resource_stat { using mercury_output_type = hermes::detail::resource_stat_out_t; // RPC public identifier - constexpr static const uint16_t public_id = 44; + constexpr static const uint16_t public_id = 45; // RPC internal Mercury identifier constexpr static const uint16_t mercury_id = public_id; @@ -611,14 +939,28 @@ struct resource_stat { friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); public: - output(uint32_t return_value, + output(uint32_t task_error, + uint32_t sys_errnum, + bool is_collection, uint64_t packed_size) : - m_return_value(return_value), + m_task_error(task_error), + m_sys_errnum(sys_errnum), + m_is_collection(is_collection), m_packed_size(packed_size) {} uint32_t - return_value() const { - return m_return_value; + task_error() const { + return m_task_error; + } + + uint32_t + sys_errnum() const { + return m_sys_errnum; + } + + bool + is_collection() const { + return m_is_collection; } uint64_t @@ -628,17 +970,22 @@ struct resource_stat { explicit output(const hermes::detail::resource_stat_out_t& out) { - m_return_value = out.return_value; + m_task_error = out.task_error; + m_sys_errnum = out.sys_errnum; + m_is_collection = out.is_collection; m_packed_size = out.packed_size; } explicit operator hermes::detail::resource_stat_out_t() { - return {m_return_value, m_packed_size}; + return {m_task_error, m_sys_errnum, + m_is_collection, m_packed_size}; } private: - uint32_t m_return_value; + uint32_t m_task_error; + uint32_t m_sys_errnum; + bool m_is_collection; uint64_t m_packed_size; }; }; diff --git a/src/urd.cpp b/src/urd.cpp index 1d87682..b9cda35 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -190,8 +190,7 @@ urd_error urd::validate_iotask_args(iotask_type type, return urd_error::bad_args; } - // src_resource cannot be remote - if(src_rinfo->type() == data::resource_type::remote_posix_path) { + if(src_rinfo->is_remote() && dst_rinfo()->is_remote()) { return urd_error::not_supported; } @@ -243,10 +242,10 @@ urd::iotask_create_handler(const request_ptr base_request) { goto log_and_return; } - if(src_rinfo->is_remote()) { - rv = urd_error::not_supported; - goto log_and_return; - } +// if(src_rinfo->is_remote()) { +// rv = urd_error::not_supported; +// goto log_and_return; +// } for(const auto& rinfo : {src_rinfo, dst_rinfo}) { if(rinfo) { @@ -792,6 +791,114 @@ urd::remote_transfer_handler(hermes::request&& req) { } } +void +urd::pull_resource_handler(hermes::request&& req) { + + const auto args = req.args(); + + LOGGER_WARN("incoming rpc::push_request(from: \"{}:{}\", to: \"{}@{}:{}\")", + args.in_nsid(), + args.in_resource_name(), + args.out_address(), + args.out_nsid(), + args.out_resource_name()); + + urd_error rv = urd_error::success; + boost::optional tsk; + boost::optional> src_backend; + std::shared_ptr dst_backend; + + /// XXX this information should be retrievable from the backend + auto src_rtype = static_cast(args.in_resource_type()); + + auth::credentials auth; //XXX fake credentials for now + + const auto create_rinfo = + [&](const data::resource_type& rtype) -> + std::shared_ptr { + + switch(rtype) { + case data::resource_type::remote_resource: + return std::make_shared( + args.out_address(), args.out_nsid(), false, + args.out_resource_name(), args.out_buffers()); + case data::resource_type::local_posix_path: + case data::resource_type::shared_posix_path: + return std::make_shared( + args.in_nsid(), args.in_resource_name()); + default: + rv = urd_error::not_supported; + return {}; + } + }; + + if(m_is_paused) { + rv = urd_error::accept_paused; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(rv), + 0, + 0); + return; + } + + auto src_rinfo = create_rinfo(src_rtype); + auto dst_rinfo = create_rinfo(data::resource_type::remote_resource); + + //TODO: check src_rinfo and dst_rinfo + + + // TODO: actually retrieve and validate credentials, etc + + { + boost::shared_lock lock(m_namespace_mgr_mutex); + src_backend = m_namespace_mgr->find(args.in_nsid()); + } + + if(!src_backend) { + rv = urd_error::no_such_namespace; + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(rv), + 0, + 0); + return; + } + + LOGGER_DEBUG("nsid: {}, bptr: {}", args.in_nsid(), src_backend); + + dst_backend = + std::make_shared(args.out_nsid()); + + auto ctx = + std::make_shared>(std::move(req)); + + std::tie(rv, tsk) = + m_task_mgr->create_remote_initiated_task( + iotask_type::remote_transfer, auth, + ctx, *src_backend, src_rinfo, + dst_backend, dst_rinfo); + + if(rv == urd_error::success) { + // run the task and check that it started correctly + (*tsk)(); + + auto req = std::move(*ctx); + + if(tsk->info()->status() == io::task_status::finished_with_error) { + rv = tsk->info()->task_error(); + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(io::task_status::finished_with_error), + static_cast(tsk->info()->task_error()), + static_cast(tsk->info()->sys_error().value()), + 0); + } + } +} + void urd::resource_stat_handler(hermes::request&& req) { @@ -814,6 +921,12 @@ urd::resource_stat_handler(hermes::request&& req) { LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); m_network_endpoint->respond(std::move(req), static_cast(rv), + false, + //XXX ENOENT should not be required: + // the transfer() interface should be + // richer rather than returning only a + // std::error_code + ENOENT, 0); return; } @@ -830,23 +943,41 @@ urd::resource_stat_handler(hermes::request&& req) { return std::make_shared( args.nsid(), args.resource_name()); default: - rv = urd_error::not_supported; return {}; } }; auto rinfo = create_rinfo(rtype); + if(!rinfo) { + LOGGER_ERROR("Failed to access resource {}", rinfo->to_string()); + rv = urd_error::not_supported; + + LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); + m_network_endpoint->respond(std::move(req), + static_cast(rv), + //XXX EOPNOTSUPP should not be required: + // the transfer() interface should be + // richer rather than returning only a + // std::error_code + EOPNOTSUPP, + false, + 0); + return; + } + std::error_code ec; auto rsrc = (*dst_backend)->get_resource(rinfo, ec); if(ec) { LOGGER_ERROR("Failed to access resource {}", rinfo->to_string()); - rv = urd_error::snafu; + rv = urd_error::no_such_resource; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); m_network_endpoint->respond(std::move(req), static_cast(rv), + ec.value(), + false, 0); return; } @@ -854,6 +985,8 @@ urd::resource_stat_handler(hermes::request&& req) { LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); m_network_endpoint->respond(std::move(req), static_cast(urd_error::success), + 0, + rsrc->is_collection(), rsrc->packed_size()); } @@ -1082,6 +1215,10 @@ void urd::init_event_handlers() { std::bind(&urd::remote_transfer_handler, this, std::placeholders::_1)); + m_network_endpoint->register_handler( + std::bind(&urd::pull_resource_handler, this, + std::placeholders::_1)); + m_network_endpoint->register_handler( std::bind(&urd::resource_stat_handler, this, std::placeholders::_1)); diff --git a/src/urd.hpp b/src/urd.hpp index fa0c484..c8c9f99 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -73,6 +73,7 @@ namespace ns { namespace rpc { struct remote_transfer; + struct pull_resource; struct resource_stat; } @@ -124,6 +125,7 @@ private: response_ptr unknown_request_handler(const request_ptr req); void remote_transfer_handler(hermes::request&& req); + void pull_resource_handler(hermes::request&& req); void resource_stat_handler(hermes::request&& req); // TODO: add helpers for remove and update diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp index 85e79be..dce9c09 100644 --- a/tests/api-copy-remote-data.cpp +++ b/tests/api-copy-remote-data.cpp @@ -33,10 +33,16 @@ namespace bfs = boost::filesystem; -SCENARIO("copy local POSIX file to remote POSIX file", - "[api::norns_submit_remote_copy_local_posix_files]") { +/******************************************************************************/ +/* tests for push transfers (errors) */ +/******************************************************************************/ +SCENARIO("errors copying local POSIX path to remote POSIX path", + "[api::norns_submit_push_errors]") { GIVEN("a running urd instance") { + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ test_env env(false); const char* nsid0 = "tmp0"; @@ -134,12 +140,11 @@ SCENARIO("copy local POSIX file to remote POSIX file", bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); REQUIRE(!ec); - // create required output directories env.add_to_namespace(nsid1, dst_subdir1); /**********************************************************************/ - /* tests for error conditions */ + /* begin tests */ /**********************************************************************/ // - trying to copy a non-existing file WHEN("copying a non-existing NORNS_LOCAL_PATH file") { @@ -153,14 +158,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { @@ -188,14 +193,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { @@ -223,14 +228,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_SUCCESS and ENOENT are reported") { @@ -264,14 +269,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " @@ -305,14 +310,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { @@ -343,14 +348,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " @@ -383,14 +388,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " @@ -423,14 +428,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { @@ -462,14 +467,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " @@ -490,7 +495,7 @@ SCENARIO("copy local POSIX file to remote POSIX file", // symlink leading out of namespace WHEN("copying a NORNS_LOCAL_PATH through a symbolic link that leads " - "out of the src namespace") { + "out of the SRC namespace") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, @@ -502,14 +507,14 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { @@ -527,13 +532,128 @@ SCENARIO("copy local POSIX file to remote POSIX file", #endif + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for push transfers (single files) */ +/******************************************************************************/ +SCENARIO("copy local POSIX file to remote POSIX file", + "[api::norns_submit_push_to_posix_file]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ - /* tests for single files */ + /* begin tests */ /**********************************************************************/ // cp -r ns0://file0.txt // -> ns1:// = ns1://file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_REMOTE_PATH at dst namespace's root " + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_REMOTE_PATH at DST namespace's root " "(keeping the name)") { norns_iotask_t task = @@ -545,24 +665,34 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + THEN("Files are equal") { - REQUIRE(compare_files(src, dst) == true); + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -570,8 +700,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://file0.txt // -> ns1://file1.txt = ns1://file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at dst namespace's root " + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at DST namespace's root " "(changing the name)") { norns_iotask_t task = @@ -583,23 +713,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root1); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -607,8 +747,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../d/file0.txt // -> ns1://file0.txt = ns1://file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at dst namespace's root (keeping " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root (keeping " "the name)") { norns_iotask_t task = @@ -620,60 +760,82 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root0); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } + } } } // cp -r ns0://a/b/c/.../d/file0.txt // -> ns1://file1.txt = ns1://file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at dst namespace's root (changing " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root (changing " "the name)") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid0, + src_file_at_subdir.c_str()), NORNS_REMOTE_PATH(nsid1, remote_host, dst_file_at_root1.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - REQUIRE(compare_files(src, dst) == true); + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root1); + + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -681,8 +843,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://file0.txt // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " "(keeping the name)") { norns_iotask_t task = @@ -694,23 +856,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir0); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir0); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -718,8 +890,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://file0.txt // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from src namespace's root to " - "another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " "(changing the name)") { norns_iotask_t task = @@ -731,23 +903,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_root); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir1); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -755,8 +937,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../file0.txt // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " "(keeping the name)") { norns_iotask_t task = @@ -768,23 +950,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir0); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir0); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -792,8 +984,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../file0.txt // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " "(changing the name)") { norns_iotask_t task = @@ -805,23 +997,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - REQUIRE(compare_files(src, dst) == true); + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir1); + + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -829,8 +1031,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../file0.txt // -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " "(changing the parents names)") { norns_iotask_t task = @@ -842,23 +1044,33 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir2); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir2); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } @@ -866,8 +1078,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r ns0://a/b/c/.../file0.txt // -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt - WHEN("copying a single NORNS_LOCAL_PATH from a src namespace's subdir " - "to another NORNS_LOCAL_PATH at a dst namespace's subdir " + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " "(changing the name)") { norns_iotask_t task = @@ -879,61 +1091,195 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_file_at_subdir); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_subdir3); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir3); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } } + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for push transfers (directories) */ +/******************************************************************************/ +SCENARIO("copy local POSIX file to remote POSIX subdir", + "[api::norns_submit_push_to_posix_subdir]") { + GIVEN("a running urd instance") { + /**********************************************************************/ - /* tests for directories */ + /* setup common environment */ /**********************************************************************/ - // cp -r /a/contents.* -> / = /contents.* - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to dst namespace's root\n" - " cp -r /a/contents.* -> / = /contents.* ") { + test_env env(false); - norns_iotask_t task = - NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), - NORNS_REMOTE_PATH(nsid1, - remote_host, - dst_root.c_str())); + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; - norns_error_t rv = norns_submit(&task); + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); - THEN("NORNS_SUCCESS is returned") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; - // wait until the task completes - rv = norns_wait(&task); + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible - THEN("NORNS_SUCCESS is returned") { - REQUIRE(rv == NORNS_SUCCESS); + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir0); - bfs::path dst = env.get_from_namespace(nsid1, dst_root); + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname - REQUIRE(compare_directories(src, dst) == true); + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + // cp -r /a/contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to DST namespace's root\n" + " cp -r /a/contents.* -> / = /contents.* ") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -941,7 +1287,7 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> / = /contents.* WHEN("copying the contents of a NORNS_LOCAL_PATH arbitrary subdir to " - "dst namespace's root\n" + "DST namespace's root\n" " cp -r /a/b/c/.../contents.* -> / = /contents.*") { norns_iotask_t task = @@ -953,23 +1299,31 @@ SCENARIO("copy local POSIX file to remote POSIX file", norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_root); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_root); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -977,8 +1331,8 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/contents.* -> /c = /c/contents.* // (c did not exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to another NORNS_REMOTE_PATH subdir at DST " "namespace's root while changing its name\n" " cp -r /a/contents.* -> /c = /c/contents.*") { @@ -987,27 +1341,1931 @@ SCENARIO("copy local POSIX file to remote POSIX file", NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), NORNS_REMOTE_PATH(nsid1, remote_host, - dst_subdir0.c_str())); + dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + // cp -r /a/contents.* -> /c = /c/contents.* + // (c did exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to another NORNS_REMOTE_PATH subdir at DST " + "namespace's root while changing its name\n" + " cp -r /a/contents.* -> /c / /c/contents.*") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* + // (c did not exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to a NORNS_REMOTE_PATH subdir at DST " + "namespace's root while changing its name\n" + "cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* + // (c did exist previously) + WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from SRC " + "namespace's root to another NORNS_REMOTE_PATH subdir at DST " + "namespace's root while changing its name:" + " cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + env.notify_success(); + } + +#ifndef USE_REAL_DAEMON + GIVEN("a non-running urd instance") { + WHEN("attempting to request a transfer") { + + norns_iotask_t task = NORNS_IOTASK( + NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), + NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_ECONNFAILED is returned") { + REQUIRE(rv == NORNS_ECONNFAILED); + } + } + } +#endif +} + +/******************************************************************************/ +/* tests for push transfers (links) */ +/******************************************************************************/ +SCENARIO("copy local POSIX path to remote POSIX path involving links", + "[api::norns_submit_push_links]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + WHEN("copying a single NORNS_LOCAL_PATH file from SRC namespace's '/' " + "through a symlink also located at '/'" ) { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH subdir from SRC " + "namespace's '/' through a symlink also located at '/'" ) { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink also located at '/'" ) { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_root2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH file from SRC namespace's '/' " + "through a symlink located in a subdir" ) { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH subdir from SRC " + "namespace's '/' through a symlink also located at subdir" ) { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink also located at a subdir" ) { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_symlink_at_subdir2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for pull transfers (errors) */ +/******************************************************************************/ +SCENARIO("errors copying remote POSIX path to local POSIX path", + "[api::norns_submit_pull_errors]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + // - trying to copy a non-existing file + WHEN("copying a non-existing NORNS_LOCAL_PATH file") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_invalid_file.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + + // - trying to copy a non-existing directory + WHEN("copying a non-existing NORNS_LOCAL_PATH directory") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_invalid_dir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + + // - trying to copy an empty directory + WHEN("copying an empty NORNS_LOCAL_PATH directory") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_empty_dir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_SUCCESS and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + REQUIRE(stats.st_task_error == NORNS_SUCCESS); + REQUIRE(stats.st_sys_errno == 0); + + REQUIRE(bfs::exists(dst_mnt / dst_file_at_root0)); + } + } + } + } + +//FIXME: DISABLED in CI until impersonation is implemented or capabilities can be added to the docker service +#ifdef __SETCAP_TESTS__ + +#if 0 // not adapted to pull semantics + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from \"/\" without appropriate " + "permissions to access it") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without " + "appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without " + "appropriate permissions to access a parent") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, src_noperms_file2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from \"/\" without " + "appropriate permissions to access it") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir0.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir " + "without appropriate permissions to access it") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir1.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without " + "appropriate permissions to access a parent") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + src_noperms_subdir2.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL " + "are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // symlink leading out of namespace + WHEN("copying a NORNS_LOCAL_PATH through a symbolic link that leads " + "out of the SRC namespace") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH(nsid0, + out_symlink.c_str()), + NORNS_REMOTE_PATH(nsid1, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } +#endif +#endif + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for pull transfers (single files) */ +/******************************************************************************/ +SCENARIO("copy remote POSIX file to local POSIX file", + "[api::norns_submit_pull_to_posix_file]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + // cp -r ns0://file0.txt + // -> ns1:// = ns1://file0.txt + WHEN("copying a single NORNS_REMOTE_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at DST namespace's root " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://file0.txt + // -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at DST namespace's root " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace(nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file0.txt = ns1://file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root (keeping " + "the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../d/file0.txt + // -> ns1://file1.txt = ns1://file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at DST namespace's root (changing " + "the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_root1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace(nsid0, + src_file_at_root); + bfs::path dst = + env.get_from_namespace(nsid1, + dst_file_at_subdir0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from SRC namespace's root to " + "another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_root.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_root); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file0.txt = ns1://a/b/c/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(keeping the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir0); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://a/b/c/.../file1.txt = ns1://a/b/c/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir1); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file0.txt = ns1://e/f/g/.../file0.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the parents names)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir2.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir2); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + // cp -r ns0://a/b/c/.../file0.txt + // -> ns1://e/f/g/.../file1.txt = ns1://e/f/g/.../file1.txt + WHEN("copying a single NORNS_LOCAL_PATH from a SRC namespace's subdir " + "to another NORNS_LOCAL_PATH at a DST namespace's subdir " + "(changing the name)") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_file_at_subdir.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_subdir3.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { + bfs::path src = + env.get_from_namespace( + nsid0, src_file_at_subdir); + bfs::path dst = + env.get_from_namespace( + nsid1, dst_file_at_subdir3); + + REQUIRE(compare_files(src, dst) == true); + } + } + } + } + } + + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for pull transfers (directories) */ +/******************************************************************************/ +SCENARIO("copy remote POSIX subdir to local POSIX subdir", + "[api::norns_submit_pull_to_posix_subdir]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + // cp -r /a/contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_REMOTE_PATH subdir from SRC " + "namespace's root to a NORNS_LOCAL_PATH at DST namespace's root\n" + " cp -r /a/contents.* -> / = /contents.* ") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied subdirs are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + // cp -r /a/b/c/.../contents.* -> / = /contents.* + WHEN("copying the contents of a NORNS_REMOTE_PATH arbitrary subdir to " + "a NORNS_LOCAL_PATH at DST namespace's root\n" + " cp -r /a/b/c/.../contents.* -> / = /contents.*") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied subdirs are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_root); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + // cp -r /a/contents.* -> /c = /c/contents.* + // (c did not exist previously) + WHEN("copying the contents of a NORNS_REMOTE_PATH subdir from SRC " + "namespace's root to another NORNS_LOCAL_PATH subdir at DST " + "namespace's root while changing its name\n" + " cp -r /a/contents.* -> /c = /c/contents.*") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + remote_host, + src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_subdir0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir0); - bfs::path dst = - env.get_from_namespace(nsid1, dst_subdir0); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied subdirs are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -1015,37 +3273,46 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/contents.* -> /c = /c/contents.* // (c did exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + WHEN("copying the contents of a NORNS_REMOTE_PATH subdir from SRC " + "namespace's root to another NORNS_LOCAL_PATH subdir at DST " "namespace's root while changing its name\n" " cp -r /a/contents.* -> /c / /c/contents.*") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_subdir1.c_str())); + src_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_subdir1.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir0); - bfs::path dst = - env.get_from_namespace(nsid1, dst_subdir1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -1053,37 +3320,46 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* // (c did not exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to a NORNS_REMOTE_PATH subdir at dst " + WHEN("copying the contents of a NORNS_REMOTE_PATH subdir from SRC " + "namespace's root to a NORNS_LOCAL_PATH subdir at DST " "namespace's root while changing its name\n" "cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_subdir0.c_str())); + src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_subdir0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_subdir0); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -1091,261 +3367,451 @@ SCENARIO("copy local POSIX file to remote POSIX file", // cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.* // (c did exist previously) - WHEN("copying the contents of a NORNS_LOCAL_PATH subdir from src " - "namespace's root to another NORNS_REMOTE_PATH subdir at dst " + WHEN("copying the contents of a NORNS_REMOTE_PATH subdir from SRC " + "namespace's root to another NORNS_LOCAL_PATH subdir at DST " "namespace's root while changing its name:" " cp -r /a/b/c/.../contents.* -> /c = /c/a/b/c/.../contents.*") { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_subdir1.c_str())); + src_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_subdir1.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_subdir1); + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } } - /**************************************************************************************************************/ - /* tests for soft links */ - /**************************************************************************************************************/ - WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " + env.notify_success(); + } + +#ifndef USE_REAL_DAEMON + GIVEN("a non-running urd instance") { + WHEN("attempting to request a transfer") { + + norns_iotask_t task = NORNS_IOTASK( + NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), + NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_ECONNFAILED") { + REQUIRE(rv == NORNS_ECONNFAILED); + } + } + } +#endif +} + +/******************************************************************************/ +/* tests for pull transfers (links) */ +/******************************************************************************/ +SCENARIO("copy remote POSIX path to local POSIX path involving links", + "[api::norns_submit_pull_links]") { + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env(false); + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = + env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = + env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 40000); + env.add_to_namespace(nsid0, src_file_at_subdir, 80000); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**********************************************************************/ + /* begin tests */ + /**********************************************************************/ + WHEN("copying a single NORNS_REMOTE_PATH file from SRC namespace's '/' " "through a symlink also located at '/'" ) { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, + NORNS_REMOTE_PATH(nsid0, + remote_host, src_symlink_at_root0.c_str()), - NORNS_REMOTE_PATH(nsid1, - remote_host, - dst_file_at_root0.c_str())); + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); - bfs::path src = - env.get_from_namespace(nsid0, src_symlink_at_root0); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + THEN("Files are equal") { - REQUIRE(compare_files(src, dst) == true); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); + + REQUIRE(compare_files(src, dst) == true); + } } } } } - WHEN("copying a single NORNS_LOCAL_PATH subdir from src " + WHEN("copying a single NORNS_REMOTE_PATH subdir from SRC " "namespace's '/' through a symlink also located at '/'" ) { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, - src_symlink_at_root1.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_file_at_root0.c_str())); + src_symlink_at_root1.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Directories are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_symlink_at_root1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } } - WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + WHEN("copying a single NORNS_REMOTE_PATH arbitrary subdir" "through a symlink also located at '/'" ) { norns_op_t task_op = NORNS_IOTASK_COPY; norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, - src_symlink_at_root2.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_file_at_root0.c_str())); + src_symlink_at_root2.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Directories are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { - bfs::path src = - env.get_from_namespace(nsid0, src_symlink_at_root2); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, src_symlink_at_root2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } } - WHEN("copying a single NORNS_LOCAL_PATH file from src namespace's '/' " + WHEN("copying a single NORNS_REMOTE_PATH file from SRC namespace's '/' " "through a symlink located in a subdir" ) { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, - src_symlink_at_subdir0.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_file_at_root0.c_str())); + src_symlink_at_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Files are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Files are equal") { - bfs::path src = - env.get_from_namespace(nsid0, - src_symlink_at_subdir0); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir0); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); - REQUIRE(compare_files(src, dst) == true); + REQUIRE(compare_files(src, dst) == true); + } } } } } - WHEN("copying a single NORNS_LOCAL_PATH subdir from src " + WHEN("copying a single NORNS_REMOTE_PATH subdir from SRC " "namespace's '/' through a symlink also located at subdir" ) { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, - src_symlink_at_subdir1.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_file_at_root0.c_str())); + src_symlink_at_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Directories are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { - bfs::path src = - env.get_from_namespace(nsid0, - src_symlink_at_subdir1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } } - WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" + WHEN("copying a single NORNS_REMOTE_PATH arbitrary subdir" "through a symlink also located at a subdir" ) { norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH(nsid0, - src_symlink_at_subdir2.c_str()), - NORNS_REMOTE_PATH(nsid1, + NORNS_REMOTE_PATH(nsid0, remote_host, - dst_file_at_root0.c_str())); + src_symlink_at_subdir2.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file_at_root0.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_submit() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); REQUIRE(task.t_id != 0); // wait until the task completes rv = norns_wait(&task); - THEN("NORNS_SUCCESS is returned") { + THEN("norns_wait() returns NORNS_SUCCESS") { REQUIRE(rv == NORNS_SUCCESS); - THEN("Directories are equal") { + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Directories are equal") { - bfs::path src = - env.get_from_namespace(nsid0, - src_symlink_at_subdir2); - bfs::path dst = - env.get_from_namespace(nsid1, dst_file_at_root0); + bfs::path src = + env.get_from_namespace(nsid0, + src_symlink_at_subdir2); + bfs::path dst = + env.get_from_namespace(nsid1, dst_file_at_root0); - REQUIRE(compare_directories(src, dst) == true); + REQUIRE(compare_directories(src, dst) == true); + } } } } @@ -1353,24 +3819,6 @@ SCENARIO("copy local POSIX file to remote POSIX file", env.notify_success(); } - -#ifndef USE_REAL_DAEMON - GIVEN("a non-running urd instance") { - WHEN("attempting to request a transfer") { - - norns_iotask_t task = NORNS_IOTASK( - NORNS_IOTASK_COPY, - NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), - NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); - - norns_error_t rv = norns_submit(&task); - - THEN("NORNS_ECONNFAILED is returned") { - REQUIRE(rv == NORNS_ECONNFAILED); - } - } - } -#endif } @@ -1378,6 +3826,9 @@ SCENARIO("copy local POSIX file to remote POSIX file", SCENARIO("copy local memory buffer to local POSIX file", "[api::norns_submit_copy_buffer_to_file]") { GIVEN("a running urd instance") { + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ test_env env; const char* nsid0 = "tmp0"; diff --git a/tests/api-task-submit.cpp b/tests/api-task-submit.cpp index f71d778..a0531d4 100644 --- a/tests/api-task-submit.cpp +++ b/tests/api-task-submit.cpp @@ -208,17 +208,21 @@ SCENARIO("submit request", "[api::norns_submit]") { } /* using a remote node as source is not allowed (yet) */ - WHEN("submitting a request to copy from NORNS_REMOTE_PATH to NORNS_LOCAL_PATH") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_REMOTE_PATH(nsid0, src_host0, src_file0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file1.c_str())); + WHEN("submitting a request to copy from NORNS_REMOTE_PATH to " + "NORNS_LOCAL_PATH") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_REMOTE_PATH(nsid0, + src_host0, + src_file0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file1.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_ENOTSUPPORTED is returned") { - REQUIRE(rv == NORNS_ENOTSUPPORTED); + THEN("norns_submit() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); } } @@ -395,17 +399,21 @@ SCENARIO("submit request", "[api::norns_submit]") { } /* using a remote node as source is not allowed (yet) */ - WHEN("submitting a request to move from NORNS_REMOTE_PATH to NORNS_LOCAL_PATH") { - - norns_op_t task_op = NORNS_IOTASK_MOVE; - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_REMOTE_PATH(nsid0, src_host0, src_file0.c_str()), - NORNS_LOCAL_PATH(nsid1, dst_file1.c_str())); + WHEN("submitting a request to move from NORNS_REMOTE_PATH to " + "NORNS_LOCAL_PATH") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_MOVE, + NORNS_REMOTE_PATH(nsid0, + src_host0, + src_file0.c_str()), + NORNS_LOCAL_PATH(nsid1, + dst_file1.c_str())); norns_error_t rv = norns_submit(&task); - THEN("NORNS_ENOTSUPPORTED is returned") { - REQUIRE(rv == NORNS_ENOTSUPPORTED); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); } } -- GitLab From cb4c52d847dd42f7c2c4d7d0b54383bbb5ac48ec Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 14:50:09 +0100 Subject: [PATCH 22/51] Update hermes --- src/externals/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/externals/hermes b/src/externals/hermes index 37da27f..94b107c 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 37da27f9b6d6ed279a96f2fae01e92b1355ae8c5 +Subproject commit 94b107cbda556bc6814e0a0454a096d6cb5c7955 -- GitLab From 38bb1d2aa2ad803fcb8e31a35d62652058db1d42 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 14:57:00 +0100 Subject: [PATCH 23/51] Fix defect when fallocate() failed with EOPNOTSUPP Closes #24 --- configure.ac | 6 +++++ .../transferors/local-path-to-local-path.cpp | 17 ++++++++---- .../local-path-to-remote-resource.cpp | 27 +++++++++++-------- src/io/transferors/memory-to-local-path.cpp | 25 ++++++++++------- .../remote-resource-to-local-path.cpp | 20 +++++++++----- 5 files changed, 63 insertions(+), 32 deletions(-) diff --git a/configure.ac b/configure.ac index 01f1a80..daa129f 100644 --- a/configure.ac +++ b/configure.ac @@ -158,7 +158,13 @@ AC_SEARCH_LIBS([tar_open], [tar], AC_CHECK_HEADER_STDBOOL # Checks for library functions. +AC_CHECK_FUNC([fallocate],[fallocate],[fallocate]) +AS_IF([test "x${PROTOC}" == "x"], + [AC_MSG_ERROR([ProtoBuf compiler "protoc" not found.])]) +AC_CHECK_FUNC([fallocate], + [AC_DEFINE([HAVE_FALLOCATE], + [1], [Define if file preallocation is available])]) ################################################################################ ### write makefiles diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 2e7fe5a..644cee4 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -30,6 +30,7 @@ #include #include #include +#include "config.h" #include "utils.hpp" #include "logger.hpp" @@ -71,15 +72,21 @@ do_sendfile(int in_fd, int out_fd) { } // preallocate output file +#ifdef HAVE_FALLOCATE if(::fallocate(out_fd, 0, 0, sz) == -1) { + if(errno != EOPNOTSUPP) { + return static_cast(-1); + } +#endif // HAVE_FALLOCATE + // filesystem doesn't support fallocate(), fallback to truncate() - if(errno == EOPNOTSUPP) { - if(::ftruncate(out_fd, sz) != 0) { - return static_cast(-1); - } + if(::ftruncate(out_fd, sz) != 0) { + return static_cast(-1); } - return static_cast(-1); + +#ifdef HAVE_FALLOCATE } +#endif // HAVE_FALLOCATE // copy data off_t offset = 0; diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index a5faded..abf5ffe 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -30,6 +30,7 @@ #include #include #include +#include "config.h" #include "utils.hpp" #include "logger.hpp" @@ -58,17 +59,23 @@ create_file(const bfs::path& filename, } // preallocate output file +#ifdef HAVE_FALLOCATE if(::fallocate(out_fd, 0, 0, size) == -1) { + if(errno != EOPNOTSUPP) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } +#endif // HAVE_FALLOCATE + // filesystem doesn't support fallocate(), fallback to truncate() - if(errno == EOPNOTSUPP) { - if(::ftruncate(out_fd, size) != 0) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } + if(::ftruncate(out_fd, size) != 0) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); } - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + +#ifdef HAVE_FALLOCATE } +#endif // HAVE_FALLOCATE retry_close: if(close(out_fd) == -1) { @@ -213,10 +220,8 @@ local_path_to_remote_resource_transferor::transfer( LOGGER_DEBUG("};"); LOGGER_FLUSH(); - auto rpc = - m_network_endpoint->post(endp, args); - - auto resp = rpc.get(); + auto resp = + m_network_endpoint->post(endp, args).get(); if(static_cast(resp.at(0).status()) == task_status::finished_with_error) { diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index a6c80ac..ac660a8 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -63,19 +63,26 @@ copy_memory_region(const std::shared_ptr& task_info, return std::make_error_code(static_cast(errno)); } + // preallocate output file +#ifdef HAVE_FALLOCATE if(::fallocate(out_fd, 0, 0, size) == -1) { + if(errno != EOPNOTSUPP) { + LOGGER_ERROR("fallocate() error"); + rv = errno; + goto cleanup_on_error; + } +#endif // HAVE_FALLOCATE + // filesystem doesn't support fallocate(), fallback to truncate() - if(errno == EOPNOTSUPP) { - if(::ftruncate(out_fd, size) != 0) { - LOGGER_ERROR("ftruncate() error on {}", dst); - rv = errno; - goto cleanup_on_error; - } + if(::ftruncate(out_fd, size) != 0) { + LOGGER_ERROR("ftruncate() error on {}", dst); + rv = errno; + goto cleanup_on_error; } - LOGGER_ERROR("fallocate() error"); - rv = errno; - goto cleanup_on_error; + +#ifdef HAVE_FALLOCATE } +#endif // HAVE_FALLOCATE dst_addr = ::mmap(NULL, size, PROT_WRITE, MAP_SHARED, out_fd, 0); diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 6111600..28782b5 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -52,17 +52,23 @@ create_file(const bfs::path& filename, } // preallocate output file +#ifdef HAVE_FALLOCATE if(::fallocate(out_fd, 0, 0, size) == -1) { + if(errno != EOPNOTSUPP) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); + } +#endif // HAVE_FALLOCATE + // filesystem doesn't support fallocate(), fallback to truncate() - if(errno == EOPNOTSUPP) { - if(::ftruncate(out_fd, size) != 0) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } + if(::ftruncate(out_fd, size) != 0) { + ec = std::make_error_code(static_cast(errno)); + return std::make_tuple(ec, nullptr); } - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + +#ifdef HAVE_FALLOCATE } +#endif // HAVE_FALLOCATE retry_close: if(close(out_fd) == -1) { -- GitLab From 03ce2d776c238621051c5c68f94084bf868dfad1 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 15:07:47 +0100 Subject: [PATCH 24/51] Return NORNS_ENOTSUPPORTED if input and output are remote --- src/urd.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/urd.cpp b/src/urd.cpp index b9cda35..61300e7 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -190,7 +190,7 @@ urd_error urd::validate_iotask_args(iotask_type type, return urd_error::bad_args; } - if(src_rinfo->is_remote() && dst_rinfo()->is_remote()) { + if(src_rinfo->is_remote() && dst_rinfo->is_remote()) { return urd_error::not_supported; } -- GitLab From 97cf0487902959e7275001adc35eb91383c1302a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 15:10:49 +0100 Subject: [PATCH 25/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 316cf88..8375d04 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -227,7 +227,14 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remote_copy_local_posix_files]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" after_script: - pwd -- GitLab From 9f81a59f13916e4c82b4cd78e6d046b485172613 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 16:46:04 +0100 Subject: [PATCH 26/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8375d04..fcb38a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -207,7 +207,7 @@ test:ubuntu:latest: - make -j$(nproc) - cd tests - make -j$(nproc) core - - ./core -as + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - make -j$(nproc) api # - NORNS_DEBUG_OUTPUT_TO_STDERR=1 NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" -- GitLab From d87ace3cb01fcbc894a35402e766f0bad952fba4 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 17:10:55 +0100 Subject: [PATCH 27/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fcb38a6..716baac 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,7 +42,6 @@ before_script: - pushd . && git clone https://github.com/mercury-hpc/mercury.git && cd mercury && - git reset --hard 3d8ed01eeff6b504862702048d7577457c71227c && mkdir build && cd build && cmake -- GitLab From 5af171e7ba38ed87aa1538a010754544b6fae75b Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 27 Feb 2019 17:32:31 +0100 Subject: [PATCH 28/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 716baac..b5d1ce5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -174,7 +174,6 @@ test:ubuntu:latest: - pushd . && git clone https://github.com/mercury-hpc/mercury.git && cd mercury && - git reset --hard 3d8ed01eeff6b504862702048d7577457c71227c && mkdir build && cd build && cmake -- GitLab From 95c268b36d326f8d409d5cfae939f87e47c70915 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 1 Mar 2019 12:35:19 +0100 Subject: [PATCH 29/51] Improve handling of temporary files --- src/Makefile.am | 5 +- .../local-path-to-remote-resource.cpp | 350 ++++++++---------- .../remote-resource-to-local-path.cpp | 322 ++++++++-------- src/utils.hpp | 1 + src/utils/file-handle.hpp | 87 +++++ src/utils/temporary-file.cpp | 197 ++++++++++ src/utils/temporary-file.hpp | 88 +++++ tests/api-send-command.cpp | 2 + 8 files changed, 698 insertions(+), 354 deletions(-) create mode 100644 src/utils/file-handle.hpp create mode 100644 src/utils/temporary-file.cpp create mode 100644 src/utils/temporary-file.hpp diff --git a/src/Makefile.am b/src/Makefile.am index 8aad795..55bed12 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -189,8 +189,11 @@ liburd_aux_la_SOURCES = \ urd.hpp \ utils.cpp \ utils.hpp \ + utils/file-handle.hpp \ utils/tar-archive.cpp \ - utils/tar-archive.hpp + utils/tar-archive.hpp \ + utils/temporary-file.hpp \ + utils/temporary-file.cpp nodist_liburd_aux_la_SOURCES = \ config/defaults.cpp \ diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index abf5ffe..6d49887 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -44,61 +44,82 @@ namespace { -std::tuple> -create_file(const bfs::path& filename, - std::size_t size) { +struct archive_entry { + bool m_is_directory; + bfs::path m_realpath; + bfs::path m_archive_path; +}; - std::error_code ec; +bfs::path +pack_archive(const std::string& name_pattern, + const bfs::path& parent_path, + const std::vector& entries, + std::error_code& ec) { + + using norns::utils::tar; + + bfs::path ar_path = parent_path / bfs::unique_path(name_pattern); - int out_fd = - ::open(filename.c_str(), O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); + tar ar(ar_path, tar::create, ec); - if(out_fd == -1) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + if(ec) { + LOGGER_ERROR("Failed to create archive: {}", ec.message()); + return {}; } - // preallocate output file -#ifdef HAVE_FALLOCATE - if(::fallocate(out_fd, 0, 0, size) == -1) { - if(errno != EOPNOTSUPP) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } -#endif // HAVE_FALLOCATE + LOGGER_INFO("Archive created in {}", ar.path()); - // filesystem doesn't support fallocate(), fallback to truncate() - if(::ftruncate(out_fd, size) != 0) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } + for(auto&& e : entries) { + e.m_is_directory ? + ar.add_directory(e.m_realpath, e.m_archive_path, ec) : + ar.add_file(e.m_realpath, e.m_archive_path, ec); -#ifdef HAVE_FALLOCATE + if(ec) { + LOGGER_ERROR("Failed to add entry to archive: {}", ec.message()); + return {}; + } } -#endif // HAVE_FALLOCATE -retry_close: - if(close(out_fd) == -1) { - if(errno == EINTR) { - goto retry_close; - } + return ar.path(); +} + +std::error_code +unpack_archive(const bfs::path& archive_path, + const bfs::path& parent_path) { + + using norns::utils::tar; + std::error_code ec; + + boost::system::error_code bec; + tar ar(archive_path, tar::open, ec); - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + if(ec) { + LOGGER_ERROR("Failed to open archive {}: {}", + archive_path, ec.message()); + return ec; } - auto output_data = - std::make_shared( - filename.string(), - hermes::access_mode::write_only, - &ec); + ar.extract(parent_path, ec); if(ec) { - LOGGER_ERROR("Failed mapping output data: {}", ec.value()); - return std::make_tuple(ec, output_data); + LOGGER_ERROR("Failed to extract archive {} into {}: {}", + ar.path(), parent_path, ec.message()); + return ec; } - return std::make_tuple(ec, output_data); + LOGGER_DEBUG("Archive {} extracted into {}, removing archive", + ar.path(), parent_path); + + bfs::remove(ar.path(), bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + ar.path(), bec.message()); + ec.assign(bec.value(), std::generic_category()); + return ec; + } + + return ec; } } // anonymous namespace @@ -134,43 +155,44 @@ local_path_to_remote_resource_transferor::transfer( (void) auth; + std::error_code ec; const auto& d_src = reinterpret_cast(*src); const auto& d_dst = reinterpret_cast(*dst); + auto tempfile = std::make_shared(); - std::string input_path = d_src.canonical_path().string(); + bfs::path input_path = + !d_src.is_collection() ? + d_src.canonical_path() : + [&]() -> bfs::path { - if(d_src.is_collection()) { - LOGGER_DEBUG("[{}] Creating archive for local directory", - task_info->id()); + LOGGER_DEBUG("[{}] Creating temporary archive from local directory", + task_info->id()); - std::error_code ec; + const bfs::path ar_path = + ::pack_archive("norns-archive-%%%%-%%%%-%%%%.tar", + "/tmp", + {{true, d_src.canonical_path(), d_dst.name()}}, + ec); - bfs::path ar_path = - "/tmp" / bfs::unique_path("norns-archive-%%%%-%%%%-%%%%.tar"); - - tar ar(ar_path, tar::create, ec); + if(ec) { + LOGGER_ERROR("Failed to create temporary archive: {}", + ec.message()); + return {}; + } - if(ec) { - LOGGER_ERROR("Failed to create archive: {}", ec.message()); - return ec; - } + tempfile->manage(ar_path, ec); - LOGGER_INFO("Archive created in {}", ar.path()); + if(ec) { + LOGGER_ERROR("Failed to create temporary archive: {}", + ec.message()); + return {}; + } - ar.add_directory(d_src.canonical_path(), - d_dst.name(), - ec); + return tempfile->path(); + }(); // <<== XXX (IILE) - if(ec) { - LOGGER_ERROR("Failed to add directory to archive: {}", - ec.message()); - return ec; - } - - input_path = ar.path().string(); - } LOGGER_DEBUG("[{}] start_transfer: {} -> {}", task_info->id(), d_src.canonical_path(), d_dst.to_string()); @@ -178,10 +200,9 @@ local_path_to_remote_resource_transferor::transfer( hermes::endpoint endp = m_network_endpoint->lookup(d_dst.address()); try { - std::error_code ec; - hermes::mapped_buffer input_data(input_path, - hermes::access_mode::read_only, - &ec); + hermes::mapped_buffer input_buffer(input_path.string(), + hermes::access_mode::read_only, + &ec); if(ec) { LOGGER_ERROR("Failed mapping input data: {}", ec.value()); @@ -189,7 +210,7 @@ local_path_to_remote_resource_transferor::transfer( } std::vector bufvec{ - hermes::mutable_buffer{input_data.data(), input_data.size()} + hermes::mutable_buffer{input_buffer.data(), input_buffer.size()} }; auto buffers = @@ -205,75 +226,39 @@ local_path_to_remote_resource_transferor::transfer( d_dst.name(), buffers); - LOGGER_DEBUG("rpc::in::args{{"); - LOGGER_DEBUG(" address: \"{}\",", m_network_endpoint->self_address()); - LOGGER_DEBUG(" in_nsid: \"{}\",", d_src.parent()->nsid()); - LOGGER_DEBUG(" out_nsid: \"{}\",", d_dst.parent()->nsid()); - LOGGER_DEBUG(" btype: {} ({}),", - static_cast(backend_type::posix_filesystem), - utils::to_string(backend_type::posix_filesystem)); - LOGGER_DEBUG(" rtype: {} ({}),", - static_cast(data::resource_type::local_posix_path), - utils::to_string(data::resource_type::local_posix_path)); - LOGGER_DEBUG(" rname: \"{}\",", d_dst.name()); - LOGGER_DEBUG(" buffers: {{...}}"); - LOGGER_DEBUG("};"); - LOGGER_FLUSH(); - auto resp = - m_network_endpoint->post(endp, args).get(); + m_network_endpoint->post( + endp, + rpc::remote_transfer::input{ + m_network_endpoint->self_address(), + d_src.parent()->nsid(), + d_dst.parent()->nsid(), + static_cast(backend_type::posix_filesystem), + static_cast( + data::resource_type::local_posix_path), + d_src.is_collection(), + d_dst.name(), + buffers + }).get(); if(static_cast(resp.at(0).status()) == task_status::finished_with_error) { - - // XXX it would probably be worth it to define - // a temporary_file class that cleans itself when - // destroyed to ease removal and avoid code duplication - // below - if(src->is_collection()) { - boost::system::error_code bec; - - bfs::remove(input_path, bec); - - if(bec) { - LOGGER_ERROR("Failed to remove archive {}: {}", - input_path, ec.message()); - //TODO - return std::make_error_code( - static_cast(bec.value())); - } - } - // XXX error interface should be improved return std::make_error_code( static_cast(resp.at(0).sys_errnum())); } - task_info->record_transfer(input_data.size(), + task_info->record_transfer(input_buffer.size(), resp.at(0).elapsed_time()); LOGGER_DEBUG("Remote request completed with output " "{{status: {}, task_error: {}, sys_errnum: {}}} " "({} bytes, {} usecs)", resp.at(0).status(), resp.at(0).task_error(), - resp.at(0).sys_errnum(), input_data.size(), + resp.at(0).sys_errnum(), input_buffer.size(), resp.at(0).elapsed_time()); - if(src->is_collection()) { - boost::system::error_code bec; - - bfs::remove(input_path, bec); - - if(bec) { - LOGGER_ERROR("Failed to remove archive {}: {}", - input_path, ec.message()); - //TODO - return std::make_error_code( - static_cast(bec.value())); - } - } - - return std::make_error_code(static_cast(0)); + return ec; } catch(const std::exception& ex) { LOGGER_ERROR(ex.what()); @@ -290,26 +275,22 @@ local_path_to_remote_resource_transferor::accept_transfer( const std::shared_ptr& dst) const { (void) auth; - (void) src; - (void) dst; - - using utils::tar; + std::error_code ec; const auto& d_src = reinterpret_cast(*src); const auto& d_dst = reinterpret_cast(*dst); + // retrieve task context const auto ctx = boost::any_cast< std::shared_ptr< hermes::request>>(task_info->context()); auto req = std::move(*ctx); - LOGGER_DEBUG("[{}] accept_transfer: {} -> {}", task_info->id(), + LOGGER_DEBUG("[{}] accept_push: {} -> {}", task_info->id(), d_src.to_string(), d_dst.canonical_path()); - LOGGER_WARN("Invoking rpc::remote_transfer({}) on {}", d_dst.name(), "xxx"); - hermes::exposed_memory remote_buffers = d_src.buffers(); LOGGER_DEBUG("remote_buffers{{count={}, total_size={}}}", @@ -320,111 +301,100 @@ local_path_to_remote_resource_transferor::accept_transfer( bool is_collection = d_src.is_collection(); - bfs::path output_path = - is_collection ? "/tmp/test.tar" : d_dst.canonical_path(); + auto tempfile = + std::make_shared( + /* output_path */ + std::string{is_collection ? + "norns-archive-%%%%-%%%%-%%%%.tar" : + d_dst.name()}, + /* parent_path */ + bfs::path{is_collection ? + "/tmp" : + d_dst.parent()->mount()}, + remote_buffers.size(), + ec); - LOGGER_DEBUG("creating local resource: {}", output_path); + if(ec) { + LOGGER_ERROR("Failed to create temporary file: {}", ec.message()); + *ctx = std::move(req); // restore ctx + return ec; + } - std::error_code ec; - std::shared_ptr output_data; + LOGGER_DEBUG("created local resource: {}", tempfile->path()); - std::tie(ec, output_data) = - ::create_file(output_path, remote_buffers.size()); + auto output_buffer = + std::make_shared( + tempfile->path().string(), + hermes::access_mode::write_only, + &ec); if(ec) { - *ctx = std::move(req); + LOGGER_ERROR("Failed mmapping output buffer: {}", ec.value()); + *ctx = std::move(req); // restore ctx return ec; } // let's prepare some local buffers - std::vector bufseq{ - hermes::mutable_buffer{output_data->data(), output_data->size()} + hermes::mutable_buffer{output_buffer->data(), output_buffer->size()} }; hermes::exposed_memory local_buffers = m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); - LOGGER_DEBUG("pulling remote data into {}", output_path); + LOGGER_DEBUG("pulling remote data into {}", tempfile->path()); auto start = std::chrono::steady_clock::now(); - // N.B. IMPORTANT: we NEED to capture output_data by value here so that + // N.B. IMPORTANT: we NEED to capture output_buffer by value here so that // the mapped_buffer doesn't get released before completion_callback() // is called. const auto completion_callback = - [this, is_collection, output_path, d_dst, output_data, start]( + [this, is_collection, tempfile, d_dst, output_buffer, start]( hermes::request&& req) { uint32_t usecs = std::chrono::duration_cast( std::chrono::steady_clock::now() - start).count(); - // default response - rpc::remote_transfer::output out( + //TODO: hermes offers no way to check for an error yet + LOGGER_DEBUG("Pull completed ({} usecs)", usecs); + + // default response (success) + rpc::remote_transfer::output out = { static_cast(task_status::finished), static_cast(urd_error::success), 0, - usecs); - - //TODO: hermes offers no way to check for an error yet - LOGGER_DEBUG("Pull completed ({} usecs)", usecs); + 0}; if(is_collection) { - std::error_code ec; - boost::system::error_code bec; - tar ar(output_path, tar::open, ec); + std::error_code ec = + ::unpack_archive(tempfile->path(), d_dst.parent()->mount()); if(ec) { - LOGGER_ERROR("Failed to open archive {}: {}", - output_path, ec.message()); - - out = std::move(rpc::remote_transfer::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(ec.value()), - 0 - }); - goto respond; - } + out = rpc::remote_transfer::output{ + static_cast( + task_status::finished_with_error), + static_cast(urd_error::system_error), + static_cast(ec.value()), + 0}; - ar.extract(d_dst.parent()->mount(), ec); - - if(ec) { - LOGGER_ERROR("Failed to extract archive into {}: {}", - ar.path(), d_dst.parent()->mount(), - ec.message()); - out = std::move(rpc::remote_transfer::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(ec.value()), - 0 - }); goto respond; } LOGGER_DEBUG("Archive {} extracted into {}", - ar.path(), d_dst.parent()->mount()); - - bfs::remove(ar.path(), bec); - - if(bec) { - LOGGER_ERROR("Failed to remove archive {}: {}", - ar.path(), ec.message()); - out = std::move(rpc::remote_transfer::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(bec.value()), - 0 - }); - } + tempfile->path(), d_dst.parent()->mount()); + + goto respond; } + // prevent output file from being removed by tempfile's destructor + (void) tempfile->release(); + respond: if(req.requires_response()) { m_network_endpoint->respond( - std::move(req), - out); + std::move(req), out); } }; @@ -433,7 +403,7 @@ respond: std::move(req), completion_callback); - return std::make_error_code(static_cast(0)); + return ec; } std::string diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 28782b5..a8750e7 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -37,61 +37,82 @@ namespace { -std::tuple> -create_file(const bfs::path& filename, - std::size_t size) { +struct archive_entry { + bool m_is_directory; + bfs::path m_realpath; + bfs::path m_archive_path; +}; - std::error_code ec; +bfs::path +pack_archive(const std::string& name_pattern, + const bfs::path& parent_path, + const std::vector& entries, + std::error_code& ec) { + + using norns::utils::tar; + + bfs::path ar_path = parent_path / bfs::unique_path(name_pattern); - int out_fd = - ::open(filename.c_str(), O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR); + tar ar(ar_path, tar::create, ec); - if(out_fd == -1) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + if(ec) { + LOGGER_ERROR("Failed to create archive: {}", ec.message()); + return {}; } - // preallocate output file -#ifdef HAVE_FALLOCATE - if(::fallocate(out_fd, 0, 0, size) == -1) { - if(errno != EOPNOTSUPP) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } -#endif // HAVE_FALLOCATE + LOGGER_INFO("Archive created in {}", ar.path()); - // filesystem doesn't support fallocate(), fallback to truncate() - if(::ftruncate(out_fd, size) != 0) { - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); - } + for(auto&& e : entries) { + e.m_is_directory ? + ar.add_directory(e.m_realpath, e.m_archive_path, ec) : + ar.add_file(e.m_realpath, e.m_archive_path, ec); -#ifdef HAVE_FALLOCATE + if(ec) { + LOGGER_ERROR("Failed to add entry to archive: {}", ec.message()); + return {}; + } } -#endif // HAVE_FALLOCATE -retry_close: - if(close(out_fd) == -1) { - if(errno == EINTR) { - goto retry_close; - } + return ar.path(); +} + +std::error_code +unpack_archive(const bfs::path& archive_path, + const bfs::path& parent_path) { + + using norns::utils::tar; + std::error_code ec; - ec = std::make_error_code(static_cast(errno)); - return std::make_tuple(ec, nullptr); + boost::system::error_code bec; + tar ar(archive_path, tar::open, ec); + + if(ec) { + LOGGER_ERROR("Failed to open archive {}: {}", + archive_path, ec.message()); + return ec; } - auto output_data = - std::make_shared( - filename.string(), - hermes::access_mode::write_only, - &ec); + ar.extract(parent_path, ec); if(ec) { - LOGGER_ERROR("Failed mapping output data: {}", ec.value()); - return std::make_tuple(ec, output_data); + LOGGER_ERROR("Failed to extract archive {} into {}: {}", + ar.path(), parent_path, ec.message()); + return ec; } - return std::make_tuple(ec, output_data); + LOGGER_DEBUG("Archive {} extracted into {}, removing archive", + ar.path(), parent_path); + + bfs::remove(ar.path(), bec); + + if(bec) { + LOGGER_ERROR("Failed to remove archive {}: {}", + ar.path(), bec.message()); + ec.assign(bec.value(), std::generic_category()); + return ec; + } + + return ec; } } // anonymous namespace @@ -124,10 +145,8 @@ remote_resource_to_local_path_transferor::transfer( const std::shared_ptr& dst) const { (void) auth; - (void) src; - (void) dst; - (void) task_info; + std::error_code ec; const auto& d_src = reinterpret_cast(*src); const auto& d_dst = @@ -138,14 +157,15 @@ remote_resource_to_local_path_transferor::transfer( hermes::endpoint endp = m_network_endpoint->lookup(d_src.address()); - rpc::resource_stat::input args( - m_network_endpoint->self_address(), - d_src.parent()->nsid(), - static_cast(data::resource_type::local_posix_path), - d_src.name()); - auto resp = - m_network_endpoint->post(endp, args).get(); + m_network_endpoint->post( + endp, + rpc::resource_stat::input{ + m_network_endpoint->self_address(), + d_src.parent()->nsid(), + static_cast(data::resource_type::local_posix_path), + d_src.name() + }).get(); LOGGER_DEBUG("remote_stat returned [task_error: {}, sys_errnum: {}, " "is_collection: {}, packed_size: {}]", @@ -160,16 +180,35 @@ remote_resource_to_local_path_transferor::transfer( static_cast(resp.at(0).sys_errnum())); } - bfs::path output_path = - resp.at(0).is_collection() ? "/tmp/test.tar" : d_dst.canonical_path(); + utils::temporary_file tempfile( + /* output_path */ + {resp.at(0).is_collection() ? + "norns-archive-%%%%-%%%%-%%%%.tar" : + d_dst.name()}, + /* parent_dir */ + {resp.at(0).is_collection() ? + "/tmp" : + d_dst.parent()->mount()}, + resp.at(0).packed_size(), + ec); - LOGGER_DEBUG("creating local resource: {}", output_path); + if(ec) { + LOGGER_ERROR("Failed to create temporary file: {}", ec.message()); + return ec; + } - std::error_code ec; - std::shared_ptr output_buffer; + LOGGER_DEBUG("created local resource: {}", tempfile.path()); + + auto output_buffer = + std::make_shared( + tempfile.path().string(), + hermes::access_mode::write_only, + &ec); - std::tie(ec, output_buffer) = - ::create_file(output_path, resp.at(0).packed_size()); + if(ec) { + LOGGER_ERROR("Failed mmapping output buffer: {}", ec.value()); + return ec; + } // let's prepare some local buffers std::vector bufseq{ @@ -179,22 +218,24 @@ remote_resource_to_local_path_transferor::transfer( hermes::exposed_memory local_buffers = m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); - rpc::pull_resource::input args2( - d_src.parent()->nsid(), - d_src.name(), - // XXX this resource_type should not be needed, but we cannot - // XXX (easily) find it out right now in the server, for now we - // XXX propagate it, but we should implement a lookup()/stat() - // XXX function in backends to retrieve this information from the - // XXX resource id - static_cast(data::resource_type::local_posix_path), - m_network_endpoint->self_address(), - d_dst.parent()->nsid(), - d_dst.name(), - local_buffers); - auto resp2 = - m_network_endpoint->post(endp, args2).get(); + m_network_endpoint->post( + endp, + rpc::pull_resource::input{ + d_src.parent()->nsid(), + d_src.name(), + // XXX this resource_type should not be needed, but we + // XXX cannot (easily) find it out right now in the server, + // XXX for now we propagate it, but we should implement + // XXX a lookup()/stat() function in backends to retrieve + // XXX this information from the resource id + static_cast( + data::resource_type::local_posix_path), + m_network_endpoint->self_address(), + d_dst.parent()->nsid(), + d_dst.name(), + local_buffers + }).get(); LOGGER_DEBUG("Remote push request completed with output " "{{status: {}, task_error: {}, sys_errnum: {}}} " @@ -211,40 +252,13 @@ remote_resource_to_local_path_transferor::transfer( } if(resp.at(0).is_collection()) { - using utils::tar; - - std::error_code ec; - boost::system::error_code bec; - tar ar(output_path, tar::open, ec); - - if(ec) { - LOGGER_ERROR("Failed to open archive {}: {}", - output_path, logger::errno_message(ec.value())); - - return std::make_error_code(static_cast(ec.value())); - } - - ar.extract(d_dst.parent()->mount(), ec); - - if(ec) { - LOGGER_ERROR("Failed to extact archive into {}: {}", - ar.path(), d_dst.parent()->mount(), ec.message()); - return std::make_error_code(static_cast(ec.value())); - } - - LOGGER_DEBUG("Archive {} extracted into {}", - ar.path(), d_dst.parent()->mount()); - - bfs::remove(ar.path(), bec); - - if(bec) { - LOGGER_ERROR("Failed to remove archive {}: {}", - ar.path(), ec.message()); - return std::make_error_code(static_cast(ec.value())); - } + return ::unpack_archive(tempfile.path(), d_dst.parent()->mount()); } - return std::make_error_code(static_cast(0)); + // prevent output file from being removed by tempfile's destructor + (void) tempfile.release(); + + return ec; } @@ -256,50 +270,44 @@ remote_resource_to_local_path_transferor::accept_transfer( const std::shared_ptr& dst) const { (void) auth; - (void) task_info; - (void) src; - (void) dst; + std::error_code ec; const auto& d_src = reinterpret_cast(*src); const auto& d_dst = reinterpret_cast(*dst); - std::error_code ec; - - std::string input_path = d_src.canonical_path().string(); - bool is_collection = d_src.is_collection(); - - if(is_collection) { - using utils::tar; - - LOGGER_DEBUG("[{}] Creating archive from local directory", - task_info->id()); - - bfs::path ar_path = - "/tmp" / bfs::unique_path("norns-archive-%%%%-%%%%-%%%%.tar"); - - tar ar(ar_path, tar::create, ec); - - if(ec) { - LOGGER_ERROR("Failed to create archive: {}", - logger::errno_message(ec.value())); - return ec; - } - - LOGGER_INFO("Archive created in {}", ar.path()); + auto tempfile = std::make_shared(); + + bfs::path input_path = + !d_src.is_collection() ? + d_src.canonical_path() : + [&]() -> bfs::path { + + LOGGER_DEBUG("[{}] Creating temporary archive from local directory", + task_info->id()); + + const bfs::path ar_path = + ::pack_archive("norns-archive-%%%%-%%%%-%%%%.tar", + "/tmp", + {{true, d_src.canonical_path(), d_dst.name()}}, + ec); + + if(ec) { + LOGGER_ERROR("Failed to create temporary archive: {}", + ec.message()); + return {}; + } - ar.add_directory(d_src.canonical_path(), - d_dst.name(), - ec); + tempfile->manage(ar_path, ec); - if(ec) { - LOGGER_ERROR("Failed to add directory to archive: {}", - logger::errno_message(ec.value())); - return ec; - } + if(ec) { + LOGGER_ERROR("Failed to create temporary archive: {}", + ec.message()); + return {}; + } - input_path = ar.path().string(); - } + return tempfile->path(); + }(); // <<== XXX (IILE) // retrieve task context const auto ctx = boost::any_cast< @@ -312,12 +320,14 @@ remote_resource_to_local_path_transferor::accept_transfer( // create local buffers from local input data auto input_buffer = - std::make_shared(input_path, - hermes::access_mode::read_only, - &ec); + std::make_shared( + input_path.string(), + hermes::access_mode::read_only, + &ec); if(ec) { LOGGER_ERROR("Failed mapping input data: {}", ec.message()); + *ctx = std::move(req); // restore ctx return ec; } @@ -335,11 +345,15 @@ remote_resource_to_local_path_transferor::accept_transfer( remote_buffers.count(), remote_buffers.size()); - // N.B. IMPORTANT: we NEED to capture input_buffer by value here so that + // N.B. IMPORTANT: we NEED to capture 'input_buffer' by value here so that // the mapped_buffer doesn't get released before completion_callback() // is called. + // We also capture 'tempfile' by value so that it gets automatically + // released (and the associated file erased) when the callback finishes + // FIXME: with C++14 we could simply std::move both into the capture rather + // than using shared_ptrs :/ const auto completion_callback = - [this, is_collection, input_path, input_buffer]( + [this, tempfile, input_buffer]( hermes::request&& req) { // default response @@ -352,24 +366,6 @@ remote_resource_to_local_path_transferor::accept_transfer( //TODO: hermes offers no way to check for an error yet LOGGER_DEBUG("Push completed"); - if(is_collection) { - boost::system::error_code bec; - bfs::remove(input_path, bec); - - if(bec) { - LOGGER_ERROR("Failed to remove archive {}: {}", - input_path, bec.message()); - out = std::move(rpc::pull_resource::output{ - static_cast(task_status::finished_with_error), - static_cast(urd_error::system_error), - static_cast(bec.value()), - 0 - }); - goto respond; - } - } - -respond: if(req.requires_response()) { m_network_endpoint->respond( std::move(req), @@ -382,7 +378,7 @@ respond: std::move(req), completion_callback); - return std::make_error_code(static_cast(0)); + return ec; } std::string diff --git a/src/utils.hpp b/src/utils.hpp index c0f56d7..b1327b5 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -37,6 +37,7 @@ #include "common.hpp" #include "utils/tar-archive.hpp" +#include "utils/temporary-file.hpp" namespace norns { namespace utils { diff --git a/src/utils/file-handle.hpp b/src/utils/file-handle.hpp new file mode 100644 index 0000000..3b885d9 --- /dev/null +++ b/src/utils/file-handle.hpp @@ -0,0 +1,87 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "logger.hpp" + +namespace norns { +namespace utils { + +struct file_handle { + + constexpr static const int init_value{-1}; + + file_handle() = default; + + explicit file_handle(int fd) noexcept : + m_fd(fd) { } + + file_handle(file_handle&& rhs) = default; + file_handle(const file_handle& other) = delete; + file_handle& operator=(file_handle&& rhs) = default; + file_handle& operator=(const file_handle& other) = delete; + + explicit operator bool() const noexcept { + return valid(); + } + + bool operator!() const noexcept { + return !valid(); + } + + bool + valid() const noexcept { + return m_fd != init_value; + } + + int + native() const noexcept { + return m_fd; + } + + int + release() noexcept { + int ret = m_fd; + m_fd = init_value; + return ret; + } + + ~file_handle() { + if(m_fd != init_value) { + if(::close(m_fd) == -1) { + LOGGER_ERROR("Failed to close file descriptor: {}", + logger::errno_message(errno)); + } + } + } + + int m_fd; +}; + + + +} // namespace utils +} // namespace norns diff --git a/src/utils/temporary-file.cpp b/src/utils/temporary-file.cpp new file mode 100644 index 0000000..35e9c59 --- /dev/null +++ b/src/utils/temporary-file.cpp @@ -0,0 +1,197 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "config.h" + +#include +#include + +#include "temporary-file.hpp" +#include "file-handle.hpp" +#include "logger.hpp" + +namespace { + +using norns::utils::file_handle; + +void +reallocate(const file_handle& fh, + std::size_t size, + std::error_code& ec) noexcept { + + if(!fh) { + ec.assign(EINVAL, std::generic_category()); + return; + } + +#ifdef HAVE_FALLOCATE + if(::fallocate(fh.native(), 0, 0, size) == -1) { + if(errno != EOPNOTSUPP) { + ec.assign(errno, std::generic_category()); + return; + } +#endif // HAVE_FALLOCATE + + // filesystem doesn't support fallocate(), + // fallback to truncate() + if(::ftruncate(fh.native(), size) != 0) { + ec.assign(errno, std::generic_category()); + return; + } + +#ifdef HAVE_FALLOCATE + } +#endif // HAVE_FALLOCATE +} + +} // anonymous namespace + +namespace norns { +namespace utils { + +temporary_file::temporary_file() noexcept { } + +temporary_file::temporary_file(const std::string& pattern, + const bfs::path& parent_dir, + std::error_code& ec) noexcept : + temporary_file(pattern, parent_dir, 0, ec) {} + +temporary_file::temporary_file(const std::string& pattern, + const bfs::path& parent_dir, + std::size_t reserve_size, + std::error_code& ec) noexcept { + + boost::system::error_code bec; + + const auto filename = parent_dir / bfs::unique_path(pattern, bec); + + if(bec) { + // LOGGER_ERROR("Failed to create unique path from pattern: {}", + // bec.message()); + ec.assign(bec.value(), std::generic_category()); + return; + } + + if(!bfs::exists(parent_dir, bec)) { + // LOGGER_ERROR("parent_dir does not exist: {}", bec.message()); + ec.assign(ENOENT, std::generic_category()); + return; + } + + if(bec) { + // LOGGER_ERROR("parent_dir does not exist: {}", bec.message()); + ec.assign(bec.value(), std::generic_category()); + return; + } + + file_handle fh( + ::open(filename.c_str(), + O_CREAT | O_WRONLY | O_EXCL, + S_IRUSR | S_IWUSR)); + + if(!fh) { + ec.assign(errno, std::generic_category()); + // LOGGER_ERROR("Failed to create temporary file: {}", ec.message()); + return; + } + + if(reserve_size != 0) { + ::reallocate(fh, reserve_size, ec); + } + + m_filename = filename; +} + +temporary_file::temporary_file(const bfs::path& filename, + std::error_code& ec) noexcept { + this->manage(filename, ec); +} + +temporary_file::~temporary_file() { + + if(m_filename.empty()) { + return; + } + + LOGGER_DEBUG("{} Removing temporary file {}", + __PRETTY_FUNCTION__, m_filename); + + boost::system::error_code bec; + bfs::remove(m_filename, bec); + + if(bec) { + LOGGER_ERROR("Failed to remove temporary_file {}: {}", + m_filename, bec.message()); + } +} + +bfs::path +temporary_file::path() const noexcept { + return m_filename; +} + +void +temporary_file::reserve(std::size_t size, + std::error_code& ec) const noexcept { + + file_handle fh{::open(m_filename.c_str(), O_WRONLY)}; + + if(!fh) { + ec.assign(errno, std::generic_category()); + // LOGGER_ERROR("Failed to allocate space for file: {}", ec.message()); + return; + } + + ::reallocate(fh, size, ec); +} + +void +temporary_file::manage(const bfs::path& filename, + std::error_code& ec) noexcept { + + boost::system::error_code bec; + const auto existing_filename = bfs::canonical(filename, bec); + + if(bec) { + // LOGGER_ERROR("Failed to get canonical path from filename: {}", + // bec.message()); + ec.assign(bec.value(), std::generic_category()); + return; + } + + m_filename = existing_filename; +} + +bfs::path +temporary_file::release() noexcept { + const bfs::path ret{std::move(m_filename)}; + m_filename.clear(); + return ret; +} + +} // namespace utils +} // namespace norns diff --git a/src/utils/temporary-file.hpp b/src/utils/temporary-file.hpp new file mode 100644 index 0000000..64d734d --- /dev/null +++ b/src/utils/temporary-file.hpp @@ -0,0 +1,88 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef NORNS_UTILS_TEMP_FILE_HPP +#define NORNS_UTILS_TEMP_FILE_HPP + +#include +#include + +namespace bfs = boost::filesystem; + +namespace norns { +namespace utils { + +struct temporary_file { + + temporary_file() noexcept; + + // create an empty temporary file from pattern at parent_dir + temporary_file(const std::string& pattern, + const bfs::path& parent_dir, + std::error_code& ec) noexcept; + + // create a temporary file of size 'prealloc_size' from pattern at parent_dir + temporary_file(const std::string& pattern, + const bfs::path& parent_dir, + std::size_t prealloc_size, + std::error_code& ec) noexcept; + + // initialize temporary file from an already existing file + temporary_file(const bfs::path& filename, + std::error_code& ec) noexcept; + + temporary_file(const temporary_file& other) = delete; + + temporary_file(temporary_file&& rhs) = default; + + temporary_file& operator=(const temporary_file& other) = delete; + + temporary_file& operator=(temporary_file&& rhs) = default; + + ~temporary_file(); + + bfs::path + path() const noexcept; + + void + reserve(std::size_t size, + std::error_code& ec) const noexcept; + + void + manage(const bfs::path& filename, + std::error_code& ec) noexcept; + + bfs::path + release() noexcept; + + bfs::path m_filename; +}; + +} // namespace utils +} // namespace norns + +#endif // NORNS_UTILS_TEMP_FILE_HPP diff --git a/tests/api-send-command.cpp b/tests/api-send-command.cpp index c5b5646..ab5d518 100644 --- a/tests/api-send-command.cpp +++ b/tests/api-send-command.cpp @@ -153,6 +153,8 @@ SCENARIO("send control commands to urd", "[api::nornsctl_send_command]") { } } } + + env.notify_success(); } #ifndef USE_REAL_DAEMON -- GitLab From 3278a2fa5c8a5eb5ed7ec7bcb58842934e070f67 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 1 Mar 2019 15:15:48 +0100 Subject: [PATCH 30/51] Add bandwidth monitoring to remote pull task --- src/io/task-copy.hpp | 4 +++- src/io/task-move.hpp | 4 +++- src/io/task-remote-transfer.hpp | 9 ++++++++- .../transferors/local-path-to-remote-resource.cpp | 4 ++-- .../transferors/remote-resource-to-local-path.cpp | 13 +++++++++++-- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/io/task-copy.hpp b/src/io/task-copy.hpp index 6a81448..e0e7093 100644 --- a/src/io/task-copy.hpp +++ b/src/io/task-copy.hpp @@ -87,7 +87,9 @@ task::operator()() { return; } - LOGGER_WARN("[{}] I/O task completed successfully", tid); + LOGGER_WARN("[{}] I/O task completed successfully [{} MiB/s]", + tid, m_task_info->bandwidth()); + m_task_info->update_status(task_status::finished, urd_error::success, std::make_error_code(static_cast(ec.value()))); } diff --git a/src/io/task-move.hpp b/src/io/task-move.hpp index 8a64ac6..61c84c7 100644 --- a/src/io/task-move.hpp +++ b/src/io/task-move.hpp @@ -87,7 +87,9 @@ task::operator()() { return; } - LOGGER_WARN("[{}] I/O task completed successfully", tid); + LOGGER_WARN("[{}] I/O task completed successfully [{} MiB/s]", + tid, m_task_info->bandwidth()); + m_task_info->update_status(task_status::finished, urd_error::success, std::make_error_code(static_cast(ec.value()))); } diff --git a/src/io/task-remote-transfer.hpp b/src/io/task-remote-transfer.hpp index 321612a..53c5966 100644 --- a/src/io/task-remote-transfer.hpp +++ b/src/io/task-remote-transfer.hpp @@ -93,7 +93,14 @@ task::operator()() { return; } - LOGGER_WARN("[{}] I/O task completed successfully", tid); + if(m_task_info->is_remote()) { + LOGGER_WARN("[{}] I/O task completed successfully", tid); + } + else { + LOGGER_WARN("[{}] I/O task completed successfully [{} MiB/s]", + tid, m_task_info->bandwidth()); + } + m_task_info->update_status(task_status::finished, urd_error::success, std::make_error_code(static_cast(ec.value()))); } diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 6d49887..97395a1 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -251,7 +251,7 @@ local_path_to_remote_resource_transferor::transfer( task_info->record_transfer(input_buffer.size(), resp.at(0).elapsed_time()); - LOGGER_DEBUG("Remote request completed with output " + LOGGER_DEBUG("Remote pull request completed with output " "{{status: {}, task_error: {}, sys_errnum: {}}} " "({} bytes, {} usecs)", resp.at(0).status(), resp.at(0).task_error(), @@ -365,7 +365,7 @@ local_path_to_remote_resource_transferor::accept_transfer( static_cast(task_status::finished), static_cast(urd_error::success), 0, - 0}; + usecs}; if(is_collection) { std::error_code ec = diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index a8750e7..85c8b93 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -251,6 +251,9 @@ remote_resource_to_local_path_transferor::transfer( static_cast(resp2.at(0).sys_errnum())); } + task_info->record_transfer(output_buffer->size(), + resp2.at(0).elapsed_time()); + if(resp.at(0).is_collection()) { return ::unpack_archive(tempfile.path(), d_dst.parent()->mount()); } @@ -345,6 +348,8 @@ remote_resource_to_local_path_transferor::accept_transfer( remote_buffers.count(), remote_buffers.size()); + auto start = std::chrono::steady_clock::now(); + // N.B. IMPORTANT: we NEED to capture 'input_buffer' by value here so that // the mapped_buffer doesn't get released before completion_callback() // is called. @@ -353,15 +358,19 @@ remote_resource_to_local_path_transferor::accept_transfer( // FIXME: with C++14 we could simply std::move both into the capture rather // than using shared_ptrs :/ const auto completion_callback = - [this, tempfile, input_buffer]( + [this, tempfile, input_buffer, start]( hermes::request&& req) { + uint32_t usecs = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + // default response rpc::pull_resource::output out( static_cast(task_status::finished), static_cast(urd_error::success), 0, - 0); + usecs); //TODO: hermes offers no way to check for an error yet LOGGER_DEBUG("Push completed"); -- GitLab From 4f7fe2eaaeea253058769665a7391842a673155a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Fri, 1 Mar 2019 16:35:57 +0100 Subject: [PATCH 31/51] Some renames to clarify rpcs --- .../local-path-to-remote-resource.cpp | 35 +-- .../local-path-to-remote-resource.hpp | 8 - .../remote-resource-to-local-path.cpp | 6 +- .../remote-resource-to-local-path.hpp | 8 - src/rpcs.cpp | 4 +- src/rpcs.hpp | 283 +++++++++--------- src/urd.cpp | 51 ++-- src/urd.hpp | 8 +- 8 files changed, 175 insertions(+), 228 deletions(-) diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 97395a1..e8bf88f 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -213,32 +213,27 @@ local_path_to_remote_resource_transferor::transfer( hermes::mutable_buffer{input_buffer.data(), input_buffer.size()} }; - auto buffers = + auto local_buffers = m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); - rpc::remote_transfer::input args( - m_network_endpoint->self_address(), - d_src.parent()->nsid(), - d_dst.parent()->nsid(), - static_cast(backend_type::posix_filesystem), - static_cast(data::resource_type::local_posix_path), - d_src.is_collection(), - d_dst.name(), - buffers); - auto resp = - m_network_endpoint->post( + m_network_endpoint->post( endp, - rpc::remote_transfer::input{ + rpc::push_resource::input{ m_network_endpoint->self_address(), d_src.parent()->nsid(), d_dst.parent()->nsid(), - static_cast(backend_type::posix_filesystem), + // XXX this resource_type should not be needed, but we + // XXX cannot (easily) find it out right now in the server, + // XXX for now we propagate it, but we should implement + // XXX a lookup()/stat() function in backends to retrieve + // XXX this information locally from the resource id static_cast( data::resource_type::local_posix_path), d_src.is_collection(), + d_src.name(), d_dst.name(), - buffers + local_buffers }).get(); if(static_cast(resp.at(0).status()) == @@ -285,7 +280,7 @@ local_path_to_remote_resource_transferor::accept_transfer( // retrieve task context const auto ctx = boost::any_cast< std::shared_ptr< - hermes::request>>(task_info->context()); + hermes::request>>(task_info->context()); auto req = std::move(*ctx); LOGGER_DEBUG("[{}] accept_push: {} -> {}", task_info->id(), @@ -351,7 +346,7 @@ local_path_to_remote_resource_transferor::accept_transfer( // is called. const auto completion_callback = [this, is_collection, tempfile, d_dst, output_buffer, start]( - hermes::request&& req) { + hermes::request&& req) { uint32_t usecs = std::chrono::duration_cast( @@ -361,7 +356,7 @@ local_path_to_remote_resource_transferor::accept_transfer( LOGGER_DEBUG("Pull completed ({} usecs)", usecs); // default response (success) - rpc::remote_transfer::output out = { + rpc::push_resource::output out = { static_cast(task_status::finished), static_cast(urd_error::success), 0, @@ -372,7 +367,7 @@ local_path_to_remote_resource_transferor::accept_transfer( ::unpack_archive(tempfile->path(), d_dst.parent()->mount()); if(ec) { - out = rpc::remote_transfer::output{ + out = rpc::push_resource::output{ static_cast( task_status::finished_with_error), static_cast(urd_error::system_error), @@ -393,7 +388,7 @@ local_path_to_remote_resource_transferor::accept_transfer( respond: if(req.requires_response()) { - m_network_endpoint->respond( + m_network_endpoint->respond( std::move(req), out); } }; diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index f7d09c4..a4144cc 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -51,10 +51,6 @@ struct resource_info; struct resource; } -namespace rpc { -struct remote_transfer; -} - namespace io { struct local_path_to_remote_resource_transferor : public transferor { @@ -85,10 +81,6 @@ struct local_path_to_remote_resource_transferor : public transferor { to_string() const override final; private: - - void - do_accept(hermes::request&& req); - std::shared_ptr m_network_endpoint; }; diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 85c8b93..0142bf3 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -158,9 +158,9 @@ remote_resource_to_local_path_transferor::transfer( hermes::endpoint endp = m_network_endpoint->lookup(d_src.address()); auto resp = - m_network_endpoint->post( + m_network_endpoint->post( endp, - rpc::resource_stat::input{ + rpc::stat_resource::input{ m_network_endpoint->self_address(), d_src.parent()->nsid(), static_cast(data::resource_type::local_posix_path), @@ -228,7 +228,7 @@ remote_resource_to_local_path_transferor::transfer( // XXX cannot (easily) find it out right now in the server, // XXX for now we propagate it, but we should implement // XXX a lookup()/stat() function in backends to retrieve - // XXX this information from the resource id + // XXX this information locally from the resource id static_cast( data::resource_type::local_posix_path), m_network_endpoint->self_address(), diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index a83688f..1042594 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -51,10 +51,6 @@ struct resource_info; struct resource; } -namespace rpc { -struct remote_transfer; -} - namespace io { struct remote_resource_to_local_path_transferor : public transferor { @@ -85,10 +81,6 @@ struct remote_resource_to_local_path_transferor : public transferor { to_string() const override final; private: - - void - do_accept(hermes::request&& req); - std::shared_ptr m_network_endpoint; }; diff --git a/src/rpcs.cpp b/src/rpcs.cpp index 05df83d..bfe2883 100644 --- a/src/rpcs.cpp +++ b/src/rpcs.cpp @@ -9,9 +9,9 @@ namespace hermes { namespace detail { // void register_user_request_types() { - (void) registered_requests().add(); + (void) registered_requests().add(); (void) registered_requests().add(); - (void) registered_requests().add(); + (void) registered_requests().add(); } }} // namespace hermes::detail diff --git a/src/rpcs.hpp b/src/rpcs.hpp index f63b798..d487c74 100644 --- a/src/rpcs.hpp +++ b/src/rpcs.hpp @@ -26,24 +26,24 @@ hg_return_t post_to_mercury(ExecutionContext* ctx); }} // namespace hermes::detail //============================================================================== -// definitions for norns::rpc::remote_transfer +// definitions for norns::rpc::push_resource namespace hermes { namespace detail { // Generate Mercury types and serialization functions (field names match -// those defined by remote_transfer::input and remote_transfer::output). These +// those defined by push_resource::input and push_resource::output). These // definitions are internal and should not be used directly. Classes -// remote_transfer::input and remote_transfer::output are provided for public use. -MERCURY_GEN_PROC(remote_transfer_in_t, - ((hg_const_string_t) (address)) +// push_resource::input and push_resource::output are provided for public use. +MERCURY_GEN_PROC(push_resource_in_t, + ((hg_const_string_t) (in_address)) ((hg_const_string_t) (in_nsid)) + ((hg_const_string_t) (in_resource_name)) + ((hg_bool_t) (in_is_collection)) + ((hg_bulk_t) (in_buffers)) ((hg_const_string_t) (out_nsid)) - ((uint32_t) (backend_type)) - ((uint32_t) (resource_type)) - ((hg_bool_t) (is_collection)) - ((hg_const_string_t) (resource_name)) - ((hg_bulk_t) (buffers))) + ((uint32_t) (out_resource_type)) + ((hg_const_string_t) (out_resource_name))) -MERCURY_GEN_PROC(remote_transfer_out_t, +MERCURY_GEN_PROC(push_resource_out_t, ((uint32_t) (status)) ((uint32_t) (task_error)) ((uint32_t) (sys_errnum)) @@ -64,13 +64,13 @@ MERCURY_GEN_PROC(pull_resource_out_t, ((uint32_t) (sys_errnum)) ((uint32_t) (elapsed_time))) -MERCURY_GEN_PROC(resource_stat_in_t, +MERCURY_GEN_PROC(stat_resource_in_t, ((hg_const_string_t) (address)) ((hg_const_string_t) (nsid)) ((uint32_t) (resource_type)) ((hg_const_string_t) (resource_name))) -MERCURY_GEN_PROC(resource_stat_out_t, +MERCURY_GEN_PROC(stat_resource_out_t, ((uint32_t) (task_error)) ((uint32_t) (sys_errnum)) ((hg_bool_t) (is_collection)) @@ -82,19 +82,19 @@ MERCURY_GEN_PROC(resource_stat_out_t, namespace norns { namespace rpc { -struct remote_transfer { +struct push_resource { // forward declarations of public input/output types for this RPC class input; class output; // traits used so that the engine knows what to do with the RPC - using self_type = remote_transfer; + using self_type = push_resource; using handle_type = hermes::rpc_handle; using input_type = input; using output_type = output; - using mercury_input_type = hermes::detail::remote_transfer_in_t; - using mercury_output_type = hermes::detail::remote_transfer_out_t; + using mercury_input_type = hermes::detail::push_resource_in_t; + using mercury_output_type = hermes::detail::push_resource_out_t; // RPC public identifier constexpr static const uint16_t public_id = 43; @@ -103,18 +103,18 @@ struct remote_transfer { constexpr static const uint16_t mercury_id = public_id; // RPC name - constexpr static const auto name = "remote_transfer"; + constexpr static const auto name = "push_resource"; // requires response? constexpr static const auto requires_response = true; // Mercury callback to serialize input arguments constexpr static const auto mercury_in_proc_cb = - HG_GEN_PROC_NAME(remote_transfer_in_t); + HG_GEN_PROC_NAME(push_resource_in_t); // Mercury callback to serialize output arguments constexpr static const auto mercury_out_proc_cb = - HG_GEN_PROC_NAME(remote_transfer_out_t); + HG_GEN_PROC_NAME(push_resource_out_t); class input { @@ -122,22 +122,22 @@ struct remote_transfer { friend hg_return_t hermes::detail::post_to_mercury(ExecutionContext*); public: - input(const std::string& address, + input(const std::string& in_address, const std::string& in_nsid, const std::string& out_nsid, - uint32_t backend_type, - uint32_t resource_type, - uint32_t is_collection, - const std::string& resource_name, - const hermes::exposed_memory& buffers) : - m_address(address), + uint32_t out_resource_type, + uint32_t in_is_collection, + const std::string& in_resource_name, + const std::string& out_resource_name, + const hermes::exposed_memory& in_buffers) : + m_in_address(in_address), m_in_nsid(in_nsid), + m_in_resource_name(in_resource_name), + m_in_is_collection(in_is_collection), + m_in_buffers(in_buffers), m_out_nsid(out_nsid), - m_backend_type(backend_type), - m_resource_type(resource_type), - m_is_collection(is_collection), - m_resource_name(resource_name), - m_buffers(buffers) { + m_out_resource_type(out_resource_type), + m_out_resource_name(out_resource_name) { #ifdef HERMES_DEBUG_BUILD this->print("this", __PRETTY_FUNCTION__); @@ -147,32 +147,31 @@ struct remote_transfer { #ifdef HERMES_DEBUG_BUILD input(input&& rhs) : - m_address(std::move(rhs.m_address)), + m_in_address(std::move(rhs.m_in_address)), m_in_nsid(std::move(rhs.m_in_nsid)), + m_in_resource_name(std::move(rhs.m_in_resource_name)), + m_in_is_collection(std::move(rhs.m_in_is_collection)), + m_in_buffers(std::move(rhs.m_in_buffers)), m_out_nsid(std::move(rhs.m_out_nsid)), - m_backend_type(std::move(rhs.m_backend_type)), - m_resource_type(std::move(rhs.m_resource_type)), - m_is_collection(std::move(rhs.m_is_collection)), - m_resource_name(std::move(rhs.m_resource_name)), - m_buffers(std::move(rhs.m_buffers)) { + m_out_resource_type(std::move(rhs.m_out_resource_type)), + m_out_resource_name(std::move(rhs.m_out_resource_name)) { - rhs.m_backend_type = 0; - rhs.m_resource_type = 0; - rhs.m_is_collection = false; + rhs.m_in_is_collection = false; + rhs.m_out_resource_type = 0; this->print("this", __PRETTY_FUNCTION__); rhs.print("rhs", __PRETTY_FUNCTION__); } input(const input& other) : - m_address(other.m_address), + m_in_address(other.m_in_address), m_in_nsid(other.m_in_nsid), + m_in_resource_name(other.m_in_resource_name), + m_in_is_collection(other.m_in_is_collection), + m_in_buffers(other.m_in_buffers), m_out_nsid(other.m_out_nsid), - m_backend_type(other.m_backend_type), - m_resource_type(other.m_resource_type), - m_is_collection(other.m_is_collection), - m_resource_name(other.m_resource_name), - m_buffers(other.m_buffers) { + m_out_resource_type(other.m_out_resource_type), + m_out_resource_name(other.m_out_resource_name) { this->print("this", __PRETTY_FUNCTION__); other.print("other", __PRETTY_FUNCTION__); @@ -182,18 +181,17 @@ struct remote_transfer { operator=(input&& rhs) { if(this != &rhs) { - m_address = std::move(rhs.m_address); + m_in_address = std::move(rhs.m_in_address); m_in_nsid = std::move(rhs.m_in_nsid); + m_in_resource_name = std::move(rhs.m_in_resource_name); + m_in_is_collection = std::move(rhs.m_in_is_collection); + m_in_buffers = std::move(rhs.m_in_buffers); m_out_nsid = std::move(rhs.m_out_nsid); - m_backend_type = std::move(rhs.m_backend_type); - m_resource_type = std::move(rhs.m_resource_type); - m_is_collection = std::move(rhs.m_is_collection); - m_resource_name = std::move(rhs.m_resource_name); - m_buffers = std::move(rhs.m_buffers); + m_out_resource_type = std::move(rhs.m_out_resource_type); + m_out_resource_name = std::move(rhs.m_out_resource_name); - rhs.m_backend_type = 0; - rhs.m_resource_type = 0; - rhs.m_is_collection = false; + rhs.m_in_is_collection = false; + rhs.m_out_resource_type = 0; } this->print("this", __PRETTY_FUNCTION__); @@ -206,14 +204,14 @@ struct remote_transfer { operator=(const input& other) { if(this != &other) { - m_address = other.m_address; + m_in_address = other.m_in_address; m_in_nsid = other.m_in_nsid; + m_in_resource_name = other.m_in_resource_name; + m_in_is_collection = other.m_in_is_collection; + m_in_buffers = other.m_in_buffers; m_out_nsid = other.m_out_nsid; - m_backend_type = other.m_backend_type; - m_resource_type = other.m_resource_type; - m_is_collection = other.m_is_collection; - m_resource_name = other.m_resource_name; - m_buffers = other.m_buffers; + m_out_resource_type = other.m_out_resource_type; + m_out_resource_name = other.m_out_resource_name; } this->print("this", __PRETTY_FUNCTION__); @@ -229,8 +227,8 @@ struct remote_transfer { #endif // ! HERMES_DEBUG_BUILD std::string - address() const { - return m_address; + in_address() const { + return m_in_address; } std::string @@ -239,33 +237,33 @@ struct remote_transfer { } std::string - out_nsid() const { - return m_out_nsid; + in_resource_name() const { + return m_in_resource_name; } - uint32_t - backend_type() const { - return m_backend_type; + bool + in_is_collection() const { + return m_in_is_collection; } - uint32_t - resource_type() const { - return m_resource_type; + hermes::exposed_memory + in_buffers() const { + return m_in_buffers; } - bool - is_collection() const { - return m_is_collection; + std::string + out_nsid() const { + return m_out_nsid; } - std::string - resource_name() const { - return m_resource_name; + uint32_t + out_resource_type() const { + return m_out_resource_type; } - hermes::exposed_memory - buffers() const { - return m_buffers; + std::string + out_resource_name() const { + return m_out_resource_name; } #ifdef HERMES_DEBUG_BUILD @@ -277,80 +275,69 @@ struct remote_transfer { auto c = caller.empty() ? "unknown_caller" : caller; HERMES_DEBUG2("{}, {} ({}) = {{", caller, id, fmt::ptr(this)); - HERMES_DEBUG2(" m_address: \"{}\" ({} -> {}),", - m_address, fmt::ptr(&m_address), - fmt::ptr(m_address.c_str())); + HERMES_DEBUG2(" m_in_address: \"{}\" ({} -> {}),", + m_in_address, fmt::ptr(&m_in_address), + fmt::ptr(m_in_address.c_str())); HERMES_DEBUG2(" m_in_nsid: \"{}\" ({} -> {}),", m_in_nsid, fmt::ptr(&m_in_nsid), fmt::ptr(m_in_nsid.c_str())); + HERMES_DEBUG2(" m_in_resource_name: \"{}\" ({} -> {}),", + m_in_resource_name, fmt::ptr(&m_in_resource_name), + fmt::ptr(m_in_resource_name.c_str())); + HERMES_DEBUG2(" m_in_is_collection: {},", + m_in_is_collection); + HERMES_DEBUG2(" m_in_buffers: {...},"); HERMES_DEBUG2(" m_out_nsid: \"{}\" ({} -> {}),", m_out_nsid, fmt::ptr(&m_out_nsid), fmt::ptr(m_out_nsid.c_str())); - HERMES_DEBUG2(" m_backend_type: {},", - m_backend_type); - HERMES_DEBUG2(" m_resource_type: {},", - m_resource_type); - HERMES_DEBUG2(" m_is_collection: {},", - m_is_collection); - HERMES_DEBUG2(" m_resource_name: \"{}\" ({} -> {}),", - m_resource_name, fmt::ptr(&m_resource_name), - fmt::ptr(m_resource_name.c_str())); - HERMES_DEBUG2(" m_buffers: {...},"); + HERMES_DEBUG2(" m_out_resource_type: {},", + m_out_resource_type); + HERMES_DEBUG2(" m_out_resource_name: \"{}\" ({} -> {}),", + m_out_resource_name, fmt::ptr(&m_out_resource_name), + fmt::ptr(m_out_resource_name.c_str())); HERMES_DEBUG2("}}"); } #endif // ! HERMES_DEBUG_BUILD //TODO: make private explicit - input(const hermes::detail::remote_transfer_in_t& other) : - m_address(other.address), + input(const hermes::detail::push_resource_in_t& other) : + m_in_address(other.in_address), m_in_nsid(other.in_nsid), + m_in_resource_name(other.in_resource_name), + m_in_is_collection(other.in_is_collection), + m_in_buffers(other.in_buffers), m_out_nsid(other.out_nsid), - m_backend_type(other.backend_type), - m_resource_type(other.resource_type), - m_is_collection(other.is_collection), - m_resource_name(other.resource_name), - m_buffers(other.buffers) { + m_out_resource_type(other.out_resource_type), + m_out_resource_name(other.out_resource_name) { - HERMES_DEBUG("input::input(const hermes::detail::remote_transfer_in_t&){{"); - HERMES_DEBUG(" m_address: {} ({}),", - m_address, fmt::ptr(&m_address)); - HERMES_DEBUG(" m_in_nsid: {} ({}),", - m_in_nsid, fmt::ptr(&m_in_nsid)); - HERMES_DEBUG(" m_out_nsid: {} ({}),", - m_out_nsid, fmt::ptr(&m_out_nsid)); - HERMES_DEBUG(" m_backend_type: {},", - m_backend_type); - HERMES_DEBUG(" m_resource_type: {},", - m_resource_type); - HERMES_DEBUG(" m_is_collection: {},", - m_is_collection); - HERMES_DEBUG(" m_buffers: {...},"); - HERMES_DEBUG("}}"); +#ifdef HERMES_DEBUG_BUILD + this->print("this", __PRETTY_FUNCTION__); +#endif // ! HERMES_DEBUG_BUILD } explicit - operator hermes::detail::remote_transfer_in_t() { - return {m_address.c_str(), + operator hermes::detail::push_resource_in_t() { + return {m_in_address.c_str(), m_in_nsid.c_str(), + m_in_resource_name.c_str(), + m_in_is_collection, + hg_bulk_t(m_in_buffers), m_out_nsid.c_str(), - m_backend_type, - m_resource_type, - m_is_collection, - m_resource_name.c_str(), - hg_bulk_t(m_buffers)}; + m_out_resource_type, + m_out_resource_name.c_str()}; } private: - std::string m_address; + std::string m_in_address; std::string m_in_nsid; + std::string m_in_resource_name; + bool m_in_is_collection; + hermes::exposed_memory m_in_buffers; std::string m_out_nsid; - uint32_t m_backend_type; - uint32_t m_resource_type; - bool m_is_collection; - std::string m_resource_name; - hermes::exposed_memory m_buffers; + uint32_t m_out_resource_type; + std::string m_out_resource_name; }; class output { @@ -404,7 +391,7 @@ struct remote_transfer { } explicit - output(const hermes::detail::remote_transfer_out_t& out) { + output(const hermes::detail::push_resource_out_t& out) { m_status = out.status; m_task_error = out.task_error; m_sys_errnum = out.sys_errnum; @@ -412,7 +399,7 @@ struct remote_transfer { } explicit - operator hermes::detail::remote_transfer_out_t() { + operator hermes::detail::push_resource_out_t() { return {m_status, m_task_error, m_sys_errnum, m_elapsed_time}; } @@ -491,7 +478,7 @@ struct pull_resource { m_out_address(std::move(rhs.m_out_address)), m_out_nsid(std::move(rhs.m_out_nsid)), m_in_resource_type(std::move(rhs.m_in_resource_type)), - m_resource_name(std::move(rhs.m_resource_name)), + m_in_resource_name(std::move(rhs.m_in_resource_name)), m_buffers(std::move(rhs.m_buffers)) { rhs.m_in_resource_type = 0; @@ -505,7 +492,7 @@ struct pull_resource { m_out_address(other.m_out_address), m_out_nsid(other.m_out_nsid), m_in_resource_type(other.m_in_resource_type), - m_resource_name(other.m_resource_name), + m_in_resource_name(other.m_in_resource_name), m_buffers(other.m_buffers) { this->print("this", __PRETTY_FUNCTION__); @@ -520,7 +507,7 @@ struct pull_resource { m_out_address = std::move(rhs.m_out_address); m_out_nsid = std::move(rhs.m_out_nsid); m_in_resource_type = std::move(rhs.m_in_resource_type); - m_resource_name = std::move(rhs.m_resource_name); + m_in_resource_name = std::move(rhs.m_in_resource_name); m_buffers = std::move(rhs.m_buffers); rhs.m_in_resource_type = 0; @@ -541,7 +528,7 @@ struct pull_resource { m_out_address = other.m_out_address; m_out_nsid = other.m_out_nsid; m_in_resource_type = other.m_in_resource_type; - m_resource_name = other.m_resource_name; + m_in_resource_name = other.m_in_resource_name; m_buffers = other.m_buffers; } @@ -618,8 +605,6 @@ struct pull_resource { HERMES_DEBUG2(" m_out_resource_name: \"{}\" ({} -> {}),", m_out_resource_name, fmt::ptr(&m_out_resource_name), fmt::ptr(m_out_resource_name.c_str())); - HERMES_DEBUG2(" m_backend_type: {},", - m_backend_type); HERMES_DEBUG2(" m_out_buffers: {...},"); HERMES_DEBUG2("}}"); } @@ -735,18 +720,18 @@ struct pull_resource { }; }; -struct resource_stat { +struct stat_resource { // forward declarations of public input/output types for this RPC class input; class output; // traits used so that the engine knows what to do with the RPC - using self_type = resource_stat; + using self_type = stat_resource; using handle_type = hermes::rpc_handle; using input_type = input; using output_type = output; - using mercury_input_type = hermes::detail::resource_stat_in_t; - using mercury_output_type = hermes::detail::resource_stat_out_t; + using mercury_input_type = hermes::detail::stat_resource_in_t; + using mercury_output_type = hermes::detail::stat_resource_out_t; // RPC public identifier constexpr static const uint16_t public_id = 45; @@ -755,18 +740,18 @@ struct resource_stat { constexpr static const uint16_t mercury_id = public_id; // RPC name - constexpr static const auto name = "resource_stat"; + constexpr static const auto name = "stat_resource"; // requires response? constexpr static const auto requires_response = true; // Mercury callback to serialize input arguments constexpr static const auto mercury_in_proc_cb = - HG_GEN_PROC_NAME(resource_stat_in_t); + HG_GEN_PROC_NAME(stat_resource_in_t); // Mercury callback to serialize output arguments constexpr static const auto mercury_out_proc_cb = - HG_GEN_PROC_NAME(resource_stat_out_t); + HG_GEN_PROC_NAME(stat_resource_out_t); class input { @@ -898,13 +883,13 @@ struct resource_stat { //TODO: make private explicit - input(const hermes::detail::resource_stat_in_t& other) : + input(const hermes::detail::stat_resource_in_t& other) : m_address(other.address), m_nsid(other.nsid), m_resource_type(other.resource_type), m_resource_name(other.resource_name) { - HERMES_DEBUG("input::input(const hermes::detail::resource_stat_in_t&){{"); + HERMES_DEBUG("input::input(const hermes::detail::stat_resource_in_t&){{"); HERMES_DEBUG(" m_address: {} ({}),", m_address, fmt::ptr(&m_address)); HERMES_DEBUG(" m_nsid: {} ({}),", @@ -918,7 +903,7 @@ struct resource_stat { } explicit - operator hermes::detail::resource_stat_in_t() { + operator hermes::detail::stat_resource_in_t() { return {m_address.c_str(), m_nsid.c_str(), m_resource_type, @@ -969,7 +954,7 @@ struct resource_stat { } explicit - output(const hermes::detail::resource_stat_out_t& out) { + output(const hermes::detail::stat_resource_out_t& out) { m_task_error = out.task_error; m_sys_errnum = out.sys_errnum; m_is_collection = out.is_collection; @@ -977,7 +962,7 @@ struct resource_stat { } explicit - operator hermes::detail::resource_stat_out_t() { + operator hermes::detail::stat_resource_out_t() { return {m_task_error, m_sys_errnum, m_is_collection, m_packed_size}; } diff --git a/src/urd.cpp b/src/urd.cpp index 61300e7..84d2da7 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -668,40 +668,22 @@ response_ptr urd::unknown_request_handler(const request_ptr /*base_request*/) { // N.B. This function is called by the progress thread internal to // m_network_endpoint rather than by the main execution thread void -urd::remote_transfer_handler(hermes::request&& req) { +urd::push_resource_handler(hermes::request&& req) { const auto args = req.args(); - LOGGER_WARN("incoming rpc::remote_transfer(from: \"{}@{}:{}\", to: \"{}:{}\")", + LOGGER_WARN("incoming rpc::push_resource(from: \"{}@{}:{}\", to: \"{}:{}\")", args.in_nsid(), - args.address(), - args.resource_name(), + args.in_address(), + args.in_resource_name(), args.out_nsid(), - args.resource_name()); - - LOGGER_DEBUG("rpc::out::args{{"); - LOGGER_DEBUG(" address: \"{}\",", args.address()); - LOGGER_DEBUG(" in_nsid: \"{}\",", args.in_nsid()); - LOGGER_DEBUG(" out_nsid: \"{}\",", args.out_nsid()); - LOGGER_DEBUG(" btype: {} ({}),", - args.backend_type(), - utils::to_string( - static_cast(args.backend_type()))); - LOGGER_DEBUG(" rtype: {} ({}),", - args.resource_type(), - utils::to_string( - static_cast(args.resource_type()))); - LOGGER_DEBUG(" is_collection: {}", args.is_collection()); - LOGGER_DEBUG(" rname: \"{}\",", args.resource_name()); - LOGGER_DEBUG(" buffers: {{...}}"); - LOGGER_DEBUG("};"); - LOGGER_FLUSH(); + args.out_resource_name()); urd_error rv = urd_error::success; boost::optional t; std::shared_ptr src_backend; boost::optional> dst_backend; - auto dst_rtype = static_cast(args.resource_type()); + auto dst_rtype = static_cast(args.out_resource_type()); auth::credentials auth; //XXX fake credentials for now const auto create_rinfo = @@ -711,12 +693,13 @@ urd::remote_transfer_handler(hermes::request&& req) { switch(rtype) { case data::resource_type::remote_resource: return std::make_shared( - args.address(), args.in_nsid(), args.is_collection(), - args.resource_name(), args.buffers()); + args.in_address(), args.in_nsid(), + args.in_is_collection(), args.in_resource_name(), + args.in_buffers()); case data::resource_type::local_posix_path: case data::resource_type::shared_posix_path: return std::make_shared( - args.out_nsid(), args.resource_name()); + args.out_nsid(), args.out_resource_name()); default: rv = urd_error::not_supported; return {}; @@ -765,7 +748,7 @@ urd::remote_transfer_handler(hermes::request&& req) { LOGGER_DEBUG("nsid: {}, bptr: {}", args.in_nsid(), dst_backend); auto ctx = - std::make_shared>(std::move(req)); + std::make_shared>(std::move(req)); std::tie(rv, t) = m_task_mgr->create_remote_initiated_task( @@ -900,11 +883,11 @@ urd::pull_resource_handler(hermes::request&& req) { } void -urd::resource_stat_handler(hermes::request&& req) { +urd::stat_resource_handler(hermes::request&& req) { const auto args = req.args(); - LOGGER_WARN("incoming rpc::resource_stat(\"{}:{}\")", + LOGGER_WARN("incoming rpc::stat_resource(\"{}:{}\")", args.nsid(), args.resource_name()); @@ -1211,16 +1194,16 @@ void urd::init_event_handlers() { std::bind(&urd::unknown_request_handler, this, std::placeholders::_1)); /* remote event handlers */ - m_network_endpoint->register_handler( - std::bind(&urd::remote_transfer_handler, this, + m_network_endpoint->register_handler( + std::bind(&urd::push_resource_handler, this, std::placeholders::_1)); m_network_endpoint->register_handler( std::bind(&urd::pull_resource_handler, this, std::placeholders::_1)); - m_network_endpoint->register_handler( - std::bind(&urd::resource_stat_handler, this, + m_network_endpoint->register_handler( + std::bind(&urd::stat_resource_handler, this, std::placeholders::_1)); diff --git a/src/urd.hpp b/src/urd.hpp index c8c9f99..1205a5e 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -72,9 +72,9 @@ namespace ns { } namespace rpc { - struct remote_transfer; + struct push_resource; struct pull_resource; - struct resource_stat; + struct stat_resource; } enum class urd_error; @@ -124,9 +124,9 @@ private: response_ptr command_handler(const request_ptr req); response_ptr unknown_request_handler(const request_ptr req); - void remote_transfer_handler(hermes::request&& req); + void push_resource_handler(hermes::request&& req); void pull_resource_handler(hermes::request&& req); - void resource_stat_handler(hermes::request&& req); + void stat_resource_handler(hermes::request&& req); // TODO: add helpers for remove and update urd_error create_namespace(const config::namespace_def& nsdef); -- GitLab From e20141d3d25bd0983d53f9a81437f2a09ebc63c3 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Mar 2019 19:51:34 +0100 Subject: [PATCH 32/51] Update hermes --- src/externals/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/externals/hermes b/src/externals/hermes index 94b107c..c3fb408 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit 94b107cbda556bc6814e0a0454a096d6cb5c7955 +Subproject commit c3fb4085b41f504817d34367bdf5717e9653469a -- GitLab From 2887fd20c5640256d61380cb31d9c91a877d44d9 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Mar 2019 19:57:42 +0100 Subject: [PATCH 33/51] First implementation of Mem -> POSIX push --- src/Makefile.am | 2 + src/io/task-manager.cpp | 2 +- src/io/transferors.hpp | 1 + .../local-path-to-remote-resource.cpp | 26 + .../transferors/memory-to-remote-resource.cpp | 266 +++++++ .../transferors/memory-to-remote-resource.hpp | 83 ++ src/urd.cpp | 16 +- src/utils/temporary-file.cpp | 3 +- tests/api-copy-remote-data.cpp | 746 +++++++++++------- tests/api-task-submit.cpp | 8 +- 10 files changed, 835 insertions(+), 318 deletions(-) create mode 100644 src/io/transferors/memory-to-remote-resource.cpp create mode 100644 src/io/transferors/memory-to-remote-resource.hpp diff --git a/src/Makefile.am b/src/Makefile.am index 55bed12..5135f53 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -174,6 +174,8 @@ liburd_aux_la_SOURCES = \ io/transferors/memory-to-shared-path.hpp \ io/transferors/memory-to-remote-path.cpp \ io/transferors/memory-to-remote-path.hpp \ + io/transferors/memory-to-remote-resource.cpp \ + io/transferors/memory-to-remote-resource.hpp \ io/transferor-registry.cpp \ io/transferor-registry.hpp \ job.hpp \ diff --git a/src/io/task-manager.cpp b/src/io/task-manager.cpp index 2dabe1f..6a1e4f4 100644 --- a/src/io/task-manager.cpp +++ b/src/io/task-manager.cpp @@ -471,7 +471,7 @@ task_manager::enqueue_task(io::generic_task&& t) { // of the consumed bandwidth by each task // N.B: we use capture-by-value here so that the task_info_ptr is valid when // the callback is invoked. - const auto completion_callback = [=]() { + const auto completion_callback = [this, t]() { assert(t.info()->status() == task_status::finished || t.info()->status() == task_status::finished_with_error); diff --git a/src/io/transferors.hpp b/src/io/transferors.hpp index 8bf44aa..6666503 100644 --- a/src/io/transferors.hpp +++ b/src/io/transferors.hpp @@ -35,5 +35,6 @@ #include "transferors/memory-to-shared-path.hpp" #include "transferors/memory-to-remote-path.hpp" #include "transferors/remote-resource-to-local-path.hpp" +#include "transferors/memory-to-remote-resource.hpp" #endif /* __IO_TRANSFERORS_HPP__ */ diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index e8bf88f..9db07c5 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -271,6 +271,11 @@ local_path_to_remote_resource_transferor::accept_transfer( (void) auth; +// LOGGER_CRITICAL("accept_remote_request: {}", +// std::chrono::duration_cast( +// std::chrono::steady_clock::now().time_since_epoch()) +// .count()); + std::error_code ec; const auto& d_src = reinterpret_cast(*src); @@ -296,6 +301,17 @@ local_path_to_remote_resource_transferor::accept_transfer( bool is_collection = d_src.is_collection(); + // TODO this should probably go into validate(), but we need to change + // its interface to accept resources rather than resource_infos + // to be able to determine whether d_dst is a directory + if(d_src.name().empty() && bfs::is_directory(d_dst.canonical_path())) { + LOGGER_ERROR("Failed to transfer unnamed resource to directory " + "(target should be a named file)"); + ec.assign(EISDIR, std::generic_category()); + *ctx = std::move(req); // restore ctx + return ec; + } + auto tempfile = std::make_shared( /* output_path */ @@ -348,6 +364,11 @@ local_path_to_remote_resource_transferor::accept_transfer( [this, is_collection, tempfile, d_dst, output_buffer, start]( hermes::request&& req) { +// LOGGER_CRITICAL("completion_callback invoked: {}", +// std::chrono::duration_cast( +// std::chrono::steady_clock::now().time_since_epoch()) +// .count()); + uint32_t usecs = std::chrono::duration_cast( std::chrono::steady_clock::now() - start).count(); @@ -393,6 +414,11 @@ respond: } }; +// LOGGER_CRITICAL("async_pull posted: {}", +// std::chrono::duration_cast( +// std::chrono::steady_clock::now().time_since_epoch()) +// .count()); + m_network_endpoint->async_pull(remote_buffers, local_buffers, std::move(req), diff --git a/src/io/transferors/memory-to-remote-resource.cpp b/src/io/transferors/memory-to-remote-resource.cpp new file mode 100644 index 0000000..fee990b --- /dev/null +++ b/src/io/transferors/memory-to-remote-resource.cpp @@ -0,0 +1,266 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "utils.hpp" +#include "logger.hpp" +#include "resources.hpp" +#include "auth.hpp" +#include "io/task-info.hpp" +#include "io/task-stats.hpp" +#include "memory-to-remote-resource.hpp" + +namespace { + +std::error_code +copy_from_process(pid_t pid, + void* input_address, + void* output_address, + size_t size) { + + std::error_code ec; + + struct iovec local_region, remote_region; + local_region.iov_base = output_address; + remote_region.iov_base = input_address; + local_region.iov_len = remote_region.iov_len = size; + + ssize_t nbytes = + ::process_vm_readv(pid, &local_region, 1, &remote_region, 1, 0); + + if(nbytes == -1) { + LOGGER_ERROR("process_vm_readv() error"); + ec.assign(errno, std::generic_category()); + return ec; + } + + // according to the documentation, partial reads should only happen + // at the granularity of iovec elements. Given that we only have one + // element in our 'src' iovec, we should never hit this, but just in case + // we return an EIO + if(static_cast(nbytes) < size) { + LOGGER_ERROR("process_vm_readv() received fewer data than expected"); + ec.assign(EIO, std::generic_category()); + return ec; + } + + if(::msync(output_address, size, MS_SYNC) != 0) { + LOGGER_ERROR("Failed to sync mapped buffer to file"); + ec.assign(errno, std::generic_category()); + return ec; + } + + return ec; +} + +} // anonymous namespace + +namespace norns { +namespace io { + +memory_region_to_remote_resource_transferor:: + memory_region_to_remote_resource_transferor( + std::shared_ptr network_endpoint) : + m_network_endpoint(network_endpoint) {} + +bool +memory_region_to_remote_resource_transferor::validate( + const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) const { + + const auto& d_src = + reinterpret_cast(*src_info); + const auto& d_dst = + reinterpret_cast(*dst_info); + + // region length cannot be zero + if(d_src.size() == 0) { + return false; + } + + // we don't allow destination paths that look like directories + if(d_dst.name().back() == '/') { + return false; + } + + return true; +} + +std::error_code +memory_region_to_remote_resource_transferor::transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + + std::error_code ec; + const auto& d_src = + reinterpret_cast(*src); + const auto& d_dst = + reinterpret_cast(*dst); + + LOGGER_DEBUG("[{}] transfer: [{} {}+{}] -> {}", task_info->id(), + auth.pid(), utils::n2hexstr(d_src.address()), d_src.size(), + d_dst.to_string()); + + // copy the user buffer to a temporary file local file using + // ::process_vm_readv() so that we can safely send it to remote peers + auto tempfile = + std::make_shared( + std::string{"norns-tempfile-%%%%-%%%%-%%%%"}, + bfs::path{"/tmp"}, + d_src.size(), + ec); + + if(ec) { + LOGGER_ERROR("Failed to create temporary file: {}", ec.message()); + return ec; + } + + LOGGER_DEBUG("created temporary file: {}", tempfile->path()); + + auto output_buffer = + std::make_shared( + tempfile->path().string(), + hermes::access_mode::write_only, + &ec); + + if(ec) { + LOGGER_ERROR("Failed mapping output buffer: {}", ec.value()); + return ec; + } + + if((ec = ::copy_from_process(auth.pid(), + reinterpret_cast(d_src.address()), + output_buffer->data(), d_src.size()))) { + LOGGER_ERROR("Failed to copy data from process memory: {}", + ec.message()); + return ec; + } + + // TODO: release() + remap() + output_buffer->protect(hermes::access_mode::read_only, &ec); + + if(ec) { + LOGGER_ERROR("Failed changing protections from output buffer: {}", + ec.value()); + return ec; + } + + try { + + hermes::endpoint endp = m_network_endpoint->lookup(d_dst.address()); + + std::vector bufvec{ + hermes::mutable_buffer{output_buffer->data(), output_buffer->size()} + }; + + auto local_buffers = + m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); + +// LOGGER_CRITICAL("push_resource RPC posted: {}", +// std::chrono::duration_cast( +// std::chrono::steady_clock::now().time_since_epoch()) +// .count()); + + auto resp = + m_network_endpoint->post( + endp, + rpc::push_resource::input{ + m_network_endpoint->self_address(), + d_src.parent()->nsid(), + d_dst.parent()->nsid(), + // XXX this resource_type should not be needed, but we + // XXX cannot (easily) find it out right now in the server, + // XXX for now we propagate it, but we should implement + // XXX a lookup()/stat() function in backends to retrieve + // XXX this information locally from the resource id + static_cast( + data::resource_type::local_posix_path), + d_src.is_collection(), + d_src.name(), + d_dst.name(), + local_buffers + }).get(); + +// LOGGER_CRITICAL("push_resource response retrieved: {}", +// std::chrono::duration_cast( +// std::chrono::steady_clock::now().time_since_epoch()) +// .count()); + + if(static_cast(resp.at(0).status()) == + task_status::finished_with_error) { + // XXX error interface should be improved + return std::make_error_code( + static_cast(resp.at(0).sys_errnum())); + } + + task_info->record_transfer(output_buffer->size(), + resp.at(0).elapsed_time()); + + LOGGER_DEBUG("Remote pull request completed with output " + "{{status: {}, task_error: {}, sys_errnum: {}}} " + "({} bytes, {} usecs)", + resp.at(0).status(), resp.at(0).task_error(), + resp.at(0).sys_errnum(), output_buffer->size(), + resp.at(0).elapsed_time()); + + return ec; + } + catch(const std::exception& ex) { + LOGGER_ERROR(ex.what()); + return std::make_error_code(static_cast(-1)); + } + + return ec; +} + +std::error_code +memory_region_to_remote_resource_transferor::accept_transfer( + const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) const { + + (void) auth; + (void) task_info; + (void) src; + (void) dst; + + LOGGER_ERROR("This function should never be called for this transfer type"); + return std::make_error_code(static_cast(0)); +} + +std::string +memory_region_to_remote_resource_transferor::to_string() const { + return "transferor[memory_region => remote_resource]"; +} + +} // namespace io +} // namespace norns diff --git a/src/io/transferors/memory-to-remote-resource.hpp b/src/io/transferors/memory-to-remote-resource.hpp new file mode 100644 index 0000000..ecc9873 --- /dev/null +++ b/src/io/transferors/memory-to-remote-resource.hpp @@ -0,0 +1,83 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef __IO_MEM_TO_REMOTE_RESOURCE_TX__ +#define __IO_MEM_TO_REMOTE_RESOURCE_TX__ + +#include +#include +#include "transferor.hpp" + +namespace norns { + +// forward declarations +namespace auth { +struct credentials; +} + +namespace data { +struct resource_info; +struct resource; +} + +namespace io { + +struct memory_region_to_remote_resource_transferor : public transferor { + + memory_region_to_remote_resource_transferor( + std::shared_ptr network_endpoint); + + bool + validate(const std::shared_ptr& src_info, + const std::shared_ptr& dst_info) + const override final; + + std::error_code + transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::error_code + accept_transfer(const auth::credentials& auth, + const std::shared_ptr& task_info, + const std::shared_ptr& src, + const std::shared_ptr& dst) + const override final; + + std::string + to_string() const override final; + +private: + std::shared_ptr m_network_endpoint; +}; + +} // namespace io +} // namespace norns + +#endif /* __MEM_TO_REMOTE_RESOURCE_TX__ */ diff --git a/src/urd.cpp b/src/urd.cpp index 84d2da7..0ee133d 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -194,10 +194,10 @@ urd_error urd::validate_iotask_args(iotask_type type, return urd_error::not_supported; } - // dst_resource cannot be a memory region - if(dst_rinfo->type() == data::resource_type::memory_region) { - return urd_error::not_supported; - } +// // dst_resource cannot be a memory region +// if(dst_rinfo->type() == data::resource_type::memory_region) { +// return urd_error::not_supported; +// } return urd_error::success; } @@ -1351,9 +1351,11 @@ void urd::load_transfer_plugins() { std::make_shared()); // memory region -> remote path - load_plugin(data::resource_type::memory_region, - data::resource_type::remote_posix_path, - std::make_shared()); + load_plugin( + data::resource_type::memory_region, + data::resource_type::remote_resource, + std::make_shared( + m_network_endpoint)); // local path -> local path load_plugin(data::resource_type::local_posix_path, diff --git a/src/utils/temporary-file.cpp b/src/utils/temporary-file.cpp index 35e9c59..64dc1b5 100644 --- a/src/utils/temporary-file.cpp +++ b/src/utils/temporary-file.cpp @@ -115,7 +115,8 @@ temporary_file::temporary_file(const std::string& pattern, if(!fh) { ec.assign(errno, std::generic_category()); - // LOGGER_ERROR("Failed to create temporary file: {}", ec.message()); + LOGGER_ERROR("Failed to create temporary file {}: {}", + filename, ec.message()); return; } diff --git a/tests/api-copy-remote-data.cpp b/tests/api-copy-remote-data.cpp index dce9c09..5447d1c 100644 --- a/tests/api-copy-remote-data.cpp +++ b/tests/api-copy-remote-data.cpp @@ -299,8 +299,6 @@ SCENARIO("errors copying local POSIX path to remote POSIX path", WHEN("copying a NORNS_LOCAL_PATH file from a subdir without " "appropriate permissions to access it") { - norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), @@ -531,7 +529,6 @@ SCENARIO("errors copying local POSIX path to remote POSIX path", } #endif - env.notify_success(); } } @@ -1490,29 +1487,454 @@ SCENARIO("copy local POSIX file to remote POSIX subdir", // wait until the task completes rv = norns_wait(&task); - THEN("norns_wait() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); + THEN("norns_wait() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Copied files are identical to original") { + bfs::path src = + env.get_from_namespace(nsid0, src_subdir1); + bfs::path dst = + env.get_from_namespace(nsid1, dst_subdir1); + + REQUIRE(compare_directories(src, dst) == true); + } + } + } + } + } + + env.notify_success(); + } +} + +/******************************************************************************/ +/* tests for push transfers (memory buffers) */ +/******************************************************************************/ +SCENARIO("copy local memory region to remote POSIX file", + "[api::norns_submit_push_memory_to_posix_file]") { + + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env; + + const char* nsid0 = "tmp0"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path dst_mnt; + + // create namespaces + std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); + + // create input data buffer (around 40MiBs) + std::vector input_data(10000000, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + // output names + const bfs::path dst_file_at_root0 = "/file0"; + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0/"; // existing + const bfs::path dst_subdir1 = "/output_dir0"; // existing but does not look as a directory + const bfs::path dst_subdir2 = "/output_dir0/a/b/c/d/"; // existing + const bfs::path dst_subdir3 = "/output_dir0/a/b/c/d"; // existing but does not look as a directory + const bfs::path dst_subdir4 = "/output_dir1/"; // non-existing + const bfs::path dst_subdir5 = "/output_dir1/a/b/c/d/"; // non-existing + + // create required output directories + env.add_to_namespace(nsid0, dst_subdir0); + env.add_to_namespace(nsid0, dst_subdir2); + + // + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH file " + "located at DST's namespace root '/'") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("Output file contains buffer data") { + + bfs::path dst = + env.get_from_namespace( + nsid0, dst_file_at_root0); + + REQUIRE(compare(input_data, dst) == true); + } + } + } + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH file " + "located at a DST's subdir") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_file_at_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_status() reports NORNS_EFINISHED") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHED); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Output file contains buffer data") { + + bfs::path dst = + env.get_from_namespace( + nsid0, dst_file_at_subdir0); + + REQUIRE(compare(input_data, dst) == true); + } + } + } + } + } + + env.notify_success(); + } + +#ifndef USE_REAL_DAEMON + GIVEN("a non-running urd instance") { + WHEN("attempting to request a transfer") { + + norns_iotask_t task = NORNS_IOTASK( + NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), + NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_ECONNFAILED is returned") { + REQUIRE(rv == NORNS_ECONNFAILED); + } + } + } +#endif +} + + +/******************************************************************************/ +/* tests for push transfers (memory buffers, errors) */ +/******************************************************************************/ +SCENARIO("errors copying local memory region to remote POSIX file", + "[api::norns_submit_push_memory_to_posix_file_errors]") { + + GIVEN("a running urd instance") { + + /**********************************************************************/ + /* setup common environment */ + /**********************************************************************/ + test_env env; + + const char* nsid0 = "tmp0"; + const char* remote_host = "127.0.0.1:42000"; + bfs::path dst_mnt; + + // create namespaces + std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); + + // create input data buffer (around 40MiBs) + std::vector input_data(10000000, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + // output names + const bfs::path dst_file_at_root0 = "/file0"; + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0/"; // existing + const bfs::path dst_subdir1 = "/output_dir0"; // existing but does not look as a directory + const bfs::path dst_subdir2 = "/output_dir0/a/b/c/d/"; // existing + const bfs::path dst_subdir3 = "/output_dir0/a/b/c/d"; // existing but does not look as a directory + const bfs::path dst_subdir4 = "/output_dir1/"; // non-existing + const bfs::path dst_subdir5 = "/output_dir1/a/b/c/d/"; // non-existing + + // create required output directories + env.add_to_namespace(nsid0, dst_subdir0); + env.add_to_namespace(nsid0, dst_subdir2); + + //TODO + // - copy a valid but removed memory region (cannot control => undefined behavior) + // - providing a non-existing directory path (i.e. finished with /) as output name + // - providing an existing path that points to a directory as output name + WHEN("copying an invalid memory region to a NORNS_REMOTE_PATH") { + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION((void*) 0x42, 42000), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and " + "EFAULT") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EFAULT); + } + } + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH at " + "DST's /") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_root.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH existing " + "directory at DST's /") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH existing " + "directory at / that does not look like a directory") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir1.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and " + "EISDIR") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EISDIR); + } + } + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH existing " + "directory at a DST's arbitary subdir") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir2.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH existing " + "directory at a DST's arbitary subdir that does not look like " + "a directory") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir3.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("norns_wait() return NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("norns_status() reports NORNS_ESYSTEMERROR and " + "EISDIR") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == EISDIR); + } + } + } + } + + + // i.e. a destination path that 'looks like' a directory + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH " + "corresponding to a non-existing directory at DST's /") { + + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); + + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir4.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); + } + } + + WHEN("copying a valid memory region to a NORNS_REMOTE_PATH " + "non-existing arbitrary subdir at DST") { - THEN("norns_status() reports NORNS_EFINISHED") { - norns_stat_t stats; - rv = norns_status(&task, &stats); + // create input data buffer + std::vector input_data(100, 42); + void* region_addr = input_data.data(); + size_t region_size = input_data.size() * sizeof(int); - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(stats.st_status == NORNS_EFINISHED); + norns_iotask_t task = + NORNS_IOTASK(NORNS_IOTASK_COPY, + NORNS_MEMORY_REGION(region_addr, region_size), + NORNS_REMOTE_PATH(nsid0, + remote_host, + dst_subdir5.c_str())); - THEN("Copied files are identical to original") { - bfs::path src = - env.get_from_namespace(nsid0, src_subdir1); - bfs::path dst = - env.get_from_namespace(nsid1, dst_subdir1); + norns_error_t rv = norns_submit(&task); - REQUIRE(compare_directories(src, dst) == true); - } - } - } + THEN("norns_submit() returns NORNS_EBADARGS") { + REQUIRE(rv == NORNS_EBADARGS); } } + env.notify_success(); } @@ -1740,8 +2162,6 @@ SCENARIO("copy local POSIX path to remote POSIX path involving links", WHEN("copying a single NORNS_LOCAL_PATH arbitrary subdir" "through a symlink also located at '/'" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, NORNS_LOCAL_PATH(nsid0, @@ -3639,8 +4059,6 @@ SCENARIO("copy remote POSIX path to local POSIX path involving links", WHEN("copying a single NORNS_REMOTE_PATH arbitrary subdir" "through a symlink also located at '/'" ) { - norns_op_t task_op = NORNS_IOTASK_COPY; - norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_COPY, NORNS_REMOTE_PATH(nsid0, @@ -3820,285 +4238,3 @@ SCENARIO("copy remote POSIX path to local POSIX path involving links", env.notify_success(); } } - - -#if 0 -SCENARIO("copy local memory buffer to local POSIX file", "[api::norns_submit_copy_buffer_to_file]") { - GIVEN("a running urd instance") { - - /**********************************************************************/ - /* setup common environment */ - /**********************************************************************/ - test_env env; - - const char* nsid0 = "tmp0"; - bfs::path dst_mnt; - - // create namespaces - std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); - - // create input data buffer - std::vector input_data(100, 42); - void* region_addr = input_data.data(); - size_t region_size = input_data.size() * sizeof(int); - - // output names - const bfs::path dst_file_at_root0 = "/file0"; - const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; - const bfs::path dst_root = "/"; - const bfs::path dst_subdir0 = "/output_dir0/"; // existing - const bfs::path dst_subdir1 = "/output_dir0"; // existing but does not look as a directory - const bfs::path dst_subdir2 = "/output_dir0/a/b/c/d/"; // existing - const bfs::path dst_subdir3 = "/output_dir0/a/b/c/d"; // existing but does not look as a directory - const bfs::path dst_subdir4 = "/output_dir1/"; // non-existing - const bfs::path dst_subdir5 = "/output_dir1/a/b/c/d/"; // non-existing - - // create required output directories - env.add_to_namespace(nsid0, dst_subdir0); - env.add_to_namespace(nsid0, dst_subdir2); - - /**************************************************************************************************************/ - /* tests for error conditions */ - /**************************************************************************************************************/ - //TODO - // - copy a valid but removed memory region (cannot control => undefined behavior) - // - providing a non-existing directory path (i.e. finished with /) as output name - // - providing an existing path that points to a directory as output name - WHEN("copying an invalid memory region to a local POSIX file") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION((void*) 0x42, region_size), - NORNS_LOCAL_PATH(nsid0, dst_file_at_root0.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - - // wait until the task completes - rv = norns_wait(&task); - - THEN("norns_wait() return NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - - THEN("norns_status() reports NORNS_ESYSTEMERROR and EFAULT") { - norns_stat_t stats; - rv = norns_status(&task, &stats); - - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); - REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); - REQUIRE(stats.st_sys_errno == EFAULT); - } - } - } - } - - WHEN("copying a valid memory region to a local POSIX file located at '/'") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_file_at_root0.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - - // wait until the task completes - rv = norns_wait(&task); - - THEN("norns_wait() return NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - - THEN("Output file contains buffer data") { - - bfs::path dst = env.get_from_namespace(nsid0, dst_file_at_root0); - - REQUIRE(compare(input_data, dst) == true); - } - } - } - } - - WHEN("copying a valid memory region to a local POSIX file located at a subdir") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_file_at_subdir0.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - - // wait until the task completes - rv = norns_wait(&task); - - THEN("norns_wait() return NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - - THEN("Output file contains buffer data") { - - bfs::path dst = env.get_from_namespace(nsid0, dst_file_at_subdir0); - - REQUIRE(compare(input_data, dst) == true); - } - } - } - } - - WHEN("copying a valid memory region to a local POSIX /") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_root.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_EBADARGS") { - REQUIRE(rv == NORNS_EBADARGS); - } - } - - WHEN("copying a valid memory region to a local POSIX existing directory at /") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir0.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_EBADARGS") { - REQUIRE(rv == NORNS_EBADARGS); - } - } - - WHEN("copying a valid memory region to a local POSIX existing directory at /") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir1.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - - // wait until the task completes - rv = norns_wait(&task); - - THEN("norns_wait() return NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - - THEN("norns_status() reports NORNS_ESYSTEMERROR and EISDIR") { - norns_stat_t stats; - rv = norns_status(&task, &stats); - - REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); - REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); - REQUIRE(stats.st_sys_errno == EISDIR); - } - } - } - } - - WHEN("copying a valid memory region to a local POSIX existing directory at an arbitary subdir") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir2.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_EBADARGS") { - REQUIRE(rv == NORNS_EBADARGS); - } - } - - WHEN("copying a valid memory region to a local POSIX existing directory at an arbitary subdir") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir3.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - REQUIRE(task.t_id != 0); - - // wait until the task completes - rv = norns_wait(&task); - - THEN("norns_wait() return NORNS_SUCCESS") { - REQUIRE(rv == NORNS_SUCCESS); - - THEN("norns_status() reports NORNS_ESYSTEMERROR and EISDIR") { - norns_stat_t stats; - rv = norns_status(&task, &stats); - - REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); - REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); - REQUIRE(stats.st_sys_errno == EISDIR); - } - } - } - } - - - // i.e. a destination path that 'looks like' a directory - WHEN("copying a valid memory region to a local POSIX non-existing directory") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir4.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_EBADARGS") { - REQUIRE(rv == NORNS_EBADARGS); - } - } - - WHEN("copying a valid memory region to a local POSIX non-existing directory") { - - norns_op_t task_op = NORNS_IOTASK_COPY; - - norns_iotask_t task = NORNS_IOTASK(task_op, - NORNS_MEMORY_REGION(region_addr, region_size), - NORNS_LOCAL_PATH(nsid0, dst_subdir5.c_str())); - - norns_error_t rv = norns_submit(&task); - - THEN("norns_submit() returns NORNS_EBADARGS") { - REQUIRE(rv == NORNS_EBADARGS); - } - } - - env.notify_success(); - } -} -#endif diff --git a/tests/api-task-submit.cpp b/tests/api-task-submit.cpp index a0531d4..1111a54 100644 --- a/tests/api-task-submit.cpp +++ b/tests/api-task-submit.cpp @@ -156,8 +156,8 @@ SCENARIO("submit request", "[api::norns_submit]") { norns_error_t rv = norns_submit(&task); - THEN("NORNS_ENOTSUPPORTED is returned") { - REQUIRE(rv == NORNS_ENOTSUPPORTED); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); } } @@ -347,8 +347,8 @@ SCENARIO("submit request", "[api::norns_submit]") { norns_error_t rv = norns_submit(&task); - THEN("NORNS_ENOTSUPPORTED is returned") { - REQUIRE(rv == NORNS_ENOTSUPPORTED); + THEN("norns_submit() returns NORNS_SUCCESS") { + REQUIRE(rv == NORNS_SUCCESS); } } -- GitLab From ae58e3459172de54b429cfc2bd5d18bef90a7637 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Mar 2019 20:19:55 +0100 Subject: [PATCH 34/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b5d1ce5..f002d59 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,31 +208,33 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - make -j$(nproc) api # - NORNS_DEBUG_OUTPUT_TO_STDERR=1 NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_status]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" - - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" after_script: - pwd -- GitLab From d86a27e8191e883f836ae0722cd52fa87586c24f Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 4 Mar 2019 20:22:30 +0100 Subject: [PATCH 35/51] Update hermes --- src/externals/hermes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/externals/hermes b/src/externals/hermes index c3fb408..c5211af 160000 --- a/src/externals/hermes +++ b/src/externals/hermes @@ -1 +1 @@ -Subproject commit c3fb4085b41f504817d34367bdf5717e9653469a +Subproject commit c5211af77d4dca1f596f7da2d3a6f13cf1bee014 -- GitLab From ca49f1487b5918f0c1b44de477063c5301981bdd Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 20:07:30 +0100 Subject: [PATCH 36/51] Add staging_directory option to configuration file --- etc/norns.conf.in | 5 +- src/Makefile.am | 1 + src/config/config-schema.hpp | 5 + src/config/defaults.hpp | 1 + src/config/keywords.hpp | 1 + src/config/settings.cpp | 167 ++++++++++++++++++++++---- src/config/settings.hpp | 227 +++++++++++++++++++++++++++-------- src/main.cpp | 8 +- src/urd.cpp | 32 +++++ src/urd.hpp | 2 + tests/fake-daemon.cpp | 156 +++++++++++++++++------- tests/fake-daemon.hpp | 2 + tests/test-env.cpp | 14 +++ 13 files changed, 501 insertions(+), 120 deletions(-) diff --git a/etc/norns.conf.in b/etc/norns.conf.in index becbdf6..4fa1a43 100644 --- a/etc/norns.conf.in +++ b/etc/norns.conf.in @@ -25,7 +25,10 @@ global_settings: [ remote_port: 42000, # number of worker threads to serve I/O requests - workers: 4 + workers: 4, + + # staging dir for temporary resources + staging_directory: "/tmp/urd/" ] ## list of namespaces available by default when service starts diff --git a/src/Makefile.am b/src/Makefile.am index 5135f53..3403e05 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -261,6 +261,7 @@ config/defaults.cpp: Makefile echo " const char* pidfile = \"$(localstatedir)/urd.pid\";"; \ \ echo " const uint32_t workers_in_pool = std::thread::hardware_concurrency();"; \ + echo " const char* staging_directory = \"/tmp/urd/\";"; \ echo " const uint32_t backlog_size = 128;"; \ echo " const char* config_file = \"$(sysconfdir)/norns.conf\";"; \ echo "} // namespace defaults"; \ diff --git a/src/config/config-schema.hpp b/src/config/config-schema.hpp index 611a6d4..e03c655 100644 --- a/src/config/config-schema.hpp +++ b/src/config/config-schema.hpp @@ -102,6 +102,11 @@ const file_schema valid_options = declare_file({ keywords::workers, opt_type::mandatory, converter(parsers::parse_number)), + + declare_option( + keywords::staging_directory, + opt_type::mandatory, + converter(parsers::parse_path)), }) ), diff --git a/src/config/defaults.hpp b/src/config/defaults.hpp index cb33fe1..ff2488f 100644 --- a/src/config/defaults.hpp +++ b/src/config/defaults.hpp @@ -52,6 +52,7 @@ namespace defaults { extern const in_port_t remote_port; extern const char* pidfile; extern const uint32_t workers_in_pool; + extern const char* staging_directory; extern const uint32_t backlog_size; extern const char* config_file; diff --git a/src/config/keywords.hpp b/src/config/keywords.hpp index 0f73457..e599c38 100644 --- a/src/config/keywords.hpp +++ b/src/config/keywords.hpp @@ -52,6 +52,7 @@ constexpr static const auto bind_address = "bind_address"; constexpr static const auto remote_port = "remote_port"; constexpr static const auto pidfile = "pidfile"; constexpr static const auto workers = "workers"; +constexpr static const auto staging_directory = "staging_directory"; // option names for 'namespaces' section constexpr static const auto nsid = "nsid"; diff --git a/src/config/settings.cpp b/src/config/settings.cpp index eb44067..f188118 100644 --- a/src/config/settings.cpp +++ b/src/config/settings.cpp @@ -44,7 +44,9 @@ namespace bfs = boost::filesystem; namespace norns { namespace config { -settings::settings() { } +settings::settings() { + this->load_defaults(); +} settings::settings(const std::string& progname, bool daemonize, @@ -60,6 +62,7 @@ settings::settings(const std::string& progname, uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, + const bfs::path& staging_directory, uint32_t backlog_size, const bfs::path& cfgfile, const std::list& defns) : @@ -77,11 +80,13 @@ settings::settings(const std::string& progname, m_remote_port(remote_port), m_daemon_pidfile(pidfile), m_workers_in_pool(workers), + m_staging_directory(staging_directory), m_backlog_size(backlog_size), m_config_file(cfgfile), m_default_namespaces(defns) { } -void settings::load_defaults() { +void +settings::load_defaults() { m_progname = defaults::progname; m_daemonize = defaults::daemonize; m_use_syslog = defaults::use_syslog; @@ -96,12 +101,14 @@ void settings::load_defaults() { m_remote_port = defaults::remote_port; m_daemon_pidfile = defaults::pidfile; m_workers_in_pool = defaults::workers_in_pool; + m_staging_directory = defaults::staging_directory; m_backlog_size = defaults::backlog_size; m_config_file = defaults::config_file; m_default_namespaces.clear(); } -void settings::load_from_file(const bfs::path& filename) { +void +settings::load_from_file(const bfs::path& filename) { file_options::options_map opt_map; file_options::parse_yaml_file(filename, config::valid_options, opt_map); @@ -131,6 +138,8 @@ void settings::load_from_file(const bfs::path& filename) { m_remote_port = gsettings.get_as(keywords::remote_port); m_daemon_pidfile = gsettings.get_as(keywords::pidfile); m_workers_in_pool = gsettings.get_as(keywords::workers); + m_staging_directory = + gsettings.get_as(keywords::staging_directory); m_backlog_size = defaults::backlog_size; // load definitions for default namespaces @@ -148,7 +157,8 @@ void settings::load_from_file(const bfs::path& filename) { } } -std::string settings::to_string() const { +std::string +settings::to_string() const { std::string str = std::string("settings {\n") + " m_progname: " + m_progname + ",\n" + " m_daemonize: " + (m_daemonize ? "true" : "false") + ",\n" + @@ -164,6 +174,7 @@ std::string settings::to_string() const { " m_remote_port: " + std::to_string(m_remote_port) + ",\n" + " m_pidfile: " + m_daemon_pidfile.string() + ",\n" + " m_workers: " + std::to_string(m_workers_in_pool) + ",\n" + + " m_staging_directory: " + m_staging_directory.string() + ",\n" + " m_backlog_size: " + std::to_string(m_backlog_size) + ",\n" + " m_config_file: " + m_config_file.string() + ",\n" + "};"; @@ -171,74 +182,186 @@ std::string settings::to_string() const { return str; } -std::string& settings::progname() { +std::string +settings::progname() const { return m_progname; } -bool& settings::daemonize() { +void +settings::progname(const std::string& progname) { + m_progname = progname; +} + +bool +settings::daemonize() const { return m_daemonize; } -bool& settings::use_syslog() { +void +settings::daemonize(bool daemonize) { + m_daemonize = daemonize; +} + +bool +settings::use_syslog() const { return m_use_syslog; } -bool& settings::use_console() { +void +settings::use_syslog(bool use_syslog) { + m_use_syslog = use_syslog; +} + +bool +settings::use_console() const { return m_use_console; } -bfs::path& settings::log_file() { +void +settings::use_console(bool use_console) { + m_use_console = use_console; +} + +bfs::path +settings::log_file() const { return m_log_file; } -uint32_t& settings::log_file_max_size() { +void +settings::log_file(const bfs::path& log_file) { + m_log_file = log_file; +} + +uint32_t +settings::log_file_max_size() const { return m_log_file_max_size; } -bool& settings::dry_run() { +void +settings::log_file_max_size(uint32_t log_file_max_size) { + m_log_file_max_size = log_file_max_size; +} + +bool +settings::dry_run() const { return m_dry_run; } -uint32_t& settings::dry_run_duration() { +void +settings::dry_run(bool dry_run) { + m_dry_run = dry_run; +} + +uint32_t +settings::dry_run_duration() const { return m_dry_run_duration; } -bfs::path& settings::global_socket() { +void +settings::dry_run_duration(uint32_t dry_run_duration) { + m_dry_run_duration = dry_run_duration; +} + +bfs::path +settings::global_socket() const { return m_global_socket; } -bfs::path& settings::control_socket() { +void +settings::global_socket(const bfs::path& global_socket) { + m_global_socket = global_socket; +} + +bfs::path +settings::control_socket() const { return m_control_socket; } -std::string& settings::bind_address() { +void +settings::control_socket(const bfs::path& control_socket) { + m_control_socket = control_socket; +} + +std::string +settings::bind_address() const { return m_bind_address; } -in_port_t& settings::remote_port() { +void +settings::bind_address(const std::string& bind_address) { + m_bind_address = bind_address; +} + +in_port_t +settings::remote_port() const { return m_remote_port; } -bfs::path& settings::pidfile() { +void +settings::remote_port(in_port_t remote_port) { + m_remote_port = remote_port; +} + +bfs::path +settings::pidfile() const { return m_daemon_pidfile; } -uint32_t& settings::workers_in_pool() { +void +settings::pidfile(const bfs::path& pidfile) { + m_daemon_pidfile = pidfile; +} + +uint32_t +settings::workers_in_pool() const { return m_workers_in_pool; } -uint32_t& settings::backlog_size() { +void +settings::workers_in_pool(uint32_t workers_in_pool) { + m_workers_in_pool = workers_in_pool; +} + +bfs::path +settings::staging_directory() const { + return m_staging_directory; +} + +void +settings::staging_directory(const bfs::path& staging_directory) { + m_staging_directory = staging_directory; +} + +uint32_t +settings::backlog_size() const { return m_backlog_size; } -bfs::path& settings::config_file() { +void +settings::backlog_size(uint32_t backlog_size) { + m_backlog_size = backlog_size; +} + +bfs::path +settings::config_file() const { return m_config_file; } -std::list& settings::default_namespaces() { +void +settings::config_file(const bfs::path& config_file) { + m_config_file = config_file; +} + +std::list +settings::default_namespaces() const { return m_default_namespaces; } +void +settings::default_namespaces( + const std::list& default_namespaces) { + m_default_namespaces = default_namespaces; +} + } // namespace config } // namespace norns - diff --git a/src/config/settings.hpp b/src/config/settings.hpp index d53bedb..d0c62b2 100644 --- a/src/config/settings.hpp +++ b/src/config/settings.hpp @@ -52,41 +52,58 @@ struct namespace_def { m_capacity(capacity), m_visibility(visibility) { } - std::string nsid() const { + namespace_def(const namespace_def& other) = default; + + namespace_def(namespace_def&& rhs) = default; + + namespace_def& + operator=(const namespace_def& other) = default; + + namespace_def& + operator=(namespace_def&& rhs) = default; + + std::string + nsid() const { return m_nsid; } - bool track() const { + bool + track() const { return m_track; } - bfs::path mountpoint() const { + bfs::path + mountpoint() const { return m_mountpoint; } - std::string alias() const { + std::string + alias() const { return m_alias; } - uint64_t capacity() const { + uint64_t + capacity() const { return m_capacity; } - std::string visibility() const { + std::string + visibility() const { return m_visibility; } - const std::string m_nsid; - const bool m_track; - const bfs::path m_mountpoint; - const std::string m_alias; - const uint64_t m_capacity; - const std::string m_visibility; + std::string m_nsid; + bool m_track; + bfs::path m_mountpoint; + std::string m_alias; + uint64_t m_capacity; + std::string m_visibility; }; struct settings { settings(); + settings(const std::string& progname, bool daemonize, bool use_syslog, @@ -101,47 +118,157 @@ struct settings { uint32_t remote_port, const bfs::path& pidfile, uint32_t workers, + const bfs::path& staging_directory, uint32_t backlog_size, const bfs::path& cfgfile, const std::list& defns); - void load_defaults(); - void load_from_file(const bfs::path& filename); - std::string to_string() const; - - std::string& progname(); - bool& daemonize(); - bool& use_syslog(); - bool& use_console(); - bfs::path& log_file(); - uint32_t& log_file_max_size(); - bool& dry_run(); - uint32_t& dry_run_duration(); - bfs::path& global_socket(); - bfs::path& control_socket(); - std::string& bind_address(); - in_port_t& remote_port(); - bfs::path& pidfile(); - uint32_t& workers_in_pool(); - uint32_t& backlog_size(); - bfs::path& config_file(); - std::list& default_namespaces(); - - std::string m_progname = defaults::progname; - bool m_daemonize = defaults::daemonize; - bool m_use_syslog = defaults::use_syslog; - bool m_use_console = defaults::use_console; - bfs::path m_log_file = defaults::log_file; - uint32_t m_log_file_max_size = defaults::log_file_max_size; - bool m_dry_run = defaults::dry_run; - uint32_t m_dry_run_duration = defaults::dry_run_duration; - bfs::path m_global_socket = defaults::global_socket; - bfs::path m_control_socket = defaults::control_socket; - std::string m_bind_address = defaults::bind_address; - in_port_t m_remote_port = defaults::remote_port; - bfs::path m_daemon_pidfile = defaults::pidfile; - uint32_t m_workers_in_pool = defaults::workers_in_pool; - uint32_t m_backlog_size = defaults::backlog_size; - bfs::path m_config_file = defaults::config_file; + + void + load_defaults(); + + void + load_from_file(const bfs::path& filename); + + std::string + to_string() const; + + settings(const settings& other) = default; + + settings(settings&& rhs) = default; + + settings& + operator=(const settings& other) = default; + + settings& + operator=(settings&& rhs) = default; + + ~settings() = default; + + std::string + progname() const; + + void + progname(const std::string& progname); + + bool + daemonize() const; + + void + daemonize(bool daemonize); + + bool + use_syslog() const; + + void + use_syslog(bool use_syslog); + + bool + use_console() const; + + void + use_console(bool use_console); + + bfs::path + log_file() const; + + void + log_file(const bfs::path& log_file); + + uint32_t + log_file_max_size() const; + + void + log_file_max_size(uint32_t log_file_max_size); + + bool + dry_run() const; + + void + dry_run(bool dry_run); + + uint32_t + dry_run_duration() const; + + void + dry_run_duration(uint32_t dry_run_duration); + + bfs::path + global_socket() const; + + void + global_socket(const bfs::path& global_socket); + + bfs::path + control_socket() const; + + void + control_socket(const bfs::path& control_socket); + + std::string + bind_address() const; + + void + bind_address(const std::string& bind_address); + + in_port_t + remote_port() const; + + void + remote_port(in_port_t remote_port); + + bfs::path + pidfile() const; + + void + pidfile(const bfs::path& pidfile); + + uint32_t + workers_in_pool() const; + + void + workers_in_pool(uint32_t workers_in_pool); + + bfs::path + staging_directory() const; + + void + staging_directory(const bfs::path& staging_directory); + + uint32_t + backlog_size() const; + + void + backlog_size(uint32_t backlog_size); + + bfs::path + config_file() const; + + void + config_file(const bfs::path& config_file); + + std::list + default_namespaces() const; + + void + default_namespaces(const std::list& default_namespaces); + + std::string m_progname; + bool m_daemonize; + bool m_use_syslog; + bool m_use_console; + bfs::path m_log_file; + uint32_t m_log_file_max_size; + bool m_dry_run; + uint32_t m_dry_run_duration; + bfs::path m_global_socket; + bfs::path m_control_socket; + std::string m_bind_address; + in_port_t m_remote_port; + bfs::path m_daemon_pidfile; + uint32_t m_workers_in_pool; + bfs::path m_staging_directory; + uint32_t m_backlog_size; + bfs::path m_config_file; std::list m_default_namespaces; }; diff --git a/src/main.cpp b/src/main.cpp index b8e30f0..ca0478e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -63,7 +63,7 @@ main(int argc, char* argv[]) { ->default_value(false) ->notifier( [&](const bool& flag_value) { - cfg.daemonize() = !flag_value; + cfg.daemonize(!flag_value); }), "foreground operation") @@ -74,8 +74,8 @@ main(int argc, char* argv[]) { ->implicit_value(100) ->notifier( [&](const uint32_t& duration_value) { - cfg.dry_run() = true; - cfg.dry_run_duration() = duration_value; + cfg.dry_run(true); + cfg.dry_run_duration(duration_value); }), "don't actually execute tasks, but wait N microseconds per task if " "an argument is provided") @@ -87,7 +87,7 @@ main(int argc, char* argv[]) { ->zero_tokens() ->notifier( [&](const std::string&) { - cfg.use_console() = true; + cfg.use_console(true); }), "override any logging options defined in configuration files and " "send all daemon output to the console" diff --git a/src/urd.cpp b/src/urd.cpp index 0ee133d..eae3896 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -1397,6 +1397,28 @@ void urd::load_default_namespaces() { } } +void +urd::check_configuration() { + + // check that the staging directory exists and that we can write to it + if(!bfs::exists(m_settings->staging_directory())) { + LOGGER_ERROR("Staging directory {} does not exist", + m_settings->staging_directory()); + teardown_and_exit(); + } + + auto s = bfs::status(m_settings->staging_directory()); + + auto expected_perms = bfs::perms::owner_read | + bfs::perms::owner_write; + + if((s.permissions() & expected_perms) != expected_perms) { + LOGGER_ERROR("Unable to read from/write to staging directory {}", + m_settings->staging_directory()); + teardown_and_exit(); + } +} + void urd::print_greeting() { const char greeting[] = "Starting {} daemon (pid {})"; const auto gsep = std::string(sizeof(greeting) - 4 + @@ -1426,6 +1448,7 @@ void urd::print_configuration() { LOGGER_INFO(" - pidfile: {}", m_settings->pidfile()); LOGGER_INFO(" - control socket: {}", m_settings->control_socket()); LOGGER_INFO(" - global socket: {}", m_settings->global_socket()); + LOGGER_INFO(" - staging directory: {}", m_settings->staging_directory()); LOGGER_INFO(" - port for remote requests: {}", m_settings->remote_port()); LOGGER_INFO(" - workers: {}", m_settings->workers_in_pool()); LOGGER_INFO(""); @@ -1495,6 +1518,9 @@ int urd::run() { // initialize logging facilities init_logger(); + // validate settings + check_configuration(); + // daemonize if needed if(m_settings->daemonize() && daemonize() != 0) { /* parent clean ups and exits, child continues */ @@ -1572,6 +1598,12 @@ void urd::teardown() { } } +void +urd::teardown_and_exit() { + teardown(); + exit(EXIT_FAILURE); +} + void urd::shutdown() { if(m_ipc_endpoint) { m_ipc_endpoint->stop(); diff --git a/src/urd.hpp b/src/urd.hpp index 1205a5e..48ad449 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -89,6 +89,7 @@ public: int run(); void shutdown(); void teardown(); + void teardown_and_exit(); private: int daemonize(); @@ -101,6 +102,7 @@ private: void load_backend_plugins(); void load_transfer_plugins(); void load_default_namespaces(); + void check_configuration(); void print_greeting(); void print_configuration(); void print_farewell(); diff --git a/tests/fake-daemon.cpp b/tests/fake-daemon.cpp index 2517804..7c82769 100644 --- a/tests/fake-daemon.cpp +++ b/tests/fake-daemon.cpp @@ -34,7 +34,7 @@ #include "nornsctl.h" #include "fake-daemon.hpp" -norns::config::settings default_cfg( +const norns::config::settings fake_daemon::default_cfg( "test_urd", /* progname */ false, /* daemonize */ false, /* use syslog */ @@ -49,6 +49,7 @@ norns::config::settings default_cfg( 42002, /* remote port */ "./test_urd.pid", /* daemon_pidfile */ 2, /* api workers */ + "./tmp/", /* staging directory */ 128, "./", {} @@ -70,80 +71,149 @@ fake_daemon::~fake_daemon() { void fake_daemon::configure(const bfs::path& config_file, const fake_daemon_cfg& override_cfg) { +#if 1 + // most settings are taken from the default configuration for tests + m_config = default_cfg; + + // some settings are taken from the configuration file auto-generated + // for this test (mostly those path-related) + norns::config::settings file_config; + file_config.load_from_file(config_file); + + m_config.log_file(file_config.log_file()); + m_config.global_socket(file_config.global_socket()); + m_config.control_socket(file_config.control_socket()); + m_config.pidfile(file_config.pidfile()); + m_config.staging_directory(file_config.staging_directory()); + m_config.config_file(config_file); + + // we allow some settings to be overriden + m_config.dry_run(override_cfg.m_dry_run); + m_config.dry_run_duration(override_cfg.m_dry_run_duration); + + // for now default_namespaces is empty + m_config.default_namespaces().clear(); + +#endif + +#if 0 m_config.load_from_file(config_file); - m_config.progname() = default_cfg.progname(); - m_config.daemonize() = default_cfg.daemonize(); - m_config.use_syslog() = default_cfg.use_syslog(); - m_config.use_console() = default_cfg.use_console(); - m_config.dry_run() = override_cfg.m_dry_run; - m_config.dry_run_duration() = override_cfg.m_dry_run_duration; - m_config.bind_address() = default_cfg.bind_address(); - m_config.remote_port() = default_cfg.remote_port(); - m_config.workers_in_pool() = default_cfg.workers_in_pool(); - m_config.config_file() = config_file; + m_config.progname(default_cfg.progname()); + m_config.daemonize(default_cfg.daemonize()); + m_config.use_syslog(default_cfg.use_syslog()); + m_config.use_console(default_cfg.use_console()); + m_config.dry_run(override_cfg.m_dry_run); + m_config.dry_run_duration(override_cfg.m_dry_run_duration); + m_config.bind_address(default_cfg.bind_address()); + m_config.remote_port(default_cfg.remote_port()); + m_config.workers_in_pool(default_cfg.workers_in_pool()); + m_config.staging_directory(default_cfg.staging_directory()); + m_config.config_file(config_file); m_config.default_namespaces().clear(); +#endif } void fake_daemon::configure(const bfs::path& config_file, const std::string& alias) { +#if 1 + // most settings are taken from the configuration file auto-generated + // for this test (mostly those path-related) m_config.load_from_file(config_file); - m_config.progname() = default_cfg.progname() + alias; - m_config.daemonize() = default_cfg.daemonize(); - m_config.use_syslog() = default_cfg.use_syslog(); - m_config.use_console() = default_cfg.use_console(); - m_config.dry_run() = default_cfg.dry_run(); - m_config.dry_run_duration() = default_cfg.dry_run_duration(); - m_config.workers_in_pool() = default_cfg.workers_in_pool(); - m_config.config_file() = config_file; + m_config.progname(default_cfg.progname() + alias); + m_config.daemonize(default_cfg.daemonize()); + m_config.use_console(default_cfg.use_console()); + m_config.config_file(config_file); +#endif + +#if 0 + m_config.load_from_file(config_file); + + m_config.progname(default_cfg.progname() + alias); + m_config.daemonize(default_cfg.daemonize()); + m_config.use_syslog(default_cfg.use_syslog()); + m_config.use_console(default_cfg.use_console()); + m_config.dry_run(default_cfg.dry_run()); + m_config.dry_run_duration(default_cfg.dry_run_duration()); + m_config.bind_address(default_cfg.bind_address()); + m_config.remote_port(default_cfg.remote_port()); + m_config.workers_in_pool(default_cfg.workers_in_pool()); + m_config.staging_directory(default_cfg.staging_directory()); + m_config.config_file(config_file); m_config.default_namespaces().clear(); +#endif } +extern "C" void __gcov_flush (void); + void fake_daemon::run() { m_running = true; - m_pid = fork(); - if(m_pid != 0) { + switch(m_pid) { + /* child code */ + case 0: + { + // disable any signal handlers set by Catch2 + // so that we can receive control signals from the parent process + for(auto signum : { SIGINT, SIGILL, SIGFPE, + SIGSEGV, SIGTERM, SIGABRT }) { + + struct sigaction sa; + sa.sa_handler = SIG_DFL; + + if(::sigaction(signum, &sa, NULL) != 0) { + throw std::runtime_error("Failed to reset Catch2 signal handler"); + } + } + + m_daemon.configure(m_config); + m_daemon.run(); + m_daemon.teardown(); #ifdef DEBUG_OUTPUT - std::cerr << "[" << getpid() << "] daemon process spawned (" << m_pid << ")\n"; + std::cerr << "[" << getpid() << "] exitting...\n"; #endif - int rv; - int retries = 20; + __gcov_flush(); + exit(0); + } - do { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - rv = nornsctl_send_command(NORNSCTL_CMD_PING, NULL); - } while(rv != NORNS_SUCCESS && --retries != 0); + /* error code */ + case -1: + throw std::runtime_error("Failed to spawn test daemon"); - if(retries == 0) { - // the daemon may be running even if we don't receive a reply, - // try to stop it to avoid leaving a dangling process - stop(); - throw std::runtime_error("Failed to ping test daemon"); - } + /* parent code */ + default: + { #ifdef DEBUG_OUTPUT - std::cerr << "[" << getpid() << "] daemon process ready\n"; + std::cerr << "[" << getpid() << "] daemon process spawned (" << m_pid << ")\n"; #endif - return; - } + int rv; + int retries = 20; + + do { + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + rv = nornsctl_send_command(NORNSCTL_CMD_PING, NULL); + } while(rv != NORNS_SUCCESS && --retries != 0); - m_daemon.configure(m_config); - m_daemon.run(); - m_daemon.teardown(); + if(retries == 0) { + // the daemon may be running even if we don't receive a reply, + // try to stop it to avoid leaving a dangling process + stop(); + throw std::runtime_error("Failed to ping test daemon"); + } #ifdef DEBUG_OUTPUT - std::cerr << "[" << getpid() << "] exitting...\n"; + std::cerr << "[" << getpid() << "] daemon process ready\n"; #endif - - exit(0); + } + } } int fake_daemon::stop() { diff --git a/tests/fake-daemon.hpp b/tests/fake-daemon.hpp index e6cb5ea..2194009 100644 --- a/tests/fake-daemon.hpp +++ b/tests/fake-daemon.hpp @@ -46,6 +46,8 @@ struct fake_daemon_cfg { struct fake_daemon { + static const norns::config::settings default_cfg; + fake_daemon(); ~fake_daemon(); void configure(const bfs::path& config_file, const fake_daemon_cfg& override_cfg); diff --git a/tests/test-env.cpp b/tests/test-env.cpp index c711243..452fb89 100644 --- a/tests/test-env.cpp +++ b/tests/test-env.cpp @@ -69,6 +69,16 @@ create_config_file(const bfs::path& basedir, REQUIRE(res == true); } + const bfs::path stagingdir = cfgdir / + fake_daemon::default_cfg.m_staging_directory; + + if(!bfs::exists(stagingdir)) { + bool res = bfs::create_directory(stagingdir); + REQUIRE(res == true); + } + + const auto stdir = bfs::canonical(stagingdir); + const std::string name = "test.conf" + alias; const bfs::path config_file = cfgdir / name; @@ -77,6 +87,10 @@ create_config_file(const bfs::path& basedir, boost::regex("@localstatedir@"), cfgdir.string()); + outstr = boost::regex_replace(outstr, + boost::regex("(staging_directory:)\\s*?\".*?\"(,?)$"), + "\\1 \"" + stdir.string() + "\"\\2"); + for(const auto& r : reps) { outstr = boost::regex_replace(outstr, boost::regex(r.first), r.second); } -- GitLab From 8ff5a08c7e037d1830e2b079065480338376c509 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 21:38:26 +0100 Subject: [PATCH 37/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 10 ++++++++-- configure.ac | 3 +++ gencov.sh | 20 ++++++++++++++++++++ src/api/signal-listener.hpp | 9 +++++---- 4 files changed, 36 insertions(+), 6 deletions(-) create mode 100755 gencov.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f002d59..775f40a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,6 +158,8 @@ test:ubuntu:latest: libcap2-bin valgrind cmake + gcov + lcov - pushd . && git clone https://github.com/ofiwg/libfabric.git && @@ -198,6 +200,9 @@ test:ubuntu:latest: - mkdir build && cd build - ../configure --enable-tests + CFLAGS="-O0 --coverage" + CXXFLAGS="-O0 --coverage" + LDFLAGS="--coverage" # CFLAGS="-fsanitize=address" # CXXFLAGS="-fsanitize=address" # LDFLAGS="-fsanitize=address" @@ -237,8 +242,9 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" after_script: + - (cd build && ../gencov.sh) - pwd - - if [[ -e tests.log ]]; + - if [[ -e build/tests.log ]]; then - cat $(tail -1 tests.log)/config/urd.log; + cat $(tail -1 build/tests.log)/config/urd.log; fi diff --git a/configure.ac b/configure.ac index daa129f..cc8858a 100644 --- a/configure.ac +++ b/configure.ac @@ -40,6 +40,9 @@ AC_CONFIG_HEADERS([config.h]) AM_INIT_AUTOMAKE([1.9 foreign subdir-objects]) +AC_LANG([C]) +AC_LANG([C++]) + # Checks for programs. AC_PROG_AWK AC_PROG_SED diff --git a/gencov.sh b/gencov.sh new file mode 100755 index 0000000..b76de58 --- /dev/null +++ b/gencov.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +GIT_ROOTDIR=`git rev-parse --show-toplevel` + +/usr/local/bin/lcov \ + `find ${GIT_ROOTDIR} -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ + --capture \ + --output-file gcov.info + +/usr/local/bin/lcov \ + --remove gcov.info \ + '/usr/include/*' \ + '/usr/local/include/*' \ + '*/externals/*' \ + '*/spdlog/*' \ + '*/build*/*' \ + -o norns.info \ + 2>/dev/null + +genhtml -o html/coverage norns.info 2>/dev/null diff --git a/src/api/signal-listener.hpp b/src/api/signal-listener.hpp index 245a994..688f574 100644 --- a/src/api/signal-listener.hpp +++ b/src/api/signal-listener.hpp @@ -37,16 +37,17 @@ namespace ba = boost::asio; namespace norns { namespace api { +template +void do_for(F /*f*/) { + // Parameter pack is empty. +} + template void do_for(F f, First first, Rest... rest) { f(first); do_for(f, rest...); } -template -void do_for(F /*f*/) { - // Parameter pack is empty. -} struct signal_listener { -- GitLab From e8e32bcc7f226e582720bcc12f27946e8769722c Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 21:50:18 +0100 Subject: [PATCH 38/51] Improve coverage reporting --- gencov.sh | 10 +++++++--- tests/fake-daemon.cpp | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/gencov.sh b/gencov.sh index b76de58..1bea0ac 100755 --- a/gencov.sh +++ b/gencov.sh @@ -5,7 +5,8 @@ GIT_ROOTDIR=`git rev-parse --show-toplevel` /usr/local/bin/lcov \ `find ${GIT_ROOTDIR} -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ --capture \ - --output-file gcov.info + --output-file gcov.info \ + 2&>1 /dev/null /usr/local/bin/lcov \ --remove gcov.info \ @@ -14,7 +15,10 @@ GIT_ROOTDIR=`git rev-parse --show-toplevel` '*/externals/*' \ '*/spdlog/*' \ '*/build*/*' \ + '*/tests/*' \ -o norns.info \ - 2>/dev/null + 2&>1 /dev/null -genhtml -o html/coverage norns.info 2>/dev/null +/usr/local/bin/lcov -l norns.info + +#genhtml -o html/coverage norns.info 2>/dev/null diff --git a/tests/fake-daemon.cpp b/tests/fake-daemon.cpp index 7c82769..ed83c7a 100644 --- a/tests/fake-daemon.cpp +++ b/tests/fake-daemon.cpp @@ -146,7 +146,7 @@ void fake_daemon::configure(const bfs::path& config_file, #endif } -extern "C" void __gcov_flush (void); +//extern "C" void __gcov_flush (void); void fake_daemon::run() { @@ -178,7 +178,7 @@ void fake_daemon::run() { std::cerr << "[" << getpid() << "] exitting...\n"; #endif - __gcov_flush(); +// __gcov_flush(); exit(0); } -- GitLab From 04fc0021a9bc8c8c25fd2fc885a9130e4505d22c Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 22:09:12 +0100 Subject: [PATCH 39/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 775f40a..b6fc088 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,7 +158,6 @@ test:ubuntu:latest: libcap2-bin valgrind cmake - gcov lcov - pushd . && -- GitLab From 8f8849fa58b9fe434024a20e946827f378d1c39e Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 22:38:08 +0100 Subject: [PATCH 40/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 ++ gencov.sh | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b6fc088..26fb290 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -241,6 +241,8 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" after_script: + - pwd + - ls -lR - (cd build && ../gencov.sh) - pwd - if [[ -e build/tests.log ]]; diff --git a/gencov.sh b/gencov.sh index 1bea0ac..bedae7a 100755 --- a/gencov.sh +++ b/gencov.sh @@ -1,14 +1,15 @@ #!/bin/bash GIT_ROOTDIR=`git rev-parse --show-toplevel` +LCOV=`which lcov` -/usr/local/bin/lcov \ +${LCOV} \ `find ${GIT_ROOTDIR} -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ --capture \ --output-file gcov.info \ 2&>1 /dev/null -/usr/local/bin/lcov \ +${LCOV} \ --remove gcov.info \ '/usr/include/*' \ '/usr/local/include/*' \ @@ -19,6 +20,6 @@ GIT_ROOTDIR=`git rev-parse --show-toplevel` -o norns.info \ 2&>1 /dev/null -/usr/local/bin/lcov -l norns.info +${LCOV} -l norns.info #genhtml -o html/coverage norns.info 2>/dev/null -- GitLab From 92d8bb526d557ed4262449c77aeb8133870cfbab Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 23:08:06 +0100 Subject: [PATCH 41/51] Update .gitlab-ci.yml --- gencov.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gencov.sh b/gencov.sh index bedae7a..bc3bd52 100755 --- a/gencov.sh +++ b/gencov.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x GIT_ROOTDIR=`git rev-parse --show-toplevel` LCOV=`which lcov` -- GitLab From 087ff78ec7eb2a61a8379826d16ab64f6a3d1667 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 6 Mar 2019 23:37:29 +0100 Subject: [PATCH 42/51] Update .gitlab-ci.yml --- gencov.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gencov.sh b/gencov.sh index bc3bd52..e61c0ac 100755 --- a/gencov.sh +++ b/gencov.sh @@ -6,8 +6,7 @@ LCOV=`which lcov` ${LCOV} \ `find ${GIT_ROOTDIR} -name "*.gcda" 2>/dev/null | xargs -I{} dirname {} | uniq | xargs -I {} echo -n " --directory "{}` \ --capture \ - --output-file gcov.info \ - 2&>1 /dev/null + --output-file gcov.info ${LCOV} \ --remove gcov.info \ @@ -17,8 +16,7 @@ ${LCOV} \ '*/spdlog/*' \ '*/build*/*' \ '*/tests/*' \ - -o norns.info \ - 2&>1 /dev/null + -o norns.info ${LCOV} -l norns.info -- GitLab From b3a775726d00f1241a7a123afff0d81f906aebd8 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 00:02:44 +0100 Subject: [PATCH 43/51] Update .gitlab-ci.yml --- gencov.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gencov.sh b/gencov.sh index e61c0ac..afef42b 100755 --- a/gencov.sh +++ b/gencov.sh @@ -14,7 +14,7 @@ ${LCOV} \ '/usr/local/include/*' \ '*/externals/*' \ '*/spdlog/*' \ - '*/build*/*' \ + '*/build/*' \ '*/tests/*' \ -o norns.info -- GitLab From 8dd820c86a1c1d75a9d5dd4e37ec16e4d19f90e0 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 06:58:27 +0000 Subject: [PATCH 44/51] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f6a60b1..cdd6aeb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Norns [![pipeline status](https://storage.bsc.es/gitlab/hpc/norns/badges/master/pipeline.svg)](https://storage.bsc.es/gitlab/hpc/norns/commits/master) +[![coverage report](https://storage.bsc.es/gitlab/hpc/norns/badges/21-add-support-for-remote-transfers-2/coverage.svg)](https://storage.bsc.es/gitlab/hpc/norns/commits/21-add-support-for-remote-transfers-2) Norns is an open-source data scheduling service that orchestrates asynchronous data transfers between different storage backends in an HPC cluster. Through -- GitLab From 46bcd5d3ba2e11691fb9419346225f057df64039 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 08:20:10 +0100 Subject: [PATCH 45/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 26 ++++++++++++++++++-------- gencov.sh | 2 +- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 26fb290..52293b1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -202,16 +202,11 @@ test:ubuntu:latest: CFLAGS="-O0 --coverage" CXXFLAGS="-O0 --coverage" LDFLAGS="--coverage" -# CFLAGS="-fsanitize=address" -# CXXFLAGS="-fsanitize=address" -# LDFLAGS="-fsanitize=address" -# CPPFLAGS="-D__LOGGER_ENABLE_DEBUG__" - make -j$(nproc) - cd tests - make -j$(nproc) core - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as - make -j$(nproc) api -# - NORNS_DEBUG_OUTPUT_TO_STDERR=1 NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" @@ -241,11 +236,26 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" after_script: - - pwd - - ls -lR - (cd build && ../gencov.sh) - - pwd + - genhtml -o build/html/coverage norns.info - if [[ -e build/tests.log ]]; then cat $(tail -1 build/tests.log)/config/urd.log; fi + + artifacts: + paths: + - build/html/coverage/ + +pages: + stage: deploy + dependencies: + - test:ubuntu:latest + script: + - mv coverage/ public/ + artifacts: + paths: + - public + expire_in: 30 days + only: + - master diff --git a/gencov.sh b/gencov.sh index afef42b..758750b 100755 --- a/gencov.sh +++ b/gencov.sh @@ -20,4 +20,4 @@ ${LCOV} \ ${LCOV} -l norns.info -#genhtml -o html/coverage norns.info 2>/dev/null +#genhtml -o buiild/html/coverage norns.info -- GitLab From c3cfdf5da711a8c7e10f8c45eabe14b1acad2bbb Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 08:21:02 +0100 Subject: [PATCH 46/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 52293b1..adfb01f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,6 +9,7 @@ variables: stages: - build - test + - deploy # Install dependencies for GCC builds before_script: -- GitLab From 8f80746faee02e99e76cd64a212da137fc56b39b Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 09:03:21 +0100 Subject: [PATCH 47/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adfb01f..b8040de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -238,7 +238,7 @@ test:ubuntu:latest: after_script: - (cd build && ../gencov.sh) - - genhtml -o build/html/coverage norns.info + - genhtml -o build/html/coverage build/norns.info - if [[ -e build/tests.log ]]; then cat $(tail -1 build/tests.log)/config/urd.log; -- GitLab From 7b87f504c542af3cc668fc9ceec5846c81683f0a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 10:29:55 +0100 Subject: [PATCH 48/51] s/network_endpoint/network_service/ --- .../local-path-to-remote-resource.cpp | 24 ++++++------ .../local-path-to-remote-resource.hpp | 4 +- .../transferors/memory-to-remote-resource.cpp | 12 +++--- .../transferors/memory-to-remote-resource.hpp | 4 +- .../remote-resource-to-local-path.cpp | 28 +++++++------- .../remote-resource-to-local-path.hpp | 4 +- src/urd.cpp | 38 +++++++++---------- src/urd.hpp | 2 +- 8 files changed, 58 insertions(+), 58 deletions(-) diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 9db07c5..6edf23e 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -128,8 +128,8 @@ namespace norns { namespace io { local_path_to_remote_resource_transferor::local_path_to_remote_resource_transferor( - std::shared_ptr network_endpoint) : - m_network_endpoint(network_endpoint) { } + std::shared_ptr network_service) : + m_network_service(network_service) { } bool local_path_to_remote_resource_transferor::validate( @@ -197,7 +197,7 @@ local_path_to_remote_resource_transferor::transfer( LOGGER_DEBUG("[{}] start_transfer: {} -> {}", task_info->id(), d_src.canonical_path(), d_dst.to_string()); - hermes::endpoint endp = m_network_endpoint->lookup(d_dst.address()); + hermes::endpoint endp = m_network_service->lookup(d_dst.address()); try { hermes::mapped_buffer input_buffer(input_path.string(), @@ -214,13 +214,13 @@ local_path_to_remote_resource_transferor::transfer( }; auto local_buffers = - m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); + m_network_service->expose(bufvec, hermes::access_mode::read_only); auto resp = - m_network_endpoint->post( + m_network_service->post( endp, rpc::push_resource::input{ - m_network_endpoint->self_address(), + m_network_service->self_address(), d_src.parent()->nsid(), d_dst.parent()->nsid(), // XXX this resource_type should not be needed, but we @@ -351,7 +351,7 @@ local_path_to_remote_resource_transferor::accept_transfer( }; hermes::exposed_memory local_buffers = - m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); + m_network_service->expose(bufseq, hermes::access_mode::write_only); LOGGER_DEBUG("pulling remote data into {}", tempfile->path()); @@ -409,7 +409,7 @@ local_path_to_remote_resource_transferor::accept_transfer( respond: if(req.requires_response()) { - m_network_endpoint->respond( + m_network_service->respond( std::move(req), out); } }; @@ -419,10 +419,10 @@ respond: // std::chrono::steady_clock::now().time_since_epoch()) // .count()); - m_network_endpoint->async_pull(remote_buffers, - local_buffers, - std::move(req), - completion_callback); + m_network_service->async_pull(remote_buffers, + local_buffers, + std::move(req), + completion_callback); return ec; } diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index a4144cc..d885470 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -56,7 +56,7 @@ namespace io { struct local_path_to_remote_resource_transferor : public transferor { local_path_to_remote_resource_transferor( - std::shared_ptr network_endpoint); + std::shared_ptr network_service); bool validate(const std::shared_ptr& src_info, @@ -81,7 +81,7 @@ struct local_path_to_remote_resource_transferor : public transferor { to_string() const override final; private: - std::shared_ptr m_network_endpoint; + std::shared_ptr m_network_service; }; diff --git a/src/io/transferors/memory-to-remote-resource.cpp b/src/io/transferors/memory-to-remote-resource.cpp index fee990b..7102e26 100644 --- a/src/io/transferors/memory-to-remote-resource.cpp +++ b/src/io/transferors/memory-to-remote-resource.cpp @@ -83,8 +83,8 @@ namespace io { memory_region_to_remote_resource_transferor:: memory_region_to_remote_resource_transferor( - std::shared_ptr network_endpoint) : - m_network_endpoint(network_endpoint) {} + std::shared_ptr network_service) : + m_network_service(network_service) {} bool memory_region_to_remote_resource_transferor::validate( @@ -175,14 +175,14 @@ memory_region_to_remote_resource_transferor::transfer( try { - hermes::endpoint endp = m_network_endpoint->lookup(d_dst.address()); + hermes::endpoint endp = m_network_service->lookup(d_dst.address()); std::vector bufvec{ hermes::mutable_buffer{output_buffer->data(), output_buffer->size()} }; auto local_buffers = - m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); + m_network_service->expose(bufvec, hermes::access_mode::read_only); // LOGGER_CRITICAL("push_resource RPC posted: {}", // std::chrono::duration_cast( @@ -190,10 +190,10 @@ memory_region_to_remote_resource_transferor::transfer( // .count()); auto resp = - m_network_endpoint->post( + m_network_service->post( endp, rpc::push_resource::input{ - m_network_endpoint->self_address(), + m_network_service->self_address(), d_src.parent()->nsid(), d_dst.parent()->nsid(), // XXX this resource_type should not be needed, but we diff --git a/src/io/transferors/memory-to-remote-resource.hpp b/src/io/transferors/memory-to-remote-resource.hpp index ecc9873..463c66c 100644 --- a/src/io/transferors/memory-to-remote-resource.hpp +++ b/src/io/transferors/memory-to-remote-resource.hpp @@ -49,7 +49,7 @@ namespace io { struct memory_region_to_remote_resource_transferor : public transferor { memory_region_to_remote_resource_transferor( - std::shared_ptr network_endpoint); + std::shared_ptr network_service); bool validate(const std::shared_ptr& src_info, @@ -74,7 +74,7 @@ struct memory_region_to_remote_resource_transferor : public transferor { to_string() const override final; private: - std::shared_ptr m_network_endpoint; + std::shared_ptr m_network_service; }; } // namespace io diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 0142bf3..228680c 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -121,8 +121,8 @@ namespace norns { namespace io { remote_resource_to_local_path_transferor::remote_resource_to_local_path_transferor( - std::shared_ptr network_endpoint) : - m_network_endpoint(network_endpoint) { } + std::shared_ptr network_service) : + m_network_service(network_service) { } bool remote_resource_to_local_path_transferor::validate( @@ -155,13 +155,13 @@ remote_resource_to_local_path_transferor::transfer( LOGGER_DEBUG("[{}] request_transfer: {} -> {}", task_info->id(), d_src.to_string(), d_dst.canonical_path()); - hermes::endpoint endp = m_network_endpoint->lookup(d_src.address()); + hermes::endpoint endp = m_network_service->lookup(d_src.address()); auto resp = - m_network_endpoint->post( + m_network_service->post( endp, rpc::stat_resource::input{ - m_network_endpoint->self_address(), + m_network_service->self_address(), d_src.parent()->nsid(), static_cast(data::resource_type::local_posix_path), d_src.name() @@ -216,10 +216,10 @@ remote_resource_to_local_path_transferor::transfer( }; hermes::exposed_memory local_buffers = - m_network_endpoint->expose(bufseq, hermes::access_mode::write_only); + m_network_service->expose(bufseq, hermes::access_mode::write_only); auto resp2 = - m_network_endpoint->post( + m_network_service->post( endp, rpc::pull_resource::input{ d_src.parent()->nsid(), @@ -231,7 +231,7 @@ remote_resource_to_local_path_transferor::transfer( // XXX this information locally from the resource id static_cast( data::resource_type::local_posix_path), - m_network_endpoint->self_address(), + m_network_service->self_address(), d_dst.parent()->nsid(), d_dst.name(), local_buffers @@ -339,7 +339,7 @@ remote_resource_to_local_path_transferor::accept_transfer( }; auto local_buffers = - m_network_endpoint->expose(bufvec, hermes::access_mode::read_only); + m_network_service->expose(bufvec, hermes::access_mode::read_only); // retrieve remote buffers descriptor hermes::exposed_memory remote_buffers = d_dst.buffers(); @@ -376,16 +376,16 @@ remote_resource_to_local_path_transferor::accept_transfer( LOGGER_DEBUG("Push completed"); if(req.requires_response()) { - m_network_endpoint->respond( + m_network_service->respond( std::move(req), out); } }; - m_network_endpoint->async_push(local_buffers, - remote_buffers, - std::move(req), - completion_callback); + m_network_service->async_push(local_buffers, + remote_buffers, + std::move(req), + completion_callback); return ec; } diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index 1042594..d841850 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -56,7 +56,7 @@ namespace io { struct remote_resource_to_local_path_transferor : public transferor { remote_resource_to_local_path_transferor( - std::shared_ptr network_endpoint); + std::shared_ptr network_service); bool validate(const std::shared_ptr& src_info, @@ -81,7 +81,7 @@ struct remote_resource_to_local_path_transferor : public transferor { to_string() const override final; private: - std::shared_ptr m_network_endpoint; + std::shared_ptr m_network_service; }; } // namespace io diff --git a/src/urd.cpp b/src/urd.cpp index eae3896..94b7766 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -666,7 +666,7 @@ response_ptr urd::unknown_request_handler(const request_ptr /*base_request*/) { } // N.B. This function is called by the progress thread internal to -// m_network_endpoint rather than by the main execution thread +// m_network_service rather than by the main execution thread void urd::push_resource_handler(hermes::request&& req) { @@ -709,7 +709,7 @@ urd::push_resource_handler(hermes::request&& req) { if(m_is_paused) { rv = urd_error::accept_paused; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), 0, @@ -737,7 +737,7 @@ urd::push_resource_handler(hermes::request&& req) { if(!dst_backend) { rv = urd_error::no_such_namespace; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), 0, @@ -765,7 +765,7 @@ urd::push_resource_handler(hermes::request&& req) { if(t->info()->status() == io::task_status::finished_with_error) { rv = t->info()->task_error(); LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(t->info()->task_error()), static_cast(t->info()->sys_error().value()), @@ -818,7 +818,7 @@ urd::pull_resource_handler(hermes::request&& req) { if(m_is_paused) { rv = urd_error::accept_paused; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), 0, @@ -842,7 +842,7 @@ urd::pull_resource_handler(hermes::request&& req) { if(!src_backend) { rv = urd_error::no_such_namespace; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(rv), 0, @@ -873,7 +873,7 @@ urd::pull_resource_handler(hermes::request&& req) { if(tsk->info()->status() == io::task_status::finished_with_error) { rv = tsk->info()->task_error(); LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(io::task_status::finished_with_error), static_cast(tsk->info()->task_error()), static_cast(tsk->info()->sys_error().value()), @@ -902,7 +902,7 @@ urd::stat_resource_handler(hermes::request&& req) { if(!dst_backend) { rv = urd_error::no_such_namespace; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(rv), false, //XXX ENOENT should not be required: @@ -937,7 +937,7 @@ urd::stat_resource_handler(hermes::request&& req) { rv = urd_error::not_supported; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(rv), //XXX EOPNOTSUPP should not be required: // the transfer() interface should be @@ -957,7 +957,7 @@ urd::stat_resource_handler(hermes::request&& req) { rv = urd_error::no_such_resource; LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(rv), ec.value(), false, @@ -966,7 +966,7 @@ urd::stat_resource_handler(hermes::request&& req) { } LOGGER_INFO("IOTASK_RECEIVE() = {}", utils::to_string(rv)); - m_network_endpoint->respond(std::move(req), + m_network_service->respond(std::move(req), static_cast(urd_error::success), 0, rsrc->is_collection(), @@ -1105,7 +1105,7 @@ void urd::init_event_handlers() { m_settings->bind_address() + ":" + std::to_string(m_settings->remote_port()); - m_network_endpoint = + m_network_service = std::make_shared( hermes::transport::ofi_tcp, bind_address, @@ -1194,15 +1194,15 @@ void urd::init_event_handlers() { std::bind(&urd::unknown_request_handler, this, std::placeholders::_1)); /* remote event handlers */ - m_network_endpoint->register_handler( + m_network_service->register_handler( std::bind(&urd::push_resource_handler, this, std::placeholders::_1)); - m_network_endpoint->register_handler( + m_network_service->register_handler( std::bind(&urd::pull_resource_handler, this, std::placeholders::_1)); - m_network_endpoint->register_handler( + m_network_service->register_handler( std::bind(&urd::stat_resource_handler, this, std::placeholders::_1)); @@ -1355,7 +1355,7 @@ void urd::load_transfer_plugins() { data::resource_type::memory_region, data::resource_type::remote_resource, std::make_shared( - m_network_endpoint)); + m_network_service)); // local path -> local path load_plugin(data::resource_type::local_posix_path, @@ -1371,13 +1371,13 @@ void urd::load_transfer_plugins() { load_plugin(data::resource_type::local_posix_path, data::resource_type::remote_resource, std::make_shared( - m_network_endpoint)); + m_network_service)); // remote resource -> local path load_plugin(data::resource_type::remote_resource, data::resource_type::local_posix_path, std::make_shared( - m_network_endpoint)); + m_network_service)); } void urd::load_default_namespaces() { @@ -1543,7 +1543,7 @@ int urd::run() { // start the listener for remote transfers // N.B. This call returns immediately - m_network_endpoint->run(); + m_network_service->run(); LOGGER_INFO(""); LOGGER_INFO("[[ Start up successful, awaiting requests... ]]"); diff --git a/src/urd.hpp b/src/urd.hpp index 48ad449..5263e5f 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -149,7 +149,7 @@ private: std::unique_ptr m_ipc_endpoint; - std::shared_ptr m_network_endpoint; + std::shared_ptr m_network_service; std::unique_ptr m_namespace_mgr; mutable boost::shared_mutex m_namespace_mgr_mutex; -- GitLab From c98577050283ceb6759eca7dc6622016fc8cf529 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 10:35:55 +0100 Subject: [PATCH 49/51] s/ipc_endpoint/ipc_service/ --- src/urd.cpp | 50 +++++++++++++++++++++++++------------------------- src/urd.hpp | 2 +- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/urd.cpp b/src/urd.cpp index 94b7766..79164e0 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -1036,7 +1036,7 @@ void urd::init_event_handlers() { // create (but not start) the API listener // and register handlers for each request type try { - m_ipc_endpoint = std::make_unique(); + m_ipc_service = std::make_unique(); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create the event listener. This should " @@ -1065,7 +1065,7 @@ void urd::init_event_handlers() { ::umask(S_IXUSR | S_IRWXG | S_IRWXO); // u=rw-, g=---, o=--- try { - m_ipc_endpoint->register_endpoint(m_settings->control_socket()); + m_ipc_service->register_endpoint(m_settings->control_socket()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create control API socket: {}", e.what()); @@ -1088,7 +1088,7 @@ void urd::init_event_handlers() { ::umask(S_IXUSR | S_IXGRP | S_IXOTH); // u=rw-, g=rw-, o=rw- try { - m_ipc_endpoint->register_endpoint(m_settings->global_socket()); + m_ipc_service->register_endpoint(m_settings->global_socket()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create user API socket: {}", e.what()); @@ -1120,7 +1120,7 @@ void urd::init_event_handlers() { #if 0 // setup socket for remote connections try { - m_ipc_endpoint->register_endpoint(m_settings->remote_port()); + m_ipc_service->register_endpoint(m_settings->remote_port()); } catch(const std::exception& e) { LOGGER_ERROR("Failed to create socket for remote connections: {}", @@ -1134,62 +1134,62 @@ void urd::init_event_handlers() { LOGGER_INFO(" * Installing message handlers..."); /* user-level functionalities */ - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::iotask_create, std::bind(&urd::iotask_create_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::iotask_status, std::bind(&urd::iotask_status_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::ping, std::bind(&urd::ping_handler, this, std::placeholders::_1)); /* admin-level functionalities */ - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::job_register, std::bind(&urd::job_register_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::job_update, std::bind(&urd::job_update_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::job_unregister, std::bind(&urd::job_remove_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::process_register, std::bind(&urd::process_add_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::process_unregister, std::bind(&urd::process_remove_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::backend_register, std::bind(&urd::namespace_register_handler, this, std::placeholders::_1)); - /* m_ipc_endpoint->register_callback( + /* m_ipc_service->register_callback( api::request_type::backend_update, std::bind(&urd::namespace_update_handler, this, std::placeholders::_1));*/ - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::backend_unregister, std::bind(&urd::namespace_remove_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::global_status, std::bind(&urd::global_status_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::command, std::bind(&urd::command_handler, this, std::placeholders::_1)); - m_ipc_endpoint->register_callback( + m_ipc_service->register_callback( api::request_type::bad_request, std::bind(&urd::unknown_request_handler, this, std::placeholders::_1)); @@ -1210,7 +1210,7 @@ void urd::init_event_handlers() { // signal handlers must be installed AFTER daemonizing LOGGER_INFO(" * Installing signal handlers..."); - m_ipc_endpoint->set_signal_handler( + m_ipc_service->set_signal_handler( std::bind(&urd::signal_handler, this, std::placeholders::_1), SIGHUP, SIGTERM, SIGINT); } @@ -1550,7 +1550,7 @@ int urd::run() { // N.B. This call blocks here, which means that everything after it // will only run when a shutdown command is received - m_ipc_endpoint->run(); + m_ipc_service->run(); print_farewell(); teardown(); @@ -1570,10 +1570,10 @@ void urd::teardown() { // //m_signal_listener.reset(); // } - if(m_ipc_endpoint) { + if(m_ipc_service) { LOGGER_INFO("* Stopping API listener..."); - m_ipc_endpoint->stop(); - m_ipc_endpoint.reset(); + m_ipc_service->stop(); + m_ipc_service.reset(); } api_listener::cleanup(); @@ -1605,8 +1605,8 @@ urd::teardown_and_exit() { } void urd::shutdown() { - if(m_ipc_endpoint) { - m_ipc_endpoint->stop(); + if(m_ipc_service) { + m_ipc_service->stop(); } } diff --git a/src/urd.hpp b/src/urd.hpp index 5263e5f..bb2973e 100644 --- a/src/urd.hpp +++ b/src/urd.hpp @@ -147,7 +147,7 @@ private: std::shared_ptr m_settings; std::unique_ptr m_transferor_registry; - std::unique_ptr m_ipc_endpoint; + std::unique_ptr m_ipc_service; std::shared_ptr m_network_service; -- GitLab From 72766a0b538f010a00a15d5e14d5d917dfe2689a Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 12:24:13 +0100 Subject: [PATCH 50/51] Plugins now receive a context object Transfer plugins now receive a new 'context' object that provides access to relevant data structures from the daemon. This way, they can query information about the current configuration and status without directly involving the daemon object. --- src/Makefile.am | 1 + src/context.hpp | 65 +++++++++++++++++++ .../transferors/local-path-to-local-path.cpp | 4 ++ .../transferors/local-path-to-local-path.hpp | 12 +++- .../local-path-to-remote-resource.cpp | 11 ++-- .../local-path-to-remote-resource.hpp | 5 +- .../transferors/local-path-to-shared-path.cpp | 4 ++ .../transferors/local-path-to-shared-path.hpp | 6 ++ src/io/transferors/memory-to-local-path.cpp | 4 ++ src/io/transferors/memory-to-local-path.hpp | 5 ++ .../transferors/memory-to-remote-resource.cpp | 8 +-- .../transferors/memory-to-remote-resource.hpp | 5 +- src/io/transferors/memory-to-shared-path.cpp | 4 ++ src/io/transferors/memory-to-shared-path.hpp | 5 ++ .../remote-resource-to-local-path.cpp | 11 ++-- .../remote-resource-to-local-path.hpp | 6 +- src/urd.cpp | 56 +++++++++------- 17 files changed, 167 insertions(+), 45 deletions(-) create mode 100644 src/context.hpp diff --git a/src/Makefile.am b/src/Makefile.am index 3403e05..82a82ca 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -145,6 +145,7 @@ liburd_aux_la_SOURCES = \ config/parsers.cpp \ config/parsers.hpp \ config/defaults.hpp \ + context.hpp \ io.hpp \ io/task.hpp \ io/task-copy.hpp \ diff --git a/src/context.hpp b/src/context.hpp new file mode 100644 index 0000000..8530e94 --- /dev/null +++ b/src/context.hpp @@ -0,0 +1,65 @@ +/************************************************************************* + * Copyright (C) 2017-2019 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#ifndef NORNS_CONTEXT_HPP +#define NORNS_CONTEXT_HPP + +#include +#include + +namespace bfs = boost::filesystem; + +namespace hermes { + class async_engine; +} // namespace hermes + +namespace norns { + +struct context { + + context(bfs::path staging_directory, + std::shared_ptr network_service) : + m_staging_directory(std::move(staging_directory)), + m_network_service(std::move(network_service)) { } + + bfs::path + staging_directory() const { + return m_staging_directory; + } + + std::shared_ptr + network_service() const { + return m_network_service; + } + + bfs::path m_staging_directory; + std::shared_ptr m_network_service; +}; + +} // namespace norns + +#endif // NORNS_CONTEXT_HPP diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 644cee4..f4d0f95 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -199,6 +199,10 @@ copy_directory(const std::shared_ptr& task_info, namespace norns { namespace io { +local_path_to_local_path_transferor::local_path_to_local_path_transferor( + const context& ctx) : + m_ctx(ctx) {} + bool local_path_to_local_path_transferor::validate( const std::shared_ptr& src_info, diff --git a/src/io/transferors/local-path-to-local-path.hpp b/src/io/transferors/local-path-to-local-path.hpp index 9fbaf11..dee04f2 100644 --- a/src/io/transferors/local-path-to-local-path.hpp +++ b/src/io/transferors/local-path-to-local-path.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace norns { @@ -48,8 +49,12 @@ namespace io { struct local_path_to_local_path_transferor : public transferor { - bool validate(const std::shared_ptr& src_info, - const std::shared_ptr& dst_info) const override final; + local_path_to_local_path_transferor(const context& ctx); + + bool + validate(const std::shared_ptr &src_info, + const std::shared_ptr &dst_info) + const override final; std::error_code transfer(const auth::credentials& auth, @@ -67,6 +72,9 @@ struct local_path_to_local_path_transferor : public transferor { std::string to_string() const override final; + +private: + context m_ctx; }; diff --git a/src/io/transferors/local-path-to-remote-resource.cpp b/src/io/transferors/local-path-to-remote-resource.cpp index 6edf23e..8399caa 100644 --- a/src/io/transferors/local-path-to-remote-resource.cpp +++ b/src/io/transferors/local-path-to-remote-resource.cpp @@ -127,9 +127,10 @@ unpack_archive(const bfs::path& archive_path, namespace norns { namespace io { -local_path_to_remote_resource_transferor::local_path_to_remote_resource_transferor( - std::shared_ptr network_service) : - m_network_service(network_service) { } +local_path_to_remote_resource_transferor:: + local_path_to_remote_resource_transferor(const context& ctx) : + m_staging_directory(ctx.staging_directory()), + m_network_service(ctx.network_service()) { } bool local_path_to_remote_resource_transferor::validate( @@ -172,7 +173,7 @@ local_path_to_remote_resource_transferor::transfer( const bfs::path ar_path = ::pack_archive("norns-archive-%%%%-%%%%-%%%%.tar", - "/tmp", + m_staging_directory, {{true, d_src.canonical_path(), d_dst.name()}}, ec); @@ -320,7 +321,7 @@ local_path_to_remote_resource_transferor::accept_transfer( d_dst.name()}, /* parent_path */ bfs::path{is_collection ? - "/tmp" : + m_staging_directory : d_dst.parent()->mount()}, remote_buffers.size(), ec); diff --git a/src/io/transferors/local-path-to-remote-resource.hpp b/src/io/transferors/local-path-to-remote-resource.hpp index d885470..a42d5f1 100644 --- a/src/io/transferors/local-path-to-remote-resource.hpp +++ b/src/io/transferors/local-path-to-remote-resource.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace hermes { @@ -55,8 +56,7 @@ namespace io { struct local_path_to_remote_resource_transferor : public transferor { - local_path_to_remote_resource_transferor( - std::shared_ptr network_service); + local_path_to_remote_resource_transferor(const context& ctx); bool validate(const std::shared_ptr& src_info, @@ -81,6 +81,7 @@ struct local_path_to_remote_resource_transferor : public transferor { to_string() const override final; private: + bfs::path m_staging_directory; std::shared_ptr m_network_service; }; diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index 6833885..5cc5447 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -41,6 +41,10 @@ namespace norns { namespace io { +local_path_to_shared_path_transferor::local_path_to_shared_path_transferor( + const context& ctx) : + m_ctx(ctx) { } + bool local_path_to_shared_path_transferor::validate( const std::shared_ptr& src_info, diff --git a/src/io/transferors/local-path-to-shared-path.hpp b/src/io/transferors/local-path-to-shared-path.hpp index 49e8921..028fb24 100644 --- a/src/io/transferors/local-path-to-shared-path.hpp +++ b/src/io/transferors/local-path-to-shared-path.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace norns { @@ -48,6 +49,8 @@ namespace io { struct local_path_to_shared_path_transferor : public transferor { + local_path_to_shared_path_transferor(const context& ctx); + bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) @@ -69,6 +72,9 @@ struct local_path_to_shared_path_transferor : public transferor { std::string to_string() const override final; + +private: + context m_ctx; }; } // namespace io diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index ac660a8..984aeda 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -148,6 +148,10 @@ retry_close: namespace norns { namespace io { +memory_region_to_local_path_transferor:: + memory_region_to_local_path_transferor(const context &ctx) : + m_ctx(ctx) { } + bool memory_region_to_local_path_transferor::validate( const std::shared_ptr& src_info, diff --git a/src/io/transferors/memory-to-local-path.hpp b/src/io/transferors/memory-to-local-path.hpp index 720f1e8..5875e51 100644 --- a/src/io/transferors/memory-to-local-path.hpp +++ b/src/io/transferors/memory-to-local-path.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace norns { @@ -48,6 +49,8 @@ namespace io { struct memory_region_to_local_path_transferor : public transferor { + memory_region_to_local_path_transferor(const context& ctx); + bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) @@ -69,6 +72,8 @@ struct memory_region_to_local_path_transferor : public transferor { std::string to_string() const override final; + + context m_ctx; }; } // namespace io diff --git a/src/io/transferors/memory-to-remote-resource.cpp b/src/io/transferors/memory-to-remote-resource.cpp index 7102e26..25cf636 100644 --- a/src/io/transferors/memory-to-remote-resource.cpp +++ b/src/io/transferors/memory-to-remote-resource.cpp @@ -82,9 +82,9 @@ namespace norns { namespace io { memory_region_to_remote_resource_transferor:: - memory_region_to_remote_resource_transferor( - std::shared_ptr network_service) : - m_network_service(network_service) {} + memory_region_to_remote_resource_transferor(const context& ctx) : + m_staging_directory(ctx.staging_directory()), + m_network_service(ctx.network_service()) {} bool memory_region_to_remote_resource_transferor::validate( @@ -134,7 +134,7 @@ memory_region_to_remote_resource_transferor::transfer( auto tempfile = std::make_shared( std::string{"norns-tempfile-%%%%-%%%%-%%%%"}, - bfs::path{"/tmp"}, + bfs::path{m_staging_directory}, d_src.size(), ec); diff --git a/src/io/transferors/memory-to-remote-resource.hpp b/src/io/transferors/memory-to-remote-resource.hpp index 463c66c..06843dd 100644 --- a/src/io/transferors/memory-to-remote-resource.hpp +++ b/src/io/transferors/memory-to-remote-resource.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace norns { @@ -48,8 +49,7 @@ namespace io { struct memory_region_to_remote_resource_transferor : public transferor { - memory_region_to_remote_resource_transferor( - std::shared_ptr network_service); + memory_region_to_remote_resource_transferor(const context& ctx); bool validate(const std::shared_ptr& src_info, @@ -74,6 +74,7 @@ struct memory_region_to_remote_resource_transferor : public transferor { to_string() const override final; private: + bfs::path m_staging_directory; std::shared_ptr m_network_service; }; diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index 9f3aac7..b094f05 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -34,6 +34,10 @@ namespace norns { namespace io { +memory_region_to_shared_path_transferor:: + memory_region_to_shared_path_transferor(const context &ctx) : + m_ctx(ctx) { } + bool memory_region_to_shared_path_transferor::validate( const std::shared_ptr& src_info, diff --git a/src/io/transferors/memory-to-shared-path.hpp b/src/io/transferors/memory-to-shared-path.hpp index b646d21..66721fc 100644 --- a/src/io/transferors/memory-to-shared-path.hpp +++ b/src/io/transferors/memory-to-shared-path.hpp @@ -30,6 +30,7 @@ #include #include +#include "context.hpp" #include "transferor.hpp" namespace norns { @@ -48,6 +49,8 @@ namespace io { struct memory_region_to_shared_path_transferor : public transferor { + memory_region_to_shared_path_transferor(const context& ctx); + bool validate(const std::shared_ptr& src_info, const std::shared_ptr& dst_info) @@ -69,6 +72,8 @@ struct memory_region_to_shared_path_transferor : public transferor { std::string to_string() const override final; + + context m_ctx; }; } // namespace io diff --git a/src/io/transferors/remote-resource-to-local-path.cpp b/src/io/transferors/remote-resource-to-local-path.cpp index 228680c..07eba07 100644 --- a/src/io/transferors/remote-resource-to-local-path.cpp +++ b/src/io/transferors/remote-resource-to-local-path.cpp @@ -120,9 +120,10 @@ unpack_archive(const bfs::path& archive_path, namespace norns { namespace io { -remote_resource_to_local_path_transferor::remote_resource_to_local_path_transferor( - std::shared_ptr network_service) : - m_network_service(network_service) { } +remote_resource_to_local_path_transferor:: + remote_resource_to_local_path_transferor(const context& ctx) : + m_staging_directory(ctx.staging_directory()), + m_network_service(ctx.network_service()) { } bool remote_resource_to_local_path_transferor::validate( @@ -187,7 +188,7 @@ remote_resource_to_local_path_transferor::transfer( d_dst.name()}, /* parent_dir */ {resp.at(0).is_collection() ? - "/tmp" : + m_staging_directory : d_dst.parent()->mount()}, resp.at(0).packed_size(), ec); @@ -291,7 +292,7 @@ remote_resource_to_local_path_transferor::accept_transfer( const bfs::path ar_path = ::pack_archive("norns-archive-%%%%-%%%%-%%%%.tar", - "/tmp", + m_staging_directory, {{true, d_src.canonical_path(), d_dst.name()}}, ec); diff --git a/src/io/transferors/remote-resource-to-local-path.hpp b/src/io/transferors/remote-resource-to-local-path.hpp index d841850..1e63a4c 100644 --- a/src/io/transferors/remote-resource-to-local-path.hpp +++ b/src/io/transferors/remote-resource-to-local-path.hpp @@ -28,8 +28,10 @@ #ifndef __IO_REMOTE_RESOURCE_TO_LOCAL_PATH_TX__ #define __IO_REMOTE_RESOURCE_TO_LOCAL_PATH_TX__ +#include #include +#include "context.hpp" #include "transferor.hpp" namespace hermes { @@ -55,8 +57,7 @@ namespace io { struct remote_resource_to_local_path_transferor : public transferor { - remote_resource_to_local_path_transferor( - std::shared_ptr network_service); + remote_resource_to_local_path_transferor(const context& ctx); bool validate(const std::shared_ptr& src_info, @@ -81,6 +82,7 @@ struct remote_resource_to_local_path_transferor : public transferor { to_string() const override final; private: + bfs::path m_staging_directory; std::shared_ptr m_network_service; }; diff --git a/src/urd.cpp b/src/urd.cpp index 79164e0..f93853d 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -58,6 +58,7 @@ #include "fmt.hpp" #include "hermes.hpp" #include "rpcs.hpp" +#include "context.hpp" #include "urd.hpp" namespace norns { @@ -1340,44 +1341,50 @@ void urd::load_transfer_plugins() { utils::to_string(t1), utils::to_string(t2)); }; + context ctx(m_settings->staging_directory(), + m_network_service); + // memory region -> local path - load_plugin(data::resource_type::memory_region, - data::resource_type::local_posix_path, - std::make_shared()); + load_plugin( + data::resource_type::memory_region, + data::resource_type::local_posix_path, + std::make_shared(ctx)); // memory region -> shared path - load_plugin(data::resource_type::memory_region, - data::resource_type::shared_posix_path, - std::make_shared()); + load_plugin( + data::resource_type::memory_region, + data::resource_type::shared_posix_path, + std::make_shared(ctx)); // memory region -> remote path load_plugin( data::resource_type::memory_region, data::resource_type::remote_resource, - std::make_shared( - m_network_service)); + std::make_shared(ctx)); // local path -> local path - load_plugin(data::resource_type::local_posix_path, - data::resource_type::local_posix_path, - std::make_shared()); + load_plugin( + data::resource_type::local_posix_path, + data::resource_type::local_posix_path, + std::make_shared(ctx)); // local path -> shared path - load_plugin(data::resource_type::local_posix_path, - data::resource_type::shared_posix_path, - std::make_shared()); + load_plugin( + data::resource_type::local_posix_path, + data::resource_type::shared_posix_path, + std::make_shared(ctx)); // local path -> remote resource - load_plugin(data::resource_type::local_posix_path, - data::resource_type::remote_resource, - std::make_shared( - m_network_service)); + load_plugin( + data::resource_type::local_posix_path, + data::resource_type::remote_resource, + std::make_shared(ctx)); // remote resource -> local path - load_plugin(data::resource_type::remote_resource, - data::resource_type::local_posix_path, - std::make_shared( - m_network_service)); + load_plugin( + data::resource_type::remote_resource, + data::resource_type::local_posix_path, + std::make_shared(ctx)); } void urd::load_default_namespaces() { @@ -1538,9 +1545,12 @@ int urd::run() { init_event_handlers(); init_namespace_manager(); load_backend_plugins(); - load_transfer_plugins(); load_default_namespaces(); + // load plugins now so that when we propagate the daemon context to them + // everything is set up + load_transfer_plugins(); + // start the listener for remote transfers // N.B. This call returns immediately m_network_service->run(); -- GitLab From 745e576a21c82b2d657a2d867a0c25e5a94b5ba9 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Thu, 7 Mar 2019 12:29:34 +0100 Subject: [PATCH 51/51] Update .gitlab-ci.yml --- .gitlab-ci.yml | 114 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b8040de..662f559 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,7 +128,7 @@ build:gcc:8: ################################################################################ # test scripts ################################################################################ -test:ubuntu:latest: +test:coverage: image: ubuntu:latest stage: test @@ -248,10 +248,120 @@ test:ubuntu:latest: paths: - build/html/coverage/ +test:optimized: + image: ubuntu:latest + stage: test + + # Install dependencies + before_script: + - apt-get update && + apt-get upgrade -y && + apt-get install -y + git + build-essential + autotools-dev + automake + autoconf + libtool + pkg-config + libboost-system-dev + libboost-filesystem-dev + libboost-program-options-dev + libboost-thread-dev + libboost-regex-dev + libprotobuf-dev + protobuf-compiler + libprotobuf-c-dev + protobuf-c-compiler + libyaml-cpp-dev + libyaml-dev + libtar-dev + libcap2-bin + valgrind + cmake + + - pushd . && + git clone https://github.com/ofiwg/libfabric.git && + cd libfabric && + ./autogen.sh && + mkdir build && + cd build && + ../configure && + make -j $(nproc) && + make install && + popd && + ldconfig + + - pushd . && + git clone https://github.com/mercury-hpc/mercury.git && + cd mercury && + mkdir build && + cd build && + cmake + -DCMAKE_BUILD_TYPE:STRING=Debug + -DBUILD_TESTING:BOOL=OFF + -DMERCURY_USE_SM_ROUTING:BOOL=OFF + -DMERCURY_USE_SELF_FORWARD:BOOL=OFF + -DMERCURY_USE_CHECKSUMS:BOOL=OFF + -DMERCURY_USE_BOOST_PP:BOOL=ON + -DMERCURY_USE_EAGER_BULK:BOOL=ON + -DBUILD_SHARED_LIBS:BOOL=ON + -DNA_USE_OFI:BOOL=ON + .. && + make -j $(nproc) && + make install && + popd && + ldconfig + + # Build and test + script: + - ./bootstrap.sh + - mkdir build && cd build + - ../configure + --enable-tests + - make -j$(nproc) + - cd tests + - make -j$(nproc) core + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./core -as + - make -j$(nproc) api + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::NORNS_TASK]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_iotask_init]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_resource_init]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_status]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_buffer_to_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_copy_local_posix_files]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_pull_to_posix_subdir]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_links]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_memory_to_posix_file_errors]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_file]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_push_to_posix_subdir]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_add_process]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_job]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_register_namespace]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_remove_process]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_job]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_unregister_namespace]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_update_job]" + + after_script: + - if [[ -e build/tests.log ]]; + then + cat $(tail -1 build/tests.log)/config/urd.log; + fi + pages: stage: deploy dependencies: - - test:ubuntu:latest + - test:coverage script: - mv coverage/ public/ artifacts: -- GitLab