From f056c93dbef24f919a6471a671aa88624bcc60fc Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Wed, 29 Aug 2018 21:28:29 +0200 Subject: [PATCH 1/3] Define NORNS_IOTASK_REMOVE and basic transferor This commit defines the NORNS_IOTASK_REMOVE subtype and also implements the basic mechanism so that a task of this type can be executed. --- include/norns/norns.h | 5 ++-- lib/communication.c | 3 +- src/api/request.cpp | 2 ++ src/common/types.hpp | 1 + src/io/task-manager.cpp | 5 ++++ src/io/task.cpp | 61 +++++++++++++++++++++++++++++++++++++++++ src/urd.cpp | 4 ++- 7 files changed, 77 insertions(+), 4 deletions(-) diff --git a/include/norns/norns.h b/include/norns/norns.h index 41489e4..1745b7d 100644 --- a/include/norns/norns.h +++ b/include/norns/norns.h @@ -59,8 +59,9 @@ typedef struct { } norns_iotask_t; /* Task types */ -#define NORNS_IOTASK_COPY 0x1 -#define NORNS_IOTASK_MOVE 0x2 +#define NORNS_IOTASK_COPY 0x1 +#define NORNS_IOTASK_MOVE 0x2 +#define NORNS_IOTASK_REMOVE 0x3 /* I/O task status descriptor */ typedef struct { diff --git a/lib/communication.c b/lib/communication.c index 263a5d5..a105166 100644 --- a/lib/communication.c +++ b/lib/communication.c @@ -63,7 +63,8 @@ send_submit_request(norns_iotask_t* task) { // XXX add missing checks: e.g. validate src resource if(task->t_id != 0 || (task->t_op != NORNS_IOTASK_COPY && - task->t_op != NORNS_IOTASK_MOVE )) { + task->t_op != NORNS_IOTASK_MOVE && + task->t_op != NORNS_IOTASK_REMOVE)) { return NORNS_EBADARGS; } diff --git a/src/api/request.cpp b/src/api/request.cpp index 32062b7..8c9a738 100644 --- a/src/api/request.cpp +++ b/src/api/request.cpp @@ -46,6 +46,8 @@ norns::iotask_type decode_iotask_type(::google::protobuf::uint32 type) { return iotask_type::copy; case NORNS_IOTASK_MOVE: return iotask_type::move; + case NORNS_IOTASK_REMOVE: + return iotask_type::remove; default: return iotask_type::unknown; } diff --git a/src/common/types.hpp b/src/common/types.hpp index 90dce2a..d2f55fe 100644 --- a/src/common/types.hpp +++ b/src/common/types.hpp @@ -47,6 +47,7 @@ using iotask_id = norns_tid_t; enum class iotask_type { copy, move, + remove, noop, unknown }; diff --git a/src/io/task-manager.cpp b/src/io/task-manager.cpp index 8a904c2..410ae1e 100644 --- a/src/io/task-manager.cpp +++ b/src/io/task-manager.cpp @@ -103,6 +103,11 @@ task_manager::create_task(iotask_type type, const auth::credentials& auth, io::task( std::move(task_info_ptr), std::move(tx_ptr)), register_completion); break; + case iotask_type::remove: + m_runners.submit_with_epilog_and_forget( + io::task( + std::move(task_info_ptr), std::move(tx_ptr)), register_completion); + break; default: m_runners.submit_and_forget( io::task( diff --git a/src/io/task.cpp b/src/io/task.cpp index 7e5f385..abbc503 100644 --- a/src/io/task.cpp +++ b/src/io/task.cpp @@ -160,6 +160,67 @@ task::operator()() { } +///////////////////////////////////////////////////////////////////////////////// +// specializations for remove tasks +///////////////////////////////////////////////////////////////////////////////// +template<> +void +task::operator()() { + + std::error_code ec; + + const auto tid = m_task_info->id(); + const auto type = m_task_info->type(); + const auto src_backend = m_task_info->src_backend(); + const auto src_rinfo = m_task_info->src_rinfo(); + const auto dst_backend = m_task_info->dst_backend(); + const auto dst_rinfo = m_task_info->dst_rinfo(); + const auto auth = m_task_info->auth(); + + // helper lambda for error reporting + const auto log_error = [&] (const std::string& msg) { + m_task_info->update_status(task_status::finished_with_error, + urd_error::system_error, ec); + + std::string r_msg = "[{}] " + msg + ": {}"; + + LOGGER_ERROR(r_msg.c_str(), tid, ec.message()); + LOGGER_WARN("[{}] I/O task completed with error", tid); + }; + + LOGGER_WARN("[{}] Starting I/O task", tid); + LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); + LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); + LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); + + m_task_info->update_status(task_status::running); + + auto src = src_backend->get_resource(src_rinfo, ec); + + if(ec) { + log_error("Could not access input data " + src_rinfo->to_string()); + return; + } + + auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); + + if(ec) { + log_error("Could not create output data " + dst_rinfo->to_string()); + return; + } + + ec = m_transferor->transfer(auth, m_task_info, src, dst); + + if(ec) { + log_error("Transfer failed"); + return; + } + + LOGGER_WARN("[{}] I/O task completed successfully", tid); + m_task_info->update_status(task_status::finished, urd_error::success, + std::make_error_code(static_cast(ec.value()))); +} + ///////////////////////////////////////////////////////////////////////////////// // specializations for noop tasks ///////////////////////////////////////////////////////////////////////////////// diff --git a/src/urd.cpp b/src/urd.cpp index c8b5129..1cf2446 100644 --- a/src/urd.cpp +++ b/src/urd.cpp @@ -180,7 +180,9 @@ urd_error urd::validate_iotask_args(iotask_type type, const resource_info_ptr& src_rinfo, const resource_info_ptr& dst_rinfo) const { - if(type != iotask_type::copy && type != iotask_type::move) { + if(type != iotask_type::copy && + type != iotask_type::move && + type != iotask_type::remove) { return urd_error::bad_args; } -- GitLab From be496c5e1202fe13775bebf1a516af90e5aa56ea Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 17 Sep 2018 15:04:49 +0200 Subject: [PATCH 2/3] Add support for local file deletion The norns library and service now supports a new type of i/o task called NORNS_IOTASK_REMOVE that enqueues the deletion of a resource from a backend. As of right now, only the deletion of NORNS_LOCAL_PATHs is supported. --- include/norns/norns.h | 4 +- include/norns/norns_resources.h | 1 + lib/communication.c | 1 + lib/libnorns.c | 25 +- lib/requests.c | 8 +- src/api/request.cpp | 101 ++- src/api/request.hpp | 2 +- src/backends/backend-base.hpp | 1 + src/backends/lustre-fs.cpp | 9 +- src/backends/lustre-fs.hpp | 1 + src/backends/nvml-dax.cpp | 9 +- src/backends/nvml-dax.hpp | 1 + src/backends/posix-fs.cpp | 38 + src/backends/posix-fs.hpp | 1 + src/backends/process-memory.cpp | 6 + src/backends/process-memory.hpp | 1 + src/backends/remote-backend.cpp | 9 +- src/backends/remote-backend.hpp | 1 + src/common/types.cpp | 2 + src/io/task-manager.cpp | 241 +++++- src/io/task-manager.hpp | 17 + src/io/task.cpp | 21 +- src/io/task.hpp | 3 + src/io/transferor-registry.cpp | 6 +- .../transferors/local-path-to-local-path.cpp | 5 + .../transferors/local-path-to-local-path.hpp | 1 + .../transferors/local-path-to-remote-path.cpp | 5 + .../transferors/local-path-to-remote-path.hpp | 1 + .../transferors/local-path-to-shared-path.cpp | 5 +- .../transferors/local-path-to-shared-path.hpp | 1 + src/io/transferors/memory-to-local-path.cpp | 4 + src/io/transferors/memory-to-local-path.hpp | 1 + src/io/transferors/memory-to-remote-path.cpp | 5 + src/io/transferors/memory-to-remote-path.hpp | 1 + src/io/transferors/memory-to-shared-path.cpp | 4 + src/io/transferors/memory-to-shared-path.hpp | 1 + src/io/transferors/transferor.hpp | 3 + src/namespaces/namespace-manager.hpp | 25 +- src/resources.hpp | 1 + src/resources/resource-type.hpp | 3 +- src/urd.cpp | 113 ++- tests/Makefile.am | 1 + tests/api-remove-local-data.cpp | 718 ++++++++++++++++++ tests/api-task-init.cpp | 41 +- 44 files changed, 1264 insertions(+), 184 deletions(-) create mode 100644 tests/api-remove-local-data.cpp diff --git a/include/norns/norns.h b/include/norns/norns.h index 1745b7d..6cc982e 100644 --- a/include/norns/norns.h +++ b/include/norns/norns.h @@ -80,8 +80,8 @@ typedef struct { void norns_iotask_init(norns_iotask_t* task, norns_op_t operation, norns_resource_t* src, norns_resource_t* dst) __THROW; -norns_iotask_t NORNS_IOTASK(norns_op_t operation, norns_resource_t src, - norns_resource_t dst) __THROW; +norns_iotask_t +NORNS_IOTASK(norns_op_t operation, norns_resource_t src, ...) __THROW; /* Submit an asynchronous I/O task */ norns_error_t norns_submit(norns_iotask_t* task) __THROW; diff --git a/include/norns/norns_resources.h b/include/norns/norns_resources.h index e0a4dd0..35ff4b9 100644 --- a/include/norns/norns_resources.h +++ b/include/norns/norns_resources.h @@ -39,6 +39,7 @@ extern "C" { /* Resource types */ #define NORNS_PROCESS_MEMORY 0x0100000 /* Memory buffer */ #define NORNS_POSIX_PATH 0x0200000 /* POSIX path */ +#define NORNS_NULL_RESOURCE 0x1000000 /* Access types */ #define R_LOCAL 0x0000010 /* Local resource (default) */ diff --git a/lib/communication.c b/lib/communication.c index a105166..fc6359c 100644 --- a/lib/communication.c +++ b/lib/communication.c @@ -65,6 +65,7 @@ send_submit_request(norns_iotask_t* task) { (task->t_op != NORNS_IOTASK_COPY && task->t_op != NORNS_IOTASK_MOVE && task->t_op != NORNS_IOTASK_REMOVE)) { + ERR("Invalid fields detected in norns_iotask_t"); return NORNS_EBADARGS; } diff --git a/lib/libnorns.c b/lib/libnorns.c index b05130f..da4fd57 100644 --- a/lib/libnorns.c +++ b/lib/libnorns.c @@ -149,10 +149,19 @@ libnorns_reload_config_file(void) { /* Public API */ norns_iotask_t -NORNS_IOTASK(norns_op_t optype, norns_resource_t src, norns_resource_t dst) { +NORNS_IOTASK(norns_op_t optype, norns_resource_t src, ...) { norns_iotask_t task; + if(optype == NORNS_IOTASK_REMOVE) { + norns_iotask_init(&task, optype, &src, NULL); + return task; + } + + va_list ap; + va_start(ap, src); + norns_resource_t dst = va_arg(ap, norns_resource_t); norns_iotask_init(&task, optype, &src, &dst); + va_end(ap); return task; } @@ -165,15 +174,23 @@ norns_iotask_init(norns_iotask_t* task, norns_op_t optype, return; } - if(src == NULL || dst == NULL) { - memset(task, 0, sizeof(*task)); + memset(task, 0, sizeof(*task)); + + if(src == NULL) { return; } task->t_id = 0; task->t_op = optype; task->t_src = *src; - task->t_dst = *dst; + + if(dst != NULL) { + task->t_dst = *dst; + return; + } + + // dst is NULL, set r_flags so that we are aware of it later + task->t_dst.r_flags = NORNS_NULL_RESOURCE; } norns_error_t diff --git a/lib/requests.c b/lib/requests.c index b0b0d55..65e9220 100644 --- a/lib/requests.c +++ b/lib/requests.c @@ -531,17 +531,21 @@ build_resource_msg(const norns_resource_t* res) { if(msg->buffer == NULL) { goto oom_cleanup; } + + return msg; } - else { - assert(res->r_flags & NORNS_POSIX_PATH); + if(res->r_flags & NORNS_POSIX_PATH) { msg->path = build_path_msg(&res->r_posix_path); if(msg->path == NULL) { goto oom_cleanup; } + + return msg; } + assert(res->r_flags & NORNS_NULL_RESOURCE); return msg; oom_cleanup: diff --git a/src/api/request.cpp b/src/api/request.cpp index 8c9a738..49ed912 100644 --- a/src/api/request.cpp +++ b/src/api/request.cpp @@ -92,8 +92,8 @@ norns::command_type decode_command(::google::protobuf::uint32 type) { bool is_valid(const norns::rpc::Request_Task_Resource& res) { - if(!(res.type() & (NORNS_PROCESS_MEMORY | NORNS_POSIX_PATH))) { - return false; + if(res.type() & NORNS_NULL_RESOURCE) { + return true; } if(res.type() & NORNS_PROCESS_MEMORY) { @@ -128,7 +128,32 @@ bool is_valid(const norns::rpc::Request_Task_Resource& res) { return true; } - return true; + return false; +} + + +bool +is_valid(const norns::rpc::Request_Task& task) { + + using norns::iotask_type; + + switch(::decode_iotask_type(task.optype())) { + case iotask_type::copy: + case iotask_type::move: + case iotask_type::remove: + if(!task.has_source() || !::is_valid(task.source())) { + return false; + } + + if(!task.has_destination() || !::is_valid(task.destination())) { + return false; + } + + return true; + + default: + return false; + } } std::shared_ptr @@ -140,29 +165,33 @@ create_from(const norns::rpc::Request_Task_Resource& res) { using norns::data::shared_path_info; using norns::data::remote_path_info; - if(is_valid(res)) { - if(res.type() & NORNS_PROCESS_MEMORY) { - return std::make_shared(res.buffer().address(), - res.buffer().size()); + assert(is_valid(res)); + + if(res.type() & NORNS_PROCESS_MEMORY) { + return std::make_shared(res.buffer().address(), + res.buffer().size()); + } + + if(res.type() & NORNS_POSIX_PATH) { + if(res.type() & R_LOCAL) { + return std::make_shared(res.path().nsid(), + res.path().datapath()); } - else { // NORNS_POSIX_PATH - if(res.type() & R_LOCAL) { - return std::make_shared(res.path().nsid(), - res.path().datapath()); - } - else if(res.type() & R_SHARED) { - return std::make_shared(res.path().nsid(), - res.path().datapath()); - } - else { // R_REMOTE - return std::make_shared(res.path().nsid(), - res.path().hostname(), - res.path().datapath()); - } + + if(res.type() & R_SHARED) { + return std::make_shared(res.path().nsid(), + res.path().datapath()); } + + // R_REMOTE + assert(res.type() & R_REMOTE); + return std::make_shared(res.path().nsid(), + res.path().hostname(), + res.path().datapath()); } - return std::shared_ptr(); + assert(res.type() & NORNS_NULL_RESOURCE); + return {}; } std::string @@ -202,15 +231,15 @@ request_ptr request::create_from_buffer(const std::vector& buffer, int auto task = rpc_req.task(); iotask_type optype = ::decode_iotask_type(task.optype()); - if(optype == iotask_type::unknown) { - return std::make_unique(); - } + if(::is_valid(task)) { + const auto src_res = ::create_from(task.source()); + const auto dst_res = ::create_from(task.destination()); - std::shared_ptr src_res = ::create_from(task.source()); - std::shared_ptr dst_res = ::create_from(task.destination()); + if(dst_res) { + return std::make_unique(optype, std::move(src_res), dst_res); + } - if(src_res != nullptr && dst_res != nullptr) { - return std::make_unique(optype, src_res, dst_res); + return std::make_unique(optype, std::move(src_res), boost::none); } return std::make_unique(); @@ -365,9 +394,17 @@ std::string iotask_create_request::to_string() const { const auto src = this->get<1>(); const auto dst = this->get<2>(); - return utils::to_string(op) + ", " - + src->to_string() + " => " - + dst->to_string(); + auto str = utils::to_string(op); + + if(src) { + str += std::string(", ") + src->to_string(); + } + + if(dst) { + str += std::string(" => ") + (*dst)->to_string(); + } + + return str; } template<> diff --git a/src/api/request.hpp b/src/api/request.hpp index 0d24f1d..bc067fc 100644 --- a/src/api/request.hpp +++ b/src/api/request.hpp @@ -188,7 +188,7 @@ using iotask_create_request = detail::request_impl< request_type::iotask_create, iotask_type, std::shared_ptr, - std::shared_ptr + boost::optional> >; using iotask_status_request = detail::request_impl< diff --git a/src/backends/backend-base.hpp b/src/backends/backend-base.hpp index 143eab1..3e5395f 100644 --- a/src/backends/backend-base.hpp +++ b/src/backends/backend-base.hpp @@ -64,6 +64,7 @@ public: virtual resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const = 0; virtual resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const = 0; + virtual void remove(const resource_info_ptr& rinfo, std::error_code& ec) const = 0; virtual std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const = 0; diff --git a/src/backends/lustre-fs.cpp b/src/backends/lustre-fs.cpp index ea5861f..ad149f8 100644 --- a/src/backends/lustre-fs.cpp +++ b/src/backends/lustre-fs.cpp @@ -54,12 +54,19 @@ lustre::new_resource(const resource_info_ptr& rinfo, return backend::resource_ptr(); } -backend::resource_ptr lustre::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { +backend::resource_ptr +lustre::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; (void) ec; return backend::resource_ptr(); //XXX } +void +lustre::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { + (void) rinfo; + (void) ec; +} + std::size_t lustre::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; diff --git a/src/backends/lustre-fs.hpp b/src/backends/lustre-fs.hpp index 5bbf90f..ff9b91a 100644 --- a/src/backends/lustre-fs.hpp +++ b/src/backends/lustre-fs.hpp @@ -48,6 +48,7 @@ public: resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const override final; resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const override final; + void remove(const resource_info_ptr& rinfo, std::error_code& ec) const override final; std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const override final; bool accepts(resource_info_ptr res) const override final; diff --git a/src/backends/nvml-dax.cpp b/src/backends/nvml-dax.cpp index dd4a487..43ed839 100644 --- a/src/backends/nvml-dax.cpp +++ b/src/backends/nvml-dax.cpp @@ -54,12 +54,19 @@ nvml_dax::new_resource(const resource_info_ptr& rinfo, return std::make_shared(shared_from_this(), ""); //XXX } -backend::resource_ptr nvml_dax::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { +backend::resource_ptr +nvml_dax::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; (void) ec; return std::make_shared(shared_from_this(), ""); //XXX } +void +nvml_dax::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { + (void) rinfo; + (void) ec; +} + std::size_t nvml_dax::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; diff --git a/src/backends/nvml-dax.hpp b/src/backends/nvml-dax.hpp index df5826d..89ba863 100644 --- a/src/backends/nvml-dax.hpp +++ b/src/backends/nvml-dax.hpp @@ -47,6 +47,7 @@ public: resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const override final; resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const override final; + void remove(const resource_info_ptr& rinfo, std::error_code& ec) const override final; std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const override final; bool accepts(resource_info_ptr res) const override final; diff --git a/src/backends/posix-fs.cpp b/src/backends/posix-fs.cpp index bed4c0f..d1427c9 100644 --- a/src/backends/posix-fs.cpp +++ b/src/backends/posix-fs.cpp @@ -134,6 +134,44 @@ backend::resource_ptr posix_filesystem::get_resource(const resource_info_ptr& ri return std::make_shared(shared_from_this(), ns_abs_subpath); } +void +posix_filesystem::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { + + const auto d_rinfo = std::static_pointer_cast(rinfo); + const bfs::path ns_subpath = utils::lexical_normalize(d_rinfo->datapath(), false); + + if(ns_subpath.empty()) { + ec = std::make_error_code(static_cast(ENOENT)); + return; + } + + // check that path exists + boost::system::error_code error; + const bfs::path canonical_path = [&]() { + + const bfs::path p{m_mount / ns_subpath}; + + if(!bfs::is_symlink(p)) { + return bfs::canonical(p, error); + } + + bfs::exists(p, error); + return p; + }(); + + if(error) { + ec = std::make_error_code(static_cast(error.value())); + return; + } + + bfs::remove_all(canonical_path, error); + + if(error) { + ec = std::make_error_code(static_cast(error.value())); + return; + } +} + std::size_t posix_filesystem::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { diff --git a/src/backends/posix-fs.hpp b/src/backends/posix-fs.hpp index 37b2eb3..1130095 100644 --- a/src/backends/posix-fs.hpp +++ b/src/backends/posix-fs.hpp @@ -48,6 +48,7 @@ public: resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const override final; resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const override final; + void remove(const resource_info_ptr& rinfo, std::error_code& ec) const override final; std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const override final; bool accepts(resource_info_ptr res) const override final; diff --git a/src/backends/process-memory.cpp b/src/backends/process-memory.cpp index 61772e3..0a39f5e 100644 --- a/src/backends/process-memory.cpp +++ b/src/backends/process-memory.cpp @@ -64,6 +64,12 @@ process_memory::get_resource(const resource_info_ptr& rinfo, shared_from_this(), d_rinfo->address(), d_rinfo->size()); } +void +process_memory::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { + (void) rinfo; + (void) ec; +} + std::size_t process_memory::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { diff --git a/src/backends/process-memory.hpp b/src/backends/process-memory.hpp index c32f81b..4b46cd9 100644 --- a/src/backends/process-memory.hpp +++ b/src/backends/process-memory.hpp @@ -49,6 +49,7 @@ public: resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const override final; resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const override final; + void remove(const resource_info_ptr& rinfo, std::error_code& ec) const override final; std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const override final; bool accepts(resource_info_ptr res) const override final; diff --git a/src/backends/remote-backend.cpp b/src/backends/remote-backend.cpp index 9979ef0..486249c 100644 --- a/src/backends/remote-backend.cpp +++ b/src/backends/remote-backend.cpp @@ -53,12 +53,19 @@ remote_backend::new_resource(const resource_info_ptr& rinfo, return std::make_shared(shared_from_this()); //XXX } -backend::resource_ptr remote_backend::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { +backend::resource_ptr +remote_backend::get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; (void) ec; return std::make_shared(shared_from_this()); //XXX } +void +remote_backend::remove(const resource_info_ptr& rinfo, std::error_code& ec) const { + (void) rinfo; + (void) ec; +} + std::size_t remote_backend::get_size(const resource_info_ptr& rinfo, std::error_code& ec) const { (void) rinfo; diff --git a/src/backends/remote-backend.hpp b/src/backends/remote-backend.hpp index 1607356..769f862 100644 --- a/src/backends/remote-backend.hpp +++ b/src/backends/remote-backend.hpp @@ -48,6 +48,7 @@ public: resource_ptr new_resource(const resource_info_ptr& rinfo, bool is_collection, std::error_code& ec) const override final; resource_ptr get_resource(const resource_info_ptr& rinfo, std::error_code& ec) const override final; + void remove(const resource_info_ptr& rinfo, std::error_code& ec) const override final; std::size_t get_size(const resource_info_ptr& rinfo, std::error_code& ec) const override final; bool accepts(resource_info_ptr res) const override final; diff --git a/src/common/types.cpp b/src/common/types.cpp index e2b73d9..f105e4c 100644 --- a/src/common/types.cpp +++ b/src/common/types.cpp @@ -53,6 +53,8 @@ std::string to_string(iotask_type type) { return "DATA_COPY"; case iotask_type::move: return "DATA_MOVE"; + case iotask_type::remove: + return "DATA_REMOVE"; default: return "UNKNOWN_IOTASK"; } diff --git a/src/io/task-manager.cpp b/src/io/task-manager.cpp index 410ae1e..0187c9f 100644 --- a/src/io/task-manager.cpp +++ b/src/io/task-manager.cpp @@ -43,30 +43,137 @@ task_manager::task_manager(uint32_t nrunners, uint32_t backlog_size, bool dry_ru m_dry_run(dry_run), m_runners(nrunners) {} -boost::optional +bool +task_manager::register_transfer_plugin(const data::resource_type t1, + const data::resource_type t2, + std::shared_ptr&& trp) { + + return m_transferor_registry.add(t1, t2, + std::forward>(trp)); +} + +/// boost::optional +/// task_manager::create_task(iotask_type type, const auth::credentials& auth, +/// const backend_ptr src_backend, const resource_info_ptr src_rinfo, +/// const backend_ptr dst_backend, const resource_info_ptr dst_rinfo, +/// const transferor_ptr&& tx_ptr) { +/// +/// boost::unique_lock lock(m_mutex); +/// +/// iotask_id tid = ++m_id_base; +/// +/// if(m_task_info.count(tid) != 0) { +/// --m_id_base; +/// return boost::none; +/// } +/// +/// auto it = m_task_info.end(); +/// std::tie(it, std::ignore) = m_task_info.emplace(tid, +/// std::make_shared(tid, type, auth, src_backend, src_rinfo, +/// dst_backend, dst_rinfo)); +/// +/// const std::shared_ptr task_info_ptr = it->second; +/// +/// // helper lambda to register the completion of tasks so that we can keep track +/// // of the consumed bandwidth by each task +/// const auto register_completion = [=]() { +/// assert(task_info_ptr->status() == task_status::finished || +/// task_info_ptr->status() == task_status::finished_with_error); +/// +/// LOGGER_DEBUG("Task {} finished [{} MiB/s]", +/// task_info_ptr->id(), task_info_ptr->bandwidth()); +/// +/// auto bw = task_info_ptr->bandwidth(); +/// +/// // bw might be nan if the task did not finish correctly +/// if(!std::isnan(bw)) { +/// +/// const auto key = std::make_pair(task_info_ptr->src_rinfo()->nsid(), +/// task_info_ptr->dst_rinfo()->nsid()); +/// +/// if(!m_bandwidth_backlog.count(key)) { +/// m_bandwidth_backlog.emplace(key, +/// boost::circular_buffer(m_backlog_size)); +/// } +/// +/// m_bandwidth_backlog.at(key).push_back(bw); +/// } +/// }; +/// +/// if(!m_dry_run) { +/// switch(type) { +/// case iotask_type::copy: +/// m_runners.submit_with_epilog_and_forget( +/// io::task( +/// std::move(task_info_ptr), std::move(tx_ptr)), register_completion); +/// break; +/// case iotask_type::move: +/// m_runners.submit_with_epilog_and_forget( +/// io::task( +/// std::move(task_info_ptr), std::move(tx_ptr)), register_completion); +/// break; +/// case iotask_type::remove: +/// m_runners.submit_with_epilog_and_forget( +/// io::task( +/// std::move(task_info_ptr), std::move(tx_ptr)), register_completion); +/// break; +/// default: +/// m_runners.submit_and_forget( +/// io::task( +/// std::move(task_info_ptr), std::move(tx_ptr))); +/// } +/// } +/// else { +/// m_runners.submit_and_forget( +/// io::task(std::move(task_info_ptr), std::move(tx_ptr))); +/// } +/// +/// return tid; +/// } + + +std::tuple> task_manager::create_task(iotask_type type, const auth::credentials& auth, - const backend_ptr src_backend, const resource_info_ptr src_rinfo, - const backend_ptr dst_backend, const resource_info_ptr dst_rinfo, - const transferor_ptr&& tx_ptr) { + const std::vector& backend_ptrs, + const std::vector& rinfo_ptrs) { + + assert(backend_ptrs.size() == rinfo_ptrs.size()); boost::unique_lock lock(m_mutex); + // fetch an iotask_id for this task iotask_id tid = ++m_id_base; if(m_task_info.count(tid) != 0) { --m_id_base; - return boost::none; + return std::make_tuple(urd_error::too_many_tasks, -1); } - auto it = m_task_info.end(); - std::tie(it, std::ignore) = m_task_info.emplace(tid, - std::make_shared(tid, type, auth, src_backend, src_rinfo, - dst_backend, dst_rinfo)); + // immediately-invoked lambda to create the appropriate metadata for the task + // and register it into m_task_info + const auto task_info_ptr = [&]() { + + assert(backend_ptrs.size() == 1 || backend_ptrs.size() == 2); - const std::shared_ptr task_info_ptr = it->second; + const backend_ptr src_backend = backend_ptrs[0]; + const resource_info_ptr src_rinfo = rinfo_ptrs[0]; + const backend_ptr dst_backend = (backend_ptrs.size() == 1 ? + nullptr : backend_ptrs[1]); + const resource_info_ptr dst_rinfo = (rinfo_ptrs.size() == 1 ? + nullptr : rinfo_ptrs[1]); + + auto it = m_task_info.end(); + std::tie(it, std::ignore) = m_task_info.emplace(tid, + std::make_shared(tid, type, auth, + src_backend, src_rinfo, + dst_backend, dst_rinfo)); + return it->second; + }(); // helper lambda to register the completion of tasks so that we can keep track // of the consumed bandwidth by each task + // N.B: we use capture-by-value here so that the task_info_ptr is valid when + // the callback is invoked. const auto register_completion = [=]() { assert(task_info_ptr->status() == task_status::finished || task_info_ptr->status() == task_status::finished_with_error); @@ -91,35 +198,95 @@ task_manager::create_task(iotask_type type, const auth::credentials& auth, } }; - if(!m_dry_run) { - switch(type) { - case iotask_type::copy: - m_runners.submit_with_epilog_and_forget( - io::task( - std::move(task_info_ptr), std::move(tx_ptr)), register_completion); - break; - case iotask_type::move: - m_runners.submit_with_epilog_and_forget( - io::task( - std::move(task_info_ptr), std::move(tx_ptr)), register_completion); - break; - case iotask_type::remove: - m_runners.submit_with_epilog_and_forget( - io::task( - std::move(task_info_ptr), std::move(tx_ptr)), register_completion); - break; - default: - m_runners.submit_and_forget( - io::task( - std::move(task_info_ptr), std::move(tx_ptr))); - } + if(m_dry_run) { + type = iotask_type::noop; } - else { - m_runners.submit_and_forget( - io::task(std::move(task_info_ptr), std::move(tx_ptr))); + + switch(type) { + case iotask_type::remove: + { + assert(backend_ptrs.size() == 1); + + m_runners.submit_and_forget( + io::task(std::move(task_info_ptr))); + break; + } + + case iotask_type::copy: + { + assert(backend_ptrs.size() == 2); + + // find a transferor for this task + const auto tx_ptr = m_transferor_registry.get(rinfo_ptrs[0]->type(), + rinfo_ptrs[1]->type()); + if(!tx_ptr) { + return std::make_tuple(urd_error::not_supported, boost::none); + } + + LOGGER_DEBUG("Selected plugin: {}", tx_ptr->to_string()); + + if(!tx_ptr->validate(rinfo_ptrs[0], rinfo_ptrs[1])) { + return std::make_tuple(urd_error::bad_args, boost::none); + } + + m_runners.submit_with_epilog_and_forget( + io::task( + std::move(task_info_ptr), std::move(tx_ptr)), + register_completion); + break; + } + + case iotask_type::move: + { + assert(backend_ptrs.size() == 2); + + // find a transferor for this task + const auto tx_ptr = m_transferor_registry.get(rinfo_ptrs[0]->type(), + rinfo_ptrs[1]->type()); + if(!tx_ptr) { + return std::make_tuple(urd_error::not_supported, boost::none); + } + + LOGGER_DEBUG("Selected plugin: {}", tx_ptr->to_string()); + + if(!tx_ptr->validate(rinfo_ptrs[0], rinfo_ptrs[1])) { + return std::make_tuple(urd_error::bad_args, boost::none); + } + + m_runners.submit_with_epilog_and_forget( + io::task( + std::move(task_info_ptr), std::move(tx_ptr)), + register_completion); + break; + } + + case iotask_type::noop: + { + assert(backend_ptrs.size() == 2); + + // find a transferor for this task + const auto tx_ptr = m_transferor_registry.get(rinfo_ptrs[0]->type(), + rinfo_ptrs[1]->type()); + if(!tx_ptr) { + return std::make_tuple(urd_error::not_supported, boost::none); + } + + LOGGER_DEBUG("Selected plugin: {}", tx_ptr->to_string()); + + if(!tx_ptr->validate(rinfo_ptrs[0], rinfo_ptrs[1])) { + return std::make_tuple(urd_error::bad_args, boost::none); + } + + m_runners.submit_and_forget( + io::task(std::move(task_info_ptr))); + break; + } + + default: + return std::make_tuple(urd_error::bad_args, boost::none); } - return tid; + return std::make_tuple(urd_error::success, tid); } std::shared_ptr @@ -143,7 +310,7 @@ task_manager::global_stats() const { uint32_t pending_tasks = 0; const auto get_avg_bandwidth = [&](const std::string& nsid1, - const std::string& nsid2) -> double { + const std::string& nsid2) -> double { if(!m_bandwidth_backlog.count({nsid1, nsid2})) { return std::numeric_limits::quiet_NaN(); diff --git a/src/io/task-manager.hpp b/src/io/task-manager.hpp index 99a6011..dadc568 100644 --- a/src/io/task-manager.hpp +++ b/src/io/task-manager.hpp @@ -37,6 +37,12 @@ #include "common.hpp" namespace norns { + +// forward declarations +namespace data { +enum class resource_type; +} + namespace io { // forward declarations @@ -64,12 +70,22 @@ struct task_manager { task_manager(uint32_t nrunners, uint32_t backlog_size, bool dry_run); + bool + register_transfer_plugin(const data::resource_type t1, + const data::resource_type t2, + std::shared_ptr&& trp); + boost::optional create_task(iotask_type type, const auth::credentials& creds, const backend_ptr src_backend, const resource_info_ptr src_rinfo, const backend_ptr dst_backend, const resource_info_ptr dst_rinfo, const transferor_ptr&& tx_ptr); + std::tuple> + create_task(iotask_type type, const auth::credentials& auth, + const std::vector>& backend_ptrs, + const std::vector>& rinfo_ptrs); + std::shared_ptr find(iotask_id) const; @@ -88,6 +104,7 @@ private: std::unordered_map, boost::circular_buffer, pair_hash> m_bandwidth_backlog; thread_pool m_runners; + io::transferor_registry m_transferor_registry; }; } // namespace io diff --git a/src/io/task.cpp b/src/io/task.cpp index abbc503..a80e7bb 100644 --- a/src/io/task.cpp +++ b/src/io/task.cpp @@ -173,8 +173,6 @@ task::operator()() { const auto type = m_task_info->type(); const auto src_backend = m_task_info->src_backend(); const auto src_rinfo = m_task_info->src_rinfo(); - const auto dst_backend = m_task_info->dst_backend(); - const auto dst_rinfo = m_task_info->dst_rinfo(); const auto auth = m_task_info->auth(); // helper lambda for error reporting @@ -191,28 +189,13 @@ task::operator()() { LOGGER_WARN("[{}] Starting I/O task", tid); LOGGER_WARN("[{}] TYPE: {}", tid, utils::to_string(type)); LOGGER_WARN("[{}] FROM: {}", tid, src_backend->to_string()); - LOGGER_WARN("[{}] TO: {}", tid, dst_backend->to_string()); m_task_info->update_status(task_status::running); - auto src = src_backend->get_resource(src_rinfo, ec); + src_backend->remove(src_rinfo, ec); if(ec) { - log_error("Could not access input data " + src_rinfo->to_string()); - return; - } - - auto dst = dst_backend->new_resource(dst_rinfo, src->is_collection(), ec); - - if(ec) { - log_error("Could not create output data " + dst_rinfo->to_string()); - return; - } - - ec = m_transferor->transfer(auth, m_task_info, src, dst); - - if(ec) { - log_error("Transfer failed"); + log_error("Failed to remove resource " + src_rinfo->to_string()); return; } diff --git a/src/io/task.hpp b/src/io/task.hpp index 21712a1..b525410 100644 --- a/src/io/task.hpp +++ b/src/io/task.hpp @@ -54,6 +54,9 @@ struct task { using task_info_ptr = std::shared_ptr; using transferor_ptr = std::shared_ptr; + task(const task_info_ptr&& task_info) + : m_task_info(std::move(task_info)) { } + task(const task_info_ptr&& task_info, const transferor_ptr&& tx_ptr) : m_task_info(std::move(task_info)), m_transferor(std::move(tx_ptr)) { } diff --git a/src/io/transferor-registry.cpp b/src/io/transferor-registry.cpp index 14bfe13..fbb954f 100644 --- a/src/io/transferor-registry.cpp +++ b/src/io/transferor-registry.cpp @@ -33,8 +33,8 @@ namespace io { bool transferor_registry::add(const data::resource_type t1, - const data::resource_type t2, - std::shared_ptr&& trp) { + const data::resource_type t2, + std::shared_ptr&& trp) { using ValueType = std::shared_ptr; @@ -49,7 +49,7 @@ transferor_registry::add(const data::resource_type t1, std::shared_ptr transferor_registry::get(const data::resource_type t1, - const data::resource_type t2) const { + const data::resource_type t2) const { using ValueType = std::shared_ptr; diff --git a/src/io/transferors/local-path-to-local-path.cpp b/src/io/transferors/local-path-to-local-path.cpp index 6468e3e..851b00d 100644 --- a/src/io/transferors/local-path-to-local-path.cpp +++ b/src/io/transferors/local-path-to-local-path.cpp @@ -231,6 +231,11 @@ local_path_to_local_path_transferor::transfer( d_dst.canonical_path()); } +std::string +local_path_to_local_path_transferor::to_string() const { + return "transferor[local_path => local_path]"; +} + } // namespace io } // namespace norns diff --git a/src/io/transferors/local-path-to-local-path.hpp b/src/io/transferors/local-path-to-local-path.hpp index 62bc962..2d4f802 100644 --- a/src/io/transferors/local-path-to-local-path.hpp +++ b/src/io/transferors/local-path-to-local-path.hpp @@ -54,6 +54,7 @@ struct local_path_to_local_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; diff --git a/src/io/transferors/local-path-to-remote-path.cpp b/src/io/transferors/local-path-to-remote-path.cpp index 7425595..1f79d4c 100644 --- a/src/io/transferors/local-path-to-remote-path.cpp +++ b/src/io/transferors/local-path-to-remote-path.cpp @@ -71,5 +71,10 @@ local_path_to_remote_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::string +local_path_to_remote_path_transferor::to_string() const { + return "transferor[local_path => remote_path]"; +} + } // namespace io } // namespace norns diff --git a/src/io/transferors/local-path-to-remote-path.hpp b/src/io/transferors/local-path-to-remote-path.hpp index 89c6e04..db5fbd3 100644 --- a/src/io/transferors/local-path-to-remote-path.hpp +++ b/src/io/transferors/local-path-to-remote-path.hpp @@ -53,6 +53,7 @@ struct local_path_to_remote_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/local-path-to-shared-path.cpp b/src/io/transferors/local-path-to-shared-path.cpp index eb6bd20..f3f1a6f 100644 --- a/src/io/transferors/local-path-to-shared-path.cpp +++ b/src/io/transferors/local-path-to-shared-path.cpp @@ -71,7 +71,10 @@ local_path_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } - +std::string +local_path_to_shared_path_transferor::to_string() const { + return "transferor[local_path => shared_path]"; +} } // namespace io } // namespace norns diff --git a/src/io/transferors/local-path-to-shared-path.hpp b/src/io/transferors/local-path-to-shared-path.hpp index 13ebfe2..40eb978 100644 --- a/src/io/transferors/local-path-to-shared-path.hpp +++ b/src/io/transferors/local-path-to-shared-path.hpp @@ -53,6 +53,7 @@ struct local_path_to_shared_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-local-path.cpp b/src/io/transferors/memory-to-local-path.cpp index ad5a1e7..5c4edf3 100644 --- a/src/io/transferors/memory-to-local-path.cpp +++ b/src/io/transferors/memory-to-local-path.cpp @@ -183,6 +183,10 @@ memory_region_to_local_path_transferor::transfer( d_src.size(), d_dst.canonical_path()); } +std::string +memory_region_to_local_path_transferor::to_string() const { + return "transferor[memory_region => local_path]"; +} } // namespace io } // namespace norns diff --git a/src/io/transferors/memory-to-local-path.hpp b/src/io/transferors/memory-to-local-path.hpp index 2ec1b10..4102739 100644 --- a/src/io/transferors/memory-to-local-path.hpp +++ b/src/io/transferors/memory-to-local-path.hpp @@ -53,6 +53,7 @@ struct memory_region_to_local_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-remote-path.cpp b/src/io/transferors/memory-to-remote-path.cpp index 48ef8e6..5135c07 100644 --- a/src/io/transferors/memory-to-remote-path.cpp +++ b/src/io/transferors/memory-to-remote-path.cpp @@ -70,5 +70,10 @@ memory_region_to_remote_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::string +memory_region_to_remote_path_transferor::to_string() const { + return "transferor[memory_region => remote_path]"; +} + } // namespace io } // namespace norns diff --git a/src/io/transferors/memory-to-remote-path.hpp b/src/io/transferors/memory-to-remote-path.hpp index 24c40dd..393495d 100644 --- a/src/io/transferors/memory-to-remote-path.hpp +++ b/src/io/transferors/memory-to-remote-path.hpp @@ -53,6 +53,7 @@ struct memory_region_to_remote_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/memory-to-shared-path.cpp b/src/io/transferors/memory-to-shared-path.cpp index d02ff2a..c573bb2 100644 --- a/src/io/transferors/memory-to-shared-path.cpp +++ b/src/io/transferors/memory-to-shared-path.cpp @@ -70,6 +70,10 @@ memory_region_to_shared_path_transferor::transfer( return std::make_error_code(static_cast(0)); } +std::string +memory_region_to_shared_path_transferor::to_string() const { + return "transferor[memory_region => shared_path]"; +} } // namespace io } // namespace norns diff --git a/src/io/transferors/memory-to-shared-path.hpp b/src/io/transferors/memory-to-shared-path.hpp index 0cff99e..25d8e4e 100644 --- a/src/io/transferors/memory-to-shared-path.hpp +++ b/src/io/transferors/memory-to-shared-path.hpp @@ -53,6 +53,7 @@ struct memory_region_to_shared_path_transferor : public transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const override final; + std::string to_string() const override final; }; } // namespace io diff --git a/src/io/transferors/transferor.hpp b/src/io/transferors/transferor.hpp index b1e4890..4fd98f3 100644 --- a/src/io/transferors/transferor.hpp +++ b/src/io/transferors/transferor.hpp @@ -28,6 +28,7 @@ #ifndef __TRANSFEROR_BASE_HPP__ #define __TRANSFEROR_BASE_HPP__ +#include #include namespace norns { @@ -55,6 +56,8 @@ struct transferor { const std::shared_ptr& task_info, const std::shared_ptr& src, const std::shared_ptr& dst) const = 0; + + virtual std::string to_string() const = 0; }; } // namespace io diff --git a/src/namespaces/namespace-manager.hpp b/src/namespaces/namespace-manager.hpp index c89c344..2d60230 100644 --- a/src/namespaces/namespace-manager.hpp +++ b/src/namespaces/namespace-manager.hpp @@ -85,12 +85,12 @@ struct namespace_manager { } bool - contains(const std::string& nsid) { + contains(const std::string& nsid) const { return m_namespaces.count(nsid) != 0; } boost::optional> - find(const std::string& nsid, bool is_remote = false) { + find(const std::string& nsid, bool is_remote = false) const { if(is_remote) { return static_cast>( @@ -104,6 +104,27 @@ struct namespace_manager { return m_namespaces.at(nsid); } + std::tuple>> + find(const std::vector& nsids, + const std::vector& remotes) const { + + bool all_found = true; + + assert(nsids.size() == remotes.size()); + + std::vector> v; + + for(std::size_t i=0; iget<0>(); const auto src_rinfo = request->get<1>(); - const auto dst_rinfo = request->get<2>(); + const auto dst_rinfo = request->get<2>().get_value_or(nullptr); - response_ptr resp; - iotask_id tid = 0; + std::vector nsids; + std::vector remotes; + std::vector> backend_ptrs; + std::vector> rinfo_ptrs; + boost::optional tid; boost::optional auth; + response_ptr resp; urd_error rv = urd_error::success; if(m_is_paused) { @@ -231,67 +235,52 @@ response_ptr urd::iotask_create_handler(const request_ptr base_request) { goto log_and_return; } - //XXX move validate() to io:: - if((rv = validate_iotask_args(type, src_rinfo, dst_rinfo)) - == urd_error::success) { - - std::shared_ptr bsrc, bdst; - - { - boost::shared_lock lock(m_namespace_mgr_mutex); - - if(auto rv = m_namespace_mgr->find(src_rinfo->nsid(), - src_rinfo->is_remote())) { - bsrc = *rv; - } - - if(auto rv = m_namespace_mgr->find(dst_rinfo->nsid(), - dst_rinfo->is_remote())) { - bdst = *rv; - } + for(const auto& rinfo : {src_rinfo, dst_rinfo}) { + if(rinfo) { + nsids.push_back(rinfo->nsid()); + remotes.push_back(rinfo->is_remote()); + rinfo_ptrs.emplace_back(rinfo); } + } - if(!bsrc || !bdst) { - rv = urd_error::no_such_namespace; - goto log_and_return; - } +#ifdef __LOGGER_ENABLE_DEBUG__ + LOGGER_DEBUG("Request metadata:"); + LOGGER_DEBUG(" rinfos: {} {}", src_rinfo, dst_rinfo); - const auto tx_ptr = m_transferor_registry->get(src_rinfo->type(), - dst_rinfo->type()); + LOGGER_DEBUG(" nsids: {"); + for(std::size_t i=0; ivalidate(src_rinfo, dst_rinfo)) { - rv = urd_error::bad_args; - goto log_and_return; - } + { + bool all_found = false; + boost::shared_lock lock(m_namespace_mgr_mutex); + std::tie(all_found, backend_ptrs) = m_namespace_mgr->find(nsids, remotes); - // register the task in the task manager - auto ret = m_task_mgr->create_task(type, *auth, bsrc, src_rinfo, - bdst, dst_rinfo, std::move(tx_ptr)); - - if(!ret) { - // this can only happen if we tried to register a task - // and the TID automatically generated collided with an - // already running task. - // This can happen in two cases: - // 1. We end up with more than 4294967295U concurrent tasks - // 2. We are not properly cleaning up dead tasks - // In both cases, we want to know about it - LOGGER_CRITICAL("Failed to create new task!"); - rv = urd_error::too_many_tasks; + if(!all_found) { + rv = urd_error::no_such_namespace; goto log_and_return; } + } - tid = *ret; - rv = urd_error::success; +#ifdef __LOGGER_ENABLE_DEBUG__ + for(std::size_t i=0; icreate_task(type, *auth, backend_ptrs, rinfo_ptrs); log_and_return: - resp = std::make_unique(tid); + resp = std::make_unique(tid.get_value_or(0)); resp->set_error_code(rv); LOGGER_INFO("IOTASK_CREATE({}) = {}", request->to_string(), resp->to_string()); @@ -912,9 +901,18 @@ void urd::load_transfer_plugins() { const data::resource_type t2, std::shared_ptr&& trp) { - if(!m_transferor_registry->add(t1, t2, - std::forward>(trp))) { + // if(!m_transferor_registry->add(t1, t2, + // std::forward>(trp))) { + // LOGGER_WARN(" Failed to load transfer plugin ({} to {}):\n" + // " Another plugin was already registered for this " + // " combination (Plugin ignored)", + // utils::to_string(t1), utils::to_string(t2)); + // return; + // } + + if(!m_task_mgr->register_transfer_plugin(t1, t2, + std::forward>(trp))) { LOGGER_WARN(" Failed to load transfer plugin ({} to {}):\n" " Another plugin was already registered for this " " combination (Plugin ignored)", @@ -922,7 +920,8 @@ void urd::load_transfer_plugins() { return; } - LOGGER_INFO(" Loaded transfer plugin ({} to {})", + + LOGGER_INFO(" Loaded transfer plugin ({} => {})", utils::to_string(t1), utils::to_string(t2)); }; @@ -934,12 +933,12 @@ void urd::load_transfer_plugins() { // memory region -> shared path load_plugin(data::resource_type::memory_region, data::resource_type::shared_posix_path, - std::make_shared()); + std::make_shared()); // memory region -> remote path load_plugin(data::resource_type::memory_region, data::resource_type::remote_posix_path, - std::make_shared()); + std::make_shared()); // local path -> local path load_plugin(data::resource_type::local_posix_path, diff --git a/tests/Makefile.am b/tests/Makefile.am index cadfce6..2ff26e6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -50,6 +50,7 @@ api_SOURCES = \ api-namespace-register.cpp \ api-namespace-unregister.cpp \ api-copy-local-data.cpp \ + api-remove-local-data.cpp \ api-job-register.cpp \ api-job-update.cpp \ api-job-unregister.cpp \ diff --git a/tests/api-remove-local-data.cpp b/tests/api-remove-local-data.cpp new file mode 100644 index 0000000..907d945 --- /dev/null +++ b/tests/api-remove-local-data.cpp @@ -0,0 +1,718 @@ +/************************************************************************* + * Copyright (C) 2017-2018 Barcelona Supercomputing Center * + * Centro Nacional de Supercomputacion * + * All rights reserved. * + * * + * This file is part of the NORNS Data Scheduler, a service that allows * + * other programs to start, track and manage asynchronous transfers of * + * data resources transfers requests between different storage backends. * + * * + * See AUTHORS file in the top level directory for information * + * regarding developers and contributors. * + * * + * The NORNS Data Scheduler is free software: you can redistribute it * + * and/or modify it under the terms of the GNU Lesser General Public * + * License as published by the Free Software Foundation, either * + * version 3 of the License, or (at your option) any later version. * + * * + * The NORNS Data Scheduler is distributed in the hope that it will be * + * useful, but WITHOUT ANY WARRANTY; without even the implied warranty * + * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * + * Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General * + * Public License along with the NORNS Data Scheduler. If not, see * + * . * + *************************************************************************/ + +#include "norns.h" +#include "nornsctl.h" +#include "test-env.hpp" +#include "compare-files.hpp" +#include "catch.hpp" + +namespace bfs = boost::filesystem; + +SCENARIO("remove a local POSIX file", "[api::norns_submit_remove_local_posix_files]") { + GIVEN("a running urd instance") { + + test_env env; + + const char* nsid0 = "tmp0"; + const char* nsid1 = "tmp1"; + bfs::path src_mnt, dst_mnt; + + // create namespaces + std::tie(std::ignore, src_mnt) = env.create_namespace(nsid0, "mnt/tmp0", 16384); + std::tie(std::ignore, dst_mnt) = env.create_namespace(nsid1, "mnt/tmp1", 16384); + + // define input names + const bfs::path src_file_at_root = "/file0"; + const bfs::path src_file_at_subdir = "/a/b/c/d/file0"; + const bfs::path src_invalid_file = "/a/b/c/d/does_not_exist_file0"; + const bfs::path src_invalid_dir = "/a/b/c/d/does_not_exist_dir0"; + const bfs::path src_subdir0 = "/input_dir0"; + const bfs::path src_subdir1 = "/input_dir0/a/b/c/input_dir1"; + const bfs::path src_empty_dir = "/empty_dir0"; + + const bfs::path src_noperms_file0 = "/noperms_file0"; + const bfs::path src_noperms_file1 = "/noperms/a/b/c/d/noperms_file0"; // parents accessible + const bfs::path src_noperms_file2 = "/noperms/noperms_subdir0/file0"; // parents non-accessible + const bfs::path src_noperms_subdir0 = "/noperms_subdir0"; // subdir non-accessible + const bfs::path src_noperms_subdir1 = "/noperms/a/b/c/d/noperms_subdir1"; // child subdir non-accessible + const bfs::path src_noperms_subdir2 = "/noperms/noperms_subdir2/a"; // parent subdir non-accessible + + const bfs::path src_symlink_at_root0 = "/symlink0"; + const bfs::path src_symlink_at_root1 = "/symlink1"; + const bfs::path src_symlink_at_root2 = "/symlink2"; + const bfs::path src_symlink_at_subdir0 = "/foo/bar/baz/symlink0"; + const bfs::path src_symlink_at_subdir1 = "/foo/bar/baz/symlink1"; + const bfs::path src_symlink_at_subdir2 = "/foo/bar/baz/symlink2"; + + const bfs::path dst_root = "/"; + const bfs::path dst_subdir0 = "/output_dir0"; + const bfs::path dst_subdir1 = "/output_dir1"; + const bfs::path dst_file_at_root0 = "/file0"; // same basename + const bfs::path dst_file_at_root1 = "/file1"; // different basename + const bfs::path dst_file_at_subdir0 = "/a/b/c/d/file0"; // same fullname + const bfs::path dst_file_at_subdir1 = "/a/b/c/d/file1"; // same parents, different basename + const bfs::path dst_file_at_subdir2 = "/e/f/g/h/i/file0"; // different parents, same basename + const bfs::path dst_file_at_subdir3 = "/e/f/g/h/i/file1"; // different fullname + + // create input data + env.add_to_namespace(nsid0, src_file_at_root, 4096); + env.add_to_namespace(nsid0, src_file_at_subdir, 8192); + env.add_to_namespace(nsid0, src_subdir0); + env.add_to_namespace(nsid0, src_subdir1); + env.add_to_namespace(nsid0, src_empty_dir); + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir0 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + for(int i=0; i<10; ++i) { + const bfs::path p{src_subdir1 / ("file" + std::to_string(i))}; + env.add_to_namespace(nsid0, p, 4096+i*10); + } + + // create input data with special permissions + auto p = env.add_to_namespace(nsid0, src_noperms_file0, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file1, 0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_file2, 0); + env.remove_access(p.parent_path()); + + p = env.add_to_namespace(nsid0, src_noperms_subdir0); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir1); + env.remove_access(p); + + p = env.add_to_namespace(nsid0, src_noperms_subdir2); + env.remove_access(p.parent_path()); + + // add symlinks to the namespace + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_root0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_root1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_root2); + + env.add_to_namespace(nsid0, src_file_at_root, src_symlink_at_subdir0); + env.add_to_namespace(nsid0, src_subdir0, src_symlink_at_subdir1); + env.add_to_namespace(nsid0, src_subdir1, src_symlink_at_subdir2); + + // manually create a symlink leading outside namespace 0 + boost::system::error_code ec; + const bfs::path out_symlink = "/out_symlink"; + bfs::create_symlink(dst_mnt, src_mnt / out_symlink, ec); + REQUIRE(!ec); + + + // create required output directories + env.add_to_namespace(nsid1, dst_subdir1); + + /**************************************************************************************************************/ + /* tests for error conditions */ + /**************************************************************************************************************/ + // - trying to remove a non-existing file + WHEN("removing a non-existing NORNS_LOCAL_PATH file") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_invalid_file.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + + + // - trying to remove a non-existing directory + WHEN("removing a non-existing NORNS_LOCAL_PATH directory") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_invalid_dir.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and ENOENT are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(stats.st_sys_errno == ENOENT); + } + } + } + } + +//FIXME: DISABLED in CI until impersonation is implemented or capabilities can be added to the docker service +#ifdef __SETCAP_TESTS__ + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from \"/\" without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a file from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH file from a subdir without appropriate permissions to access a parent") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_file2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from \"/\" without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir0.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access it") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir1.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } + + // - trying to copy a subdir from namespace root with invalid access permissions + WHEN("copying a NORNS_LOCAL_PATH subdir from another subdir without appropriate permissions to access a parent") { + + norns_op_t task_op = NORNS_IOTASK_COPY; + + norns_iotask_t task = NORNS_IOTASK(task_op, + NORNS_LOCAL_PATH(nsid0, src_noperms_subdir2.c_str()), + NORNS_LOCAL_PATH(nsid1, dst_file_at_root0.c_str())); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("NORNS_ESYSTEMERROR and EACCES|EPERM|EINVAL are reported") { + norns_stat_t stats; + rv = norns_status(&task, &stats); + + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(stats.st_status == NORNS_EFINISHEDWERROR); + REQUIRE(stats.st_task_error == NORNS_ESYSTEMERROR); + REQUIRE(( (stats.st_sys_errno == EACCES) || + (stats.st_sys_errno == EPERM ) || + (stats.st_sys_errno == EINVAL) )); + } + } + } + } +#endif + + /**************************************************************************************************************/ + /* tests for single files */ + /**************************************************************************************************************/ + // rm ns0://file0.txt + WHEN("removing a single NORNS_LOCAL_PATH from src namespace's root") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_file_at_root.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, src_file_at_root); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("File no longer exists") { + REQUIRE(!bfs::exists(p)); + } + } + } + } + + // rm ns0://a/b/c/.../d/file0.txt + WHEN("removing a single NORNS_LOCAL_PATH from a src namespace's subdir") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_file_at_subdir.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, src_file_at_subdir); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("File no longer exists") { + REQUIRE(!bfs::exists(p)); + } + } + } + } + + /**************************************************************************************************************/ + /* tests for directories */ + /**************************************************************************************************************/ + // rm -r /a/contents.* + WHEN("removing the contents of a NORNS_LOCAL_PATH subdir from src namespace's root") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_subdir0.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, src_subdir0); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directory no longer exists") { + REQUIRE(!bfs::exists(p)); + } + } + } + } + + // rm -r /a/b/c/.../contents.* + WHEN("removing the contents of a NORNS_LOCAL_PATH arbitrary subdir to") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_subdir1.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, src_subdir1); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directory no longer exists") { + REQUIRE(!bfs::exists(p)); + } + } + } + } + + WHEN("removing an empty NORNS_LOCAL_PATH directory") { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_empty_dir.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, src_empty_dir); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("Directory no longer exists") { + REQUIRE(!bfs::exists(p)); + } + } + } + } + + + /**************************************************************************************************************/ + /* tests for soft links */ + /**************************************************************************************************************/ + WHEN("removing a single NORNS_LOCAL_PATH file from src namespace's '/' " + "through a symlink also located at '/'" ) { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_root0.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, "/") / src_symlink_at_root0; + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("symlink no longer exists and original file is left intact") { + REQUIRE(!bfs::exists(p)); + REQUIRE(bfs::exists(env.get_from_namespace(nsid0, src_file_at_root))); + } + } + } + } + + WHEN("removing a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink located at '/'" ) { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_root2.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, "/") / src_symlink_at_root2; + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("symlink no longer exists and original file is left intact") { + REQUIRE(!bfs::exists(p)); + REQUIRE(bfs::exists(env.get_from_namespace(nsid0, src_subdir1))); + } + } + } + } + + WHEN("removing a single NORNS_LOCAL_PATH file from src namespace's '/' " + "through a symlink located in a subdir" ) { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir0.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, "/") / src_symlink_at_subdir0; + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("symlink no longer exists and original file is left intact") { + REQUIRE(!bfs::exists(p)); + REQUIRE(bfs::exists(env.get_from_namespace(nsid0, src_file_at_root))); + } + } + } + } + + WHEN("removing a single NORNS_LOCAL_PATH subdir from src namespace's '/' " + "through a symlink also located at subdir" ) { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir1.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, "/") / src_symlink_at_subdir1; + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("symlink no longer exists and original file is left intact") { + REQUIRE(!bfs::exists(p)); + REQUIRE(bfs::exists(env.get_from_namespace(nsid0, src_subdir0))); + } + } + } + } + + WHEN("removing a single NORNS_LOCAL_PATH arbitrary subdir" + "through a symlink also located at a subdir" ) { + + norns_iotask_t task = NORNS_IOTASK(NORNS_IOTASK_REMOVE, + NORNS_LOCAL_PATH(nsid0, src_symlink_at_subdir2.c_str())); + const bfs::path p = env.get_from_namespace(nsid0, "/") / src_symlink_at_subdir2; + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + REQUIRE(task.t_id != 0); + + // wait until the task completes + rv = norns_wait(&task); + + THEN("NORNS_SUCCESS is returned") { + REQUIRE(rv == NORNS_SUCCESS); + + THEN("symlink no longer exists and original file is left intact") { + REQUIRE(!bfs::exists(p)); + REQUIRE(bfs::exists(env.get_from_namespace(nsid0, src_subdir1))); + } + } + } + } + + env.notify_success(); + } + + +#ifndef USE_REAL_DAEMON + GIVEN("a non-running urd instance") { + WHEN("attempting to request a transfer") { + + norns_iotask_t task = NORNS_IOTASK( + NORNS_IOTASK_COPY, + NORNS_LOCAL_PATH("nvml0://", "/a/b/c/"), + NORNS_REMOTE_PATH("nvml0://", "node1", "/a/b/d/")); + + norns_error_t rv = norns_submit(&task); + + THEN("NORNS_ECONNFAILED is returned") { + REQUIRE(rv == NORNS_ECONNFAILED); + } + } + } +#endif +} diff --git a/tests/api-task-init.cpp b/tests/api-task-init.cpp index 9a53a1d..15e6b48 100644 --- a/tests/api-task-init.cpp +++ b/tests/api-task-init.cpp @@ -68,26 +68,27 @@ SCENARIO("initialize a task with norns_iotask_init", "[api::norns_iotask_init]") } } - GIVEN("invalid task information") { - WHEN("initializing a task with a NULL dst") { - - norns_iotask_t task; - norns_op_t task_op = NORNS_IOTASK_COPY; - - void* src_addr = (void*) 0xdeadbeef; - size_t src_size = (size_t) 42; - - norns_resource_t src = NORNS_MEMORY_REGION(src_addr, src_size); - - norns_iotask_init(&task, task_op, &src, NULL); - - THEN("task is set to 0") { - norns_iotask_t dummy; - memset(&dummy, 0, sizeof(dummy)); - REQUIRE(memcmp(&task, &dummy, sizeof(norns_iotask_t)) == 0); - } - } - } +///XXX this case is no longer invalid +/// GIVEN("invalid task information") { +/// WHEN("initializing a task with a NULL dst") { +/// +/// norns_iotask_t task; +/// norns_op_t task_op = NORNS_IOTASK_COPY; +/// +/// void* src_addr = (void*) 0xdeadbeef; +/// size_t src_size = (size_t) 42; +/// +/// norns_resource_t src = NORNS_MEMORY_REGION(src_addr, src_size); +/// +/// norns_iotask_init(&task, task_op, &src, NULL); +/// +/// THEN("task is set to 0") { +/// norns_iotask_t dummy; +/// memset(&dummy, 0, sizeof(dummy)); +/// REQUIRE(memcmp(&task, &dummy, sizeof(norns_iotask_t)) == 0); +/// } +/// } +/// } GIVEN("valid task information") { WHEN("initializing a task with src=NORNS_PROCESS_MEMORY and dst=NORNS_POSIX_PATH | R_LOCAL") { -- GitLab From 2ded5b761223e48a5fb8a421704999dc8127ed32 Mon Sep 17 00:00:00 2001 From: Alberto Miranda Date: Mon, 17 Sep 2018 15:58:38 +0200 Subject: [PATCH 3/3] Update .gitlab-ci.yml with new tests --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1655851..6d7b6e8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -156,6 +156,7 @@ test:ubuntu:latest: - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_status]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_status]" - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::nornsctl_send_command]" + - NORNS_DEBUG_CONFIG_FILE_OVERRIDE=1 ./api -as "[api::norns_submit_remove_local_posix_files]" after_script: - pwd - if [[ -e tests.log ]]; -- GitLab