Implemented remote single file transfers (305b9a92) · Commits · hpc / norns

.gitmodules

0 → 100644

+3 −0

Original line number	Diff line number	Diff line
		[submodule "src/externals/hermes"]
		path = src/externals/hermes
		url = ssh://git@bscssrg.bsc.es:2222/hpc/hermes.git

configure.ac

+3 −0

Original line number	Diff line number	Diff line
		@@ -119,6 +119,9 @@ AS_IF([test "x$is_enabled_build_tests" = "xyes"],
		AC_CONFIG_FILES(tests/Makefile)
		], [])

		# check for mercury
		PKG_CHECK_MODULES([MERCURY], [mercury >= 0.26])

		# check for libyaml-cpp
		PKG_CHECK_MODULES([YAMLCPP], [yaml-cpp >= 0.5.1])

src/Makefile.am

+25 −3

Original line number	Diff line number	Diff line
		@@ -48,6 +48,11 @@ liburd_resources_la_SOURCES = \
		resources/memory_buffer/detail/memory-region-impl.hpp \
		resources/memory_buffer/detail/memory-region-info.cpp \
		resources/memory_buffer/detail/memory-region-info.hpp \
		resources/remote_resource/remote-resource.hpp \
		resources/remote_resource/detail/remote-resource-impl.cpp \
		resources/remote_resource/detail/remote-resource-impl.hpp \
		resources/remote_resource/detail/remote-resource-info.cpp \
		resources/remote_resource/detail/remote-resource-info.hpp \
		resources/remote_posix_path/remote-path.hpp \
		resources/remote_posix_path/detail/remote-path-impl.cpp \
		resources/remote_posix_path/detail/remote-path-impl.hpp \
		@@ -67,9 +72,11 @@ liburd_resources_la_CXXFLAGS = \

		liburd_resources_la_CPPFLAGS = \
		-DSPDLOG_ENABLE_SYSLOG \
		-DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \
		@BOOST_CPPFLAGS@ \
		-I$(top_srcdir)/include \
		-I$(top_srcdir)/src \
		-I$(top_srcdir)/src/externals/hermes/include \
		-I$(top_srcdir)/src/resources \
		-I$(top_srcdir)/rpc \
		-I$(top_builddir)/rpc
		@@ -81,6 +88,7 @@ liburd_resources_la_LDFLAGS = \
		@BOOST_PROGRAM_OPTIONS_LIB@ \
		@BOOST_SYSTEM_LIB@ \
		@BOOST_THREAD_LIB@ \
		@MERCURY_LIBS@ \
		@PROTOBUF_LIBS@ \
		-pthread

		@@ -137,12 +145,16 @@ liburd_aux_la_SOURCES = \
		config/parsers.hpp \
		config/defaults.hpp \
		io.hpp \
		io/task.cpp \
		io/task.hpp \
		io/task-copy.hpp \
		io/task-info.cpp \
		io/task-info.hpp \
		io/task-manager.cpp \
		io/task-manager.hpp \
		io/task-move.hpp \
		io/task-noop.hpp \
		io/task-remote-transfer.hpp \
		io/task-remove.hpp \
		io/task-stats.cpp \
		io/task-stats.hpp \
		io/transferors.hpp \
		@@ -151,8 +163,10 @@ liburd_aux_la_SOURCES = \
		io/transferors/local-path-to-local-path.hpp \
		io/transferors/local-path-to-shared-path.cpp \
		io/transferors/local-path-to-shared-path.hpp \
		io/transferors/local-path-to-remote-path.cpp \
		io/transferors/local-path-to-remote-path.hpp \
		io/transferors/local-path-to-remote-resource.cpp \
		io/transferors/local-path-to-remote-resource.hpp \
		io/transferors/remote-resource-to-local-path.cpp \
		io/transferors/remote-resource-to-local-path.hpp \
		io/transferors/memory-to-local-path.cpp \
		io/transferors/memory-to-local-path.hpp \
		io/transferors/memory-to-shared-path.cpp \
		@@ -164,6 +178,8 @@ liburd_aux_la_SOURCES = \
		job.hpp \
		logger.hpp \
		resources.hpp \
		rpcs.cpp \
		rpcs.hpp \
		signal-listener.hpp \
		thread-pool.hpp \
		thread-pool/thread-pool.hpp \
		@@ -183,9 +199,11 @@ liburd_aux_la_CXXFLAGS = \

		liburd_aux_la_CPPFLAGS = \
		-DSPDLOG_ENABLE_SYSLOG \
		-DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \
		@BOOST_CPPFLAGS@ \
		-I$(top_srcdir)/include \
		-I$(top_srcdir)/src \
		-I$(top_srcdir)/src/externals/hermes/include \
		-I$(top_srcdir)/src/resources \
		-I$(top_srcdir)/rpc \
		-I$(top_builddir)/rpc
		@@ -197,6 +215,7 @@ liburd_aux_la_LDFLAGS = \
		@BOOST_PROGRAM_OPTIONS_LIB@ \
		@BOOST_SYSTEM_LIB@ \
		@BOOST_THREAD_LIB@ \
		@MERCURY_LIBS@ \
		@PROTOBUF_LIBS@ \
		@YAMLCPP_LIBS@ \
		liburd_resources.la \
		@@ -256,9 +275,11 @@ urd_CXXFLAGS = \

		urd_CPPFLAGS = \
		-DSPDLOG_ENABLE_SYSLOG \
		-DHERMES_DISABLE_INTERNAL_MAKE_UNIQUE \
		@BOOST_CPPFLAGS@ \
		-I$(top_srcdir)/include \
		-I$(top_srcdir)/src \
		-I$(top_srcdir)/src/externals/hermes/include \
		-I$(top_srcdir)/rpc \
		-I$(top_builddir)/rpc

		@@ -269,6 +290,7 @@ urd_LDFLAGS = \
		@BOOST_PROGRAM_OPTIONS_LIB@ \
		@BOOST_SYSTEM_LIB@ \
		@BOOST_THREAD_LIB@ \
		@MERCURY_LIBS@ \
		@PROTOBUF_LIBS@ \
		liburd_aux.la

src/api/request.cpp

+8 −3

Original line number	Diff line number	Diff line
		@@ -166,6 +166,7 @@ create_from(const norns::rpc::Request_Task_Resource& res) {
		using norns::data::local_path_info;
		using norns::data::shared_path_info;
		using norns::data::remote_path_info;
		using norns::data::remote_resource_info;

		assert(is_valid(res));

		@@ -187,8 +188,12 @@ create_from(const norns::rpc::Request_Task_Resource& res) {

		// R_REMOTE
		assert(res.type() & R_REMOTE);
		return std::make_shared<remote_path_info>(res.path().nsid(),
		res.path().hostname(),
		//return std::make_shared<remote_path_info>(res.path().nsid(),
		// res.path().hostname(),
		// res.path().datapath());

		return std::make_shared<remote_resource_info>(res.path().hostname(),
		res.path().nsid(),
		res.path().datapath());
		}

src/backends.hpp

+5 −2

Original line number	Diff line number	Diff line
		@@ -38,8 +38,11 @@

		namespace norns {
		namespace storage {
		static const auto process_memory_backend = std::make_shared<detail::process_memory>();
		static const auto remote_backend = std::make_shared<detail::remote_backend>();

		constexpr static const auto memory_nsid = "[[internal::memory]]";

		static const auto process_memory_backend = std::make_shared<detail::process_memory>(memory_nsid);
		// static const auto remote_backend = std::make_shared<detail::remote_backend>();
		} // namespace storage
		} // namespace norns