Merge branch 'rnou/111-integrate-with-cargo' into 'main' (4c72718e) · Commits · eu / ADMIRE / scord

CMakeLists.txt

+2 −2

Original line number	Diff line number	Diff line
		@@ -157,7 +157,7 @@ set(SCORD_CTL_BIND_PORT
		)
		message(STATUS "[${PROJECT_NAME}] server bind port: ${SCORD_CTL_BIND_PORT}")

		set(CARGO_PORT
		set(CARGO_BIND_PORT
		"62000"
		CACHE STRING
		"Define the port through wich we should commmunicate with Cargo"
		@@ -303,7 +303,7 @@ find_package(RedisPlusPlus 1.3.3 REQUIRED)

		### Cargo: required for transferring datasets between storage tiers
		message(STATUS "[${PROJECT_NAME}] Checking for Cargo")
		find_package(Cargo 0.2.0 REQUIRED)
		find_package(Cargo 0.3.1 REQUIRED)


		# ##############################################################################

etc/scord-ctl.conf.in

+138 −0

Original line number	Diff line number	Diff line
		@@ -35,7 +35,145 @@ config:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		shrink
		--hosts {ADHOC_NODES}
		dataclay:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/dataclay
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		shrink
		--hosts {ADHOC_NODES}

		expand:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/expand
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		shrink
		--hosts {ADHOC_NODES}

		hercules:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/hercules
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		shrink
		--hosts {ADHOC_NODES}

		# default storage tiers made available to applications
		storage:

examples/CMakeLists.txt

+17 −0

Original line number	Diff line number	Diff line
		@@ -36,6 +36,9 @@ if(SCORD_BUILD_TESTS)
		set(SCORD_ADDRESS_STRING
		${SCORD_TRANSPORT_PROTOCOL}://${SCORD_BIND_ADDRESS}:${SCORD_BIND_PORT})

		set(DATA_STAGER_ADDRESS_STRING
		${SCORD_TRANSPORT_PROTOCOL}://${SCORD_BIND_ADDRESS}:${CARGO_BIND_PORT})

		add_test(start_scord_daemon
		${CMAKE_SOURCE_DIR}/scripts/runner.sh start scord.pid
		${CMAKE_BINARY_DIR}/src/scord/scord -f -c ${CMAKE_CURRENT_BINARY_DIR}/scord.conf
		@@ -73,6 +76,20 @@ if(SCORD_BUILD_TESTS)

		set_tests_properties(stop_scord_ctl PROPERTIES FIXTURES_CLEANUP scord_ctl)


		add_test(start_cargo
		${CMAKE_SOURCE_DIR}/scripts/runner.sh start cargo.pid
		mpirun --allow-run-as-root -n 2 ${CARGO_BIN_INSTALL_DIR}/cargo -l ${DATA_STAGER_ADDRESS_STRING} -o ${TEST_DIRECTORY}/cargo.log
		)
		set_tests_properties(start_cargo
		PROPERTIES FIXTURES_SETUP cargo)

		add_test(stop_cargo
		${CMAKE_SOURCE_DIR}/scripts/runner.sh stop TERM cargo.pid)

		set_tests_properties(stop_cargo
		PROPERTIES FIXTURES_CLEANUP cargo)

		endif()

		add_subdirectory(c)

examples/c/ADM_cancel_transfer.c

+15 −12

Original line number	Diff line number	Diff line
		@@ -28,11 +28,6 @@
		#include <assert.h>
		#include "common.h"

		#define NJOB_NODES 50
		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -40,6 +35,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -55,10 +51,15 @@ main(int argc, char* argv[]) {
		assert(job_nodes);
		ADM_node_t* adhoc_nodes = prepare_nodes(NADHOC_NODES);
		assert(adhoc_nodes);
		ADM_dataset_t* inputs = prepare_datasets("input-dataset-%d", NINPUTS);
		ADM_dataset_route_t* inputs =
		prepare_routes("%s-input-dataset-%d", NINPUTS);
		assert(inputs);
		ADM_dataset_t* outputs = prepare_datasets("output-dataset-%d", NOUTPUTS);
		ADM_dataset_route_t* outputs =
		prepare_routes("%s-output-dataset-%d", NOUTPUTS);
		assert(outputs);
		ADM_dataset_route_t* expected_outputs =
		prepare_routes("%s-exp-output-dataset-%d", NEXPOUTPUTS);
		assert(expected_outputs);

		ADM_job_resources_t job_resources =
		ADM_job_resources_create(job_nodes, NJOB_NODES);
		@@ -69,8 +70,8 @@ main(int argc, char* argv[]) {
		assert(adhoc_resources);

		ADM_adhoc_context_t ctx = ADM_adhoc_context_create(
		cli_args.controller_address, ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);
		assert(ctx);

		const char* name = "adhoc_storage_42";
		@@ -90,7 +91,8 @@ main(int argc, char* argv[]) {
		}

		ADM_job_requirements_t reqs = ADM_job_requirements_create(
		inputs, NINPUTS, outputs, NOUTPUTS, adhoc_storage);
		inputs, NINPUTS, outputs, NOUTPUTS, expected_outputs, NEXPOUTPUTS,
		adhoc_storage);
		assert(reqs);

		uint64_t slurm_job_id = 42;
		@@ -143,7 +145,8 @@ main(int argc, char* argv[]) {
		cleanup:
		ADM_remove_job(server, job);
		ADM_server_destroy(server);
		destroy_datasets(inputs, NINPUTS);
		destroy_datasets(outputs, NOUTPUTS);
		destroy_routes(inputs, NINPUTS);
		destroy_routes(outputs, NOUTPUTS);
		destroy_routes(expected_outputs, NEXPOUTPUTS);
		exit(exit_status);
		}

examples/c/ADM_connect_data_operation.c

+16 −12

Original line number	Diff line number	Diff line
		@@ -28,11 +28,6 @@
		#include <assert.h>
		#include "common.h"

		#define NJOB_NODES 50
		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -40,6 +35,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -55,18 +51,23 @@ main(int argc, char* argv[]) {
		assert(job_nodes);
		ADM_node_t* adhoc_nodes = prepare_nodes(NADHOC_NODES);
		assert(adhoc_nodes);
		ADM_dataset_t* inputs = prepare_datasets("input-dataset-%d", NINPUTS);
		ADM_dataset_route_t* inputs =
		prepare_routes("%s-input-dataset-%d", NINPUTS);
		assert(inputs);
		ADM_dataset_t* outputs = prepare_datasets("output-dataset-%d", NOUTPUTS);
		ADM_dataset_route_t* outputs =
		prepare_routes("%s-output-dataset-%d", NOUTPUTS);
		assert(outputs);
		ADM_dataset_route_t* expected_outputs =
		prepare_routes("%s-exp-output-dataset-%d", NEXPOUTPUTS);
		assert(expected_outputs);

		ADM_adhoc_resources_t adhoc_resources =
		ADM_adhoc_resources_create(adhoc_nodes, NADHOC_NODES);
		assert(adhoc_resources);

		ADM_adhoc_context_t ctx = ADM_adhoc_context_create(
		cli_args.controller_address, ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);
		assert(ctx);

		const char* name = "adhoc_storage_42";
		@@ -90,7 +91,8 @@ main(int argc, char* argv[]) {
		assert(job_resources);

		ADM_job_requirements_t reqs = ADM_job_requirements_create(
		inputs, NINPUTS, outputs, NOUTPUTS, adhoc_storage);
		inputs, NINPUTS, outputs, NOUTPUTS, expected_outputs, NEXPOUTPUTS,
		adhoc_storage);
		assert(reqs);

		uint64_t slurm_job_id = 42;
		@@ -108,8 +110,8 @@ main(int argc, char* argv[]) {
		exit_status = EXIT_SUCCESS;

		bool should_stream = false;
		ret = ADM_connect_data_operation(server, job, inputs[0], outputs[0],
		should_stream);
		ret = ADM_connect_data_operation(server, job, /inputs[0]/ NULL,
		/outputs[0]/ NULL, should_stream);


		if(ret != ADM_SUCCESS) {
		@@ -125,6 +127,7 @@ main(int argc, char* argv[]) {
		"successfully\n");

		cleanup:
		/*
		for(int i = 0; i < NINPUTS; ++i) {
		ADM_dataset_destroy(inputs[i]);
		}
		@@ -132,6 +135,7 @@ cleanup:
		for(int i = 0; i < NOUTPUTS; ++i) {
		ADM_dataset_destroy(outputs[i]);
		}
		*/

		ADM_remove_job(server, job);
		ADM_server_destroy(server);