integrate with cargo (!118) · Merge requests · eu / ADMIRE / scord

etc/scord-ctl.conf.in

+138 −0

Changes for etc/scord-ctl.conf.in: 138 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -35,7 +35,145 @@ config:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/gekkofs.sh
		shrink
		--hosts {ADHOC_NODES}
		dataclay:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/dataclay
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/dataclay.sh
		shrink
		--hosts {ADHOC_NODES}

		expand:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/expand
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/expand.sh
		shrink
		--hosts {ADHOC_NODES}

		hercules:
		# The default working directory for adhoc instances of this type
		working_directory: /tmp/hercules
		startup:
		# Specific environment variables that should be set for the adhoc
		# instance. These will be merged with the environment variables
		# already set by Slurm.
		environment:
		VAR0: value0
		VAR1: value1
		# The command that `scord-ctl` will use to start an adhoc instance of
		# this type. The following variables are supported that will be
		# automatically replaced by scord-ctl if found between curly braces:
		# * ADHOC_NODES: A comma separated list of valid job hostnames that
		# can be used to start the adhoc instance.
		# * ADHOC_DIRECTORY: A unique working directory for each specific
		# adhoc instance. This directory will be created by scord-ctl under
		# `working_directory` and automatically removed after the adhoc
		# instance has been shut down.
		# * ADHOC_ID: - A unique ID for the adhoc instance.
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		start
		--hosts {ADHOC_NODES}
		--workdir {ADHOC_DIRECTORY}
		--datadir {ADHOC_DIRECTORY}/data
		--mountdir {ADHOC_DIRECTORY}/mnt
		shutdown:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		stop
		--workdir {ADHOC_DIRECTORY}
		expand:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		expand
		--hosts {ADHOC_NODES}

		shrink:
		environment:
		command: @CMAKE_INSTALL_FULL_DATADIR@/@PROJECT_NAME@/adhoc_services.d/hercules.sh
		shrink
		--hosts {ADHOC_NODES}

		# default storage tiers made available to applications
		storage:

examples/c/ADM_cancel_transfer.c

+15 −12

Changes for examples/c/ADM_cancel_transfer.c: 15 added lines, 12 removed lines.
Original line number	Diff line number	Diff line
		@@ -28,11 +28,6 @@
		#include <assert.h>
		#include "common.h"

		#define NJOB_NODES 50
		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -40,6 +35,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -55,10 +51,15 @@ main(int argc, char* argv[]) {
		assert(job_nodes);
		ADM_node_t* adhoc_nodes = prepare_nodes(NADHOC_NODES);
		assert(adhoc_nodes);
		ADM_dataset_t* inputs = prepare_datasets("input-dataset-%d", NINPUTS);
		ADM_dataset_route_t* inputs =
		prepare_routes("%s-input-dataset-%d", NINPUTS);
		assert(inputs);
		ADM_dataset_t* outputs = prepare_datasets("output-dataset-%d", NOUTPUTS);
		ADM_dataset_route_t* outputs =
		prepare_routes("%s-output-dataset-%d", NOUTPUTS);
		assert(outputs);
		ADM_dataset_route_t* expected_outputs =
		prepare_routes("%s-exp-output-dataset-%d", NEXPOUTPUTS);
		assert(expected_outputs);

		ADM_job_resources_t job_resources =
		ADM_job_resources_create(job_nodes, NJOB_NODES);
		@@ -69,8 +70,8 @@ main(int argc, char* argv[]) {
		assert(adhoc_resources);

		ADM_adhoc_context_t ctx = ADM_adhoc_context_create(
		cli_args.controller_address, ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);
		assert(ctx);

		const char* name = "adhoc_storage_42";
		@@ -90,7 +91,8 @@ main(int argc, char* argv[]) {
		}

		ADM_job_requirements_t reqs = ADM_job_requirements_create(
		inputs, NINPUTS, outputs, NOUTPUTS, adhoc_storage);
		inputs, NINPUTS, outputs, NOUTPUTS, expected_outputs, NEXPOUTPUTS,
		adhoc_storage);
		assert(reqs);

		uint64_t slurm_job_id = 42;
		@@ -143,7 +145,8 @@ main(int argc, char* argv[]) {
		cleanup:
		ADM_remove_job(server, job);
		ADM_server_destroy(server);
		destroy_datasets(inputs, NINPUTS);
		destroy_datasets(outputs, NOUTPUTS);
		destroy_routes(inputs, NINPUTS);
		destroy_routes(outputs, NOUTPUTS);
		destroy_routes(expected_outputs, NEXPOUTPUTS);
		exit(exit_status);
		}

examples/c/ADM_connect_data_operation.c

+16 −12

Changes for examples/c/ADM_connect_data_operation.c: 16 added lines, 12 removed lines.
Original line number	Diff line number	Diff line
		@@ -28,11 +28,6 @@
		#include <assert.h>
		#include "common.h"

		#define NJOB_NODES 50
		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -40,6 +35,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -55,18 +51,23 @@ main(int argc, char* argv[]) {
		assert(job_nodes);
		ADM_node_t* adhoc_nodes = prepare_nodes(NADHOC_NODES);
		assert(adhoc_nodes);
		ADM_dataset_t* inputs = prepare_datasets("input-dataset-%d", NINPUTS);
		ADM_dataset_route_t* inputs =
		prepare_routes("%s-input-dataset-%d", NINPUTS);
		assert(inputs);
		ADM_dataset_t* outputs = prepare_datasets("output-dataset-%d", NOUTPUTS);
		ADM_dataset_route_t* outputs =
		prepare_routes("%s-output-dataset-%d", NOUTPUTS);
		assert(outputs);
		ADM_dataset_route_t* expected_outputs =
		prepare_routes("%s-exp-output-dataset-%d", NEXPOUTPUTS);
		assert(expected_outputs);

		ADM_adhoc_resources_t adhoc_resources =
		ADM_adhoc_resources_create(adhoc_nodes, NADHOC_NODES);
		assert(adhoc_resources);

		ADM_adhoc_context_t ctx = ADM_adhoc_context_create(
		cli_args.controller_address, ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);
		assert(ctx);

		const char* name = "adhoc_storage_42";
		@@ -90,7 +91,8 @@ main(int argc, char* argv[]) {
		assert(job_resources);

		ADM_job_requirements_t reqs = ADM_job_requirements_create(
		inputs, NINPUTS, outputs, NOUTPUTS, adhoc_storage);
		inputs, NINPUTS, outputs, NOUTPUTS, expected_outputs, NEXPOUTPUTS,
		adhoc_storage);
		assert(reqs);

		uint64_t slurm_job_id = 42;
		@@ -108,8 +110,8 @@ main(int argc, char* argv[]) {
		exit_status = EXIT_SUCCESS;

		bool should_stream = false;
		ret = ADM_connect_data_operation(server, job, inputs[0], outputs[0],
		should_stream);
		ret = ADM_connect_data_operation(server, job, /inputs[0]/ NULL,
		/outputs[0]/ NULL, should_stream);


		if(ret != ADM_SUCCESS) {
		@@ -125,6 +127,7 @@ main(int argc, char* argv[]) {
		"successfully\n");

		cleanup:
		/*
		for(int i = 0; i < NINPUTS; ++i) {
		ADM_dataset_destroy(inputs[i]);
		}
		@@ -132,6 +135,7 @@ cleanup:
		for(int i = 0; i < NOUTPUTS; ++i) {
		ADM_dataset_destroy(outputs[i]);
		}
		*/

		ADM_remove_job(server, job);
		ADM_server_destroy(server);

examples/c/ADM_define_data_operation.c

+12 −10

Changes for examples/c/ADM_define_data_operation.c: 12 added lines, 10 removed lines.
Original line number	Diff line number	Diff line
		@@ -28,11 +28,6 @@
		#include <assert.h>
		#include "common.h"

		#define NJOB_NODES 50
		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -40,6 +35,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -56,10 +52,15 @@ main(int argc, char* argv[]) {
		assert(job_nodes);
		ADM_node_t* adhoc_nodes = prepare_nodes(NADHOC_NODES);
		assert(adhoc_nodes);
		ADM_dataset_t* inputs = prepare_datasets("input-dataset-%d", NINPUTS);
		ADM_dataset_route_t* inputs =
		prepare_routes("%s-input-dataset-%d", NINPUTS);
		assert(inputs);
		ADM_dataset_t* outputs = prepare_datasets("output-dataset-%d", NOUTPUTS);
		ADM_dataset_route_t* outputs =
		prepare_routes("%s-output-dataset-%d", NOUTPUTS);
		assert(outputs);
		ADM_dataset_route_t* expected_outputs =
		prepare_routes("%s-exp-output-dataset-%d", NEXPOUTPUTS);
		assert(expected_outputs);

		ADM_job_resources_t job_resources =
		ADM_job_resources_create(job_nodes, NJOB_NODES);
		@@ -70,8 +71,8 @@ main(int argc, char* argv[]) {
		assert(adhoc_resources);

		ADM_adhoc_context_t ctx = ADM_adhoc_context_create(
		cli_args.controller_address, ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);
		assert(ctx);

		const char* name = "adhoc_storage_42";
		@@ -91,7 +92,8 @@ main(int argc, char* argv[]) {
		}

		ADM_job_requirements_t reqs = ADM_job_requirements_create(
		inputs, NINPUTS, outputs, NOUTPUTS, adhoc_storage);
		inputs, NINPUTS, outputs, NOUTPUTS, expected_outputs, NEXPOUTPUTS,
		adhoc_storage);
		assert(reqs);

		uint64_t slurm_job_id = 42;

examples/c/ADM_deploy_adhoc_storage.c

+7 −10

Changes for examples/c/ADM_deploy_adhoc_storage.c: 7 added lines, 10 removed lines.
Original line number	Diff line number	Diff line
		@@ -27,10 +27,6 @@
		#include <scord/scord.h>
		#include "common.h"

		#define NADHOC_NODES 25
		#define NINPUTS 10
		#define NOUTPUTS 5

		int
		main(int argc, char* argv[]) {

		@@ -38,6 +34,7 @@ main(int argc, char* argv[]) {
		.name = TESTNAME,
		.requires_server = true,
		.requires_controller = true,
		.requires_data_stager = true,
		};

		cli_args_t cli_args;
		@@ -82,9 +79,9 @@ main(int argc, char* argv[]) {
		}

		// 3. the adhoc storage execution context
		adhoc_ctx = ADM_adhoc_context_create(cli_args.controller_address,
		ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 100, false);
		adhoc_ctx = ADM_adhoc_context_create(
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 100, false);

		if(adhoc_ctx == NULL) {
		fprintf(stderr, "Fatal error preparing adhoc context\n");
		@@ -115,9 +112,9 @@ main(int argc, char* argv[]) {
		// system, let's prepare a new execution context for the adhoc
		// storage system

		new_adhoc_ctx = ADM_adhoc_context_create(cli_args.controller_address,
		ADM_ADHOC_MODE_SEPARATE_NEW,
		ADM_ADHOC_ACCESS_RDWR, 200, false);
		new_adhoc_ctx = ADM_adhoc_context_create(
		cli_args.controller_address, cli_args.data_stager_address,
		ADM_ADHOC_MODE_SEPARATE_NEW, ADM_ADHOC_ACCESS_RDWR, 200, false);

		if(new_adhoc_ctx == NULL) {
		fprintf(stderr, "Fatal error preparing new adhoc context\n");

integrate with cargo

Merge request reports