diff --git a/CHANGELOG.md b/CHANGELOG.md index ab565a45082ece8f595df175dcc627452e939e21..1264486e942a4eadbba6847bdde27dce50b52a2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### New + - Metadata batching ([!305](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/305)) + - Added client-side metadata batching for file/node creation to reduce metadata RPC bottlenecks. + - Introduced new environment variables: `LIBGKFS_METADATA_BATCH` and `LIBGKFS_METADATA_BATCH_THRESHOLD`. - directory optimization with compression and reattemp ([!270](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/270)) - Refactor sfind so it can use SLURM_ environment variables to ask to different servers. - Create a sample bash script to gather all the info (map->reduce) diff --git a/README.md b/README.md index 183829293f543386ea4b21f74dcf8f964df855a8..93e0c29ae9d6f5882b81fbc0a1109a33fb340ee6 100644 --- a/README.md +++ b/README.md @@ -738,6 +738,14 @@ Using two environment variables - `LIBGKFS_PROTECT_FILES_GENERATOR=1` enables the application as generator, so a open will create (and increase) a file .lockgekko and close will remove it (or decrease its value). The behaviour only uses metadata at server side. - `LIBGKFS_PROTECT_FILES_CONSUMER=1` enables the application as consumer, so a open will wait until the .lockgekko dissapears. The wait is limited to ~40 seconds. +##### Metadata batching +During file/node creation (e.g., `open` with `O_CREAT`), the client sends a separate RPC for each file creation. The metadata batching feature allows buffering these creation operations on the client-side and sending them in batches to the daemons, reducing network RPC overhead during massive file creations. + +Remaining buffered creation requests are automatically flushed when the application exits. + +- `LIBGKFS_METADATA_BATCH=ON` - Enable client-side metadata batching for file creation (default: OFF). +- `LIBGKFS_METADATA_BATCH_THRESHOLD` - Set the number of file creation operations per host after which the batch is flushed (default: 64). + ### Daemon #### Core - `GKFS_DAEMON_CREATE_CHECK_PARENTS` - Enable checking parent directory for existence before creating children. diff --git a/include/client/env.hpp b/include/client/env.hpp index 7dc9fa5393abe4c5a8257c99215f8cee591fcfe7..5da3df9dfc41d39a832384706c106071eb907290 100644 --- a/include/client/env.hpp +++ b/include/client/env.hpp @@ -95,6 +95,9 @@ static constexpr auto CREATE_WRITE_OPTIMIZATION = ADD_PREFIX("CREATE_WRITE_OPTIMIZATION"); static constexpr auto READ_INLINE_PREFETCH = ADD_PREFIX("READ_INLINE_PREFETCH"); static constexpr auto ENABLE_FORK = ADD_PREFIX("ENABLE_FORK"); +static constexpr auto METADATA_BATCH = ADD_PREFIX("METADATA_BATCH"); +static constexpr auto METADATA_BATCH_THRESHOLD = + ADD_PREFIX("METADATA_BATCH_THRESHOLD"); } // namespace gkfs::env diff --git a/include/client/preload_context.hpp b/include/client/preload_context.hpp index 780140684b836ee0f0a360699e82f4d2e7e53d63..844421f70a372b6f77654f9a008383c203f2dfd9 100644 --- a/include/client/preload_context.hpp +++ b/include/client/preload_context.hpp @@ -41,6 +41,7 @@ #define GEKKOFS_PRELOAD_CTX_HPP #include +#include #include #include #include @@ -156,6 +157,12 @@ private: std::shared_ptr rpc_engine_; std::shared_ptr ipc_engine_; + bool use_metadata_batch_{false}; + size_t metadata_batch_threshold_{64}; + std::unordered_map>> + metadata_batch_buffer_; + mutable std::mutex metadata_batch_mutex_; + public: static PreloadContext* @@ -375,6 +382,28 @@ public: void ipc_engine(std::shared_ptr engine); + + bool + use_metadata_batch() const; + + void + use_metadata_batch(bool use_metadata_batch); + + size_t + metadata_batch_threshold() const; + + void + metadata_batch_threshold(size_t threshold); + + void + flush_metadata_batches(); + + void + flush_metadata_batch(uint64_t host_id); + + void + add_metadata_batch_entry(uint64_t host_id, const std::string& path, + mode_t mode); }; } // namespace preload diff --git a/include/client/rpc/forward_metadata.hpp b/include/client/rpc/forward_metadata.hpp index 7242da6d7bdb308908ae6eea144d2326a2f95d93..cf7d1e5747392105c7d8f38e26c79672eeb03c7a 100644 --- a/include/client/rpc/forward_metadata.hpp +++ b/include/client/rpc/forward_metadata.hpp @@ -63,6 +63,10 @@ namespace rpc { int forward_create(const std::string& path, mode_t mode, const int copy); +int +forward_batch_create(uint64_t host_id, const std::vector& paths, + const std::vector& modes); + int forward_create_write_inline(const std::string& path, mode_t mode, const std::string& data, uint64_t count, diff --git a/include/common/common_defs.hpp b/include/common/common_defs.hpp index d92b80c60282a430e73d075cd04ca23989097fc8..897842a3c73688fe240c4c9f44ba16796c7e7225 100644 --- a/include/common/common_defs.hpp +++ b/include/common/common_defs.hpp @@ -72,6 +72,7 @@ constexpr auto get_dirents_extended = "rpc_srv_get_dirents_extended"; constexpr auto get_dirents_filtered = "rpc_srv_get_dirents_filtered"; constexpr auto mk_symlink = "rpc_srv_mk_symlink"; constexpr auto rename = "rpc_srv_rename"; +constexpr auto batch_create = "rpc_srv_batch_mk_node"; constexpr auto write = "rpc_srv_write_data"; constexpr auto read = "rpc_srv_read_data"; diff --git a/include/common/rpc/rpc_types_thallium.hpp b/include/common/rpc/rpc_types_thallium.hpp index 95f2122b6421360a2099fa35e50c58d2c1e08155..8fd9fdf7fbb45cb634522e74df07a36f09612ee2 100644 --- a/include/common/rpc/rpc_types_thallium.hpp +++ b/include/common/rpc/rpc_types_thallium.hpp @@ -73,6 +73,28 @@ struct rpc_mk_node_in_t { } }; +struct rpc_batch_mk_node_in_t { + std::vector paths; + std::vector modes; + + template + void + serialize(Archive& ar) { + ar(paths, modes); + } +}; + +struct rpc_batch_mk_node_out_t { + int32_t err; + std::vector errs; + + template + void + serialize(Archive& ar) { + ar(err, errs); + } +}; + struct rpc_path_only_in_t { std::string path; bool include_inline; diff --git a/include/daemon/handler/rpc_defs.hpp b/include/daemon/handler/rpc_defs.hpp index 748d0d65a49b507738a1336ed017493d708172bf..81d39011ee30d2fa20b607427352e093b9e4289c 100644 --- a/include/daemon/handler/rpc_defs.hpp +++ b/include/daemon/handler/rpc_defs.hpp @@ -51,6 +51,10 @@ void rpc_srv_create(const tl::request& req, const gkfs::rpc::rpc_mk_node_in_t& in); +void +rpc_srv_batch_create(const tl::request& req, + const gkfs::rpc::rpc_batch_mk_node_in_t& in); + void rpc_srv_stat(const tl::request& req, const gkfs::rpc::rpc_path_only_in_t& in); diff --git a/src/client/preload.cpp b/src/client/preload.cpp index ca3b1742f3118d5912a8956957ca72247603597a..4fdbe4262ab95faa1e5b8e794cc3d777da717a5b 100644 --- a/src/client/preload.cpp +++ b/src/client/preload.cpp @@ -344,6 +344,24 @@ init_environment() { srand(time(nullptr)); } + auto use_batch = + gkfs::env::get_var(gkfs::env::METADATA_BATCH, "OFF") == "ON"; + if(use_batch) { + CTX->use_metadata_batch(true); + auto batch_threshold_str = + gkfs::env::get_var(gkfs::env::METADATA_BATCH_THRESHOLD, "64"); + try { + CTX->metadata_batch_threshold(std::stoul(batch_threshold_str)); + } catch(...) { + CTX->metadata_batch_threshold(64); + } + LOG(INFO, "Metadata batching enabled with threshold: {}", + CTX->metadata_batch_threshold()); + } else { + CTX->use_metadata_batch(false); + LOG(INFO, "Metadata batching disabled."); + } + LOG(INFO, "Environment initialization successful."); } @@ -507,6 +525,11 @@ destroy_preload() { std::lock_guard lock(internal_mutex); if(init.exchange(false) == false) return; // Prevent double destruction + + if(CTX->use_metadata_batch()) { + LOG(INFO, "Flushing final metadata batches..."); + CTX->flush_metadata_batches(); + } auto forwarding_map_file = gkfs::env::get_var( gkfs::env::FORWARDING_MAP_FILE, gkfs::config::forwarding_file_path); if(!forwarding_map_file.empty()) { diff --git a/src/client/preload_context.cpp b/src/client/preload_context.cpp index cdb06b54d49b6a99f45d8d274ca8b5eb72348ebc..5c27fa31aac6b3cb2e5f144cc104e43989c29850 100644 --- a/src/client/preload_context.cpp +++ b/src/client/preload_context.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -794,5 +795,97 @@ PreloadContext::dirents_buff_size(size_t size) { dirents_buff_size_ = size; } +bool +PreloadContext::use_metadata_batch() const { + return use_metadata_batch_; +} + +void +PreloadContext::use_metadata_batch(bool use_metadata_batch) { + use_metadata_batch_ = use_metadata_batch; +} + +size_t +PreloadContext::metadata_batch_threshold() const { + return metadata_batch_threshold_; +} + +void +PreloadContext::metadata_batch_threshold(size_t threshold) { + metadata_batch_threshold_ = threshold; +} + +void +PreloadContext::flush_metadata_batches() { + std::vector, + std::vector>>> + batches; + { + std::lock_guard lock(metadata_batch_mutex_); + for(auto& [host_id, queue] : metadata_batch_buffer_) { + if(!queue.empty()) { + std::vector paths; + std::vector modes; + paths.reserve(queue.size()); + modes.reserve(queue.size()); + for(const auto& entry : queue) { + paths.push_back(entry.first); + modes.push_back(entry.second); + } + queue.clear(); + batches.emplace_back(host_id, std::make_pair(std::move(paths), + std::move(modes))); + } + } + } + + if(!batches.empty()) { + LOG(DEBUG, "{}() Flushing all {} batches...", __func__, batches.size()); + } + + for(const auto& [host_id, data] : batches) { + gkfs::rpc::forward_batch_create(host_id, data.first, data.second); + } +} + +void +PreloadContext::flush_metadata_batch(uint64_t host_id) { + std::vector paths; + std::vector modes; + { + std::lock_guard lock(metadata_batch_mutex_); + auto it = metadata_batch_buffer_.find(host_id); + if(it != metadata_batch_buffer_.end() && !it->second.empty()) { + paths.reserve(it->second.size()); + modes.reserve(it->second.size()); + for(const auto& entry : it->second) { + paths.push_back(entry.first); + modes.push_back(entry.second); + } + it->second.clear(); + } + } + + if(!paths.empty()) { + LOG(INFO, "{}() Flushing batch of size {} to host {}", __func__, + paths.size(), host_id); + gkfs::rpc::forward_batch_create(host_id, paths, modes); + } +} + +void +PreloadContext::add_metadata_batch_entry(uint64_t host_id, + const std::string& path, mode_t mode) { + size_t queue_size = 0; + { + std::lock_guard lock(metadata_batch_mutex_); + metadata_batch_buffer_[host_id].emplace_back(path, mode); + queue_size = metadata_batch_buffer_[host_id].size(); + } + if(queue_size >= metadata_batch_threshold_) { + flush_metadata_batch(host_id); + } +} + } // namespace preload } // namespace gkfs diff --git a/src/client/rpc/forward_metadata.cpp b/src/client/rpc/forward_metadata.cpp index 44ef5cafc6faaf58d02164ef7395f677168a6290..2208139ad5cf5d3d9fddf5e827c1366e9cfff7d8 100644 --- a/src/client/rpc/forward_metadata.cpp +++ b/src/client/rpc/forward_metadata.cpp @@ -52,8 +52,14 @@ forward_create(const std::string& path, const mode_t mode, const int copy) { LOG(ERROR, "{}() Distributor not initialized!", __func__); return ENOTCONN; } - auto endp = CTX->hosts().at( - CTX->distributor()->locate_file_metadata(path, copy)); + auto host_id = CTX->distributor()->locate_file_metadata(path, copy); + + if(CTX->use_metadata_batch()) { + CTX->add_metadata_batch_entry(host_id, path, mode); + return 0; + } + + auto endp = CTX->hosts().at(host_id); if(!CTX->rpc_engine()) { LOG(ERROR, "{}() RPC engine not initialized!", __func__); @@ -71,6 +77,41 @@ forward_create(const std::string& path, const mode_t mode, const int copy) { return out.err; } +int +forward_batch_create(uint64_t host_id, const std::vector& paths, + const std::vector& modes) { + if(paths.empty()) { + return 0; + } + + if(!CTX->rpc_engine()) { + LOG(ERROR, "{}() RPC engine not initialized!", __func__); + return ENOTCONN; + } + + auto endp = CTX->hosts().at(host_id); + + gkfs::rpc::rpc_batch_mk_node_in_t in; + in.paths = paths; + in.modes = modes; + + LOG(DEBUG, "{}() Sending batch creation RPC with {} entries to host {}...", + __func__, paths.size(), host_id); + + auto out = gkfs::rpc::forward_call( + CTX->rpc_engine(), endp, gkfs::rpc::tag::batch_create, in, __func__, + paths.front()); + + for(size_t i = 0; i < out.errs.size(); ++i) { + if(out.errs[i] != 0) { + LOG(WARNING, "{}() server returned error {} for path '{}'", + __func__, out.errs[i], paths[i]); + } + } + + return 0; +} + int forward_create_write_inline(const std::string& path, mode_t mode, const std::string& data, uint64_t count, diff --git a/src/daemon/backend/metadata/rocksdb_backend.cpp b/src/daemon/backend/metadata/rocksdb_backend.cpp index 8649ba275d4c0570c407ff5d5d1f7d3e9dd49df7..bf0ece2539eb6bd373855db4bbaadc50e90995fc 100644 --- a/src/daemon/backend/metadata/rocksdb_backend.cpp +++ b/src/daemon/backend/metadata/rocksdb_backend.cpp @@ -53,12 +53,80 @@ #include #include #include +#include +#include extern "C" { #include } namespace gkfs::metadata { +// Fast-path parsing: extracts only the first few mandatory integer fields +bool +parse_metadata_fast(std::string_view val, mode_t& mode, size_t& size, + time_t& ctime, blkcnt_t& blocks) { + auto next_field = [](std::string_view& s, auto& val) -> bool { + size_t pos = s.find('|'); + std::string_view field = + (pos == std::string_view::npos) ? s : s.substr(0, pos); + if(field.empty()) + return false; + + auto res = + std::from_chars(field.data(), field.data() + field.size(), val); + if(res.ec != std::errc{}) + return false; + + if(pos != std::string_view::npos) { + s.remove_prefix(pos + 1); + } else { + s = {}; + } + return true; + }; + + std::string_view s = val; + unsigned int parsed_mode = 0; + if(!next_field(s, parsed_mode)) + return false; + mode = parsed_mode; + + long long parsed_size = 0; + if(!next_field(s, parsed_size)) + return false; + size = parsed_size; + + if constexpr(gkfs::config::metadata::use_atime) { + long long dummy; + if(!next_field(s, dummy)) + return false; + } + if constexpr(gkfs::config::metadata::use_mtime) { + long long dummy; + if(!next_field(s, dummy)) + return false; + } + if constexpr(gkfs::config::metadata::use_ctime) { + long long parsed_ctime = 0; + if(!next_field(s, parsed_ctime)) + return false; + ctime = parsed_ctime; + } + if constexpr(gkfs::config::metadata::use_link_cnt) { + unsigned long long dummy; + if(!next_field(s, dummy)) + return false; + } + if constexpr(gkfs::config::metadata::use_blocks) { + long long parsed_blocks = 0; + if(!next_field(s, parsed_blocks)) + return false; + blocks = parsed_blocks; + } + + return true; +} + /** * Called when the daemon is started: Connects to the KV store * @param path where KV store data is stored @@ -462,24 +530,38 @@ RocksDBBackend::get_dirents_extended_impl(const std::string& dir, continue; } - Metadata md(it->value().ToString()); + mode_t mode = 0; + size_t size = 0; + time_t ctime = 0; + blkcnt_t blocks = 0; + + std::string_view val_view(it->value().data(), it->value().size()); + + if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { + Metadata md(it->value().ToString()); + mode = md.mode(); + size = md.size(); + ctime = md.ctime(); + blocks = md.blocks(); + } + if(gkfs::config::metadata::rename_support) { // Remove entries with negative blocks (rename) - if(md.blocks() == -1) { + if(blocks == -1) { continue; } } unsigned char type = 0; - if(S_ISDIR(md.mode())) { + if(S_ISDIR(mode)) { type = 1; - } else if(S_ISLNK(md.mode())) { + } else if(S_ISLNK(mode)) { type = 2; } - entries.emplace_back(std::forward_as_tuple(std::move(name), type, - md.size(), md.ctime())); + entries.emplace_back( + std::forward_as_tuple(std::move(name), type, size, ctime)); } assert(it->status().ok()); return entries; @@ -535,10 +617,24 @@ RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, continue; } - Metadata md(it->value().ToString()); + mode_t mode = 0; + size_t size = 0; + time_t ctime = 0; + blkcnt_t blocks = 0; + + std::string_view val_view(it->value().data(), it->value().size()); + + if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { + Metadata md(it->value().ToString()); + mode = md.mode(); + size = md.size(); + ctime = md.ctime(); + blocks = md.blocks(); + } + if(gkfs::config::metadata::rename_support) { // Remove entries with negative blocks (rename) - if(md.blocks() == -1) { + if(blocks == -1) { continue; } } @@ -546,14 +642,14 @@ RocksDBBackend::get_all_dirents_extended_impl(const std::string& dir, // 0: regular, 1: directory, 2: symlink unsigned char type = 0; - if(S_ISDIR(md.mode())) { + if(S_ISDIR(mode)) { type = 1; - } else if(S_ISLNK(md.mode())) { + } else if(S_ISLNK(mode)) { type = 2; } - entries.emplace_back(std::forward_as_tuple(std::move(name), type, - md.size(), md.ctime())); + entries.emplace_back( + std::forward_as_tuple(std::move(name), type, size, ctime)); } assert(it->status().ok()); return entries; @@ -632,10 +728,24 @@ RocksDBBackend::get_dirents_filtered_impl( scanned_count++; - Metadata md(it->value().ToString()); + mode_t mode = 0; + size_t size = 0; + time_t ctime = 0; + blkcnt_t blocks = 0; + + std::string_view val_view(it->value().data(), it->value().size()); + + if(!parse_metadata_fast(val_view, mode, size, ctime, blocks)) { + Metadata md(it->value().ToString()); + mode = md.mode(); + size = md.size(); + ctime = md.ctime(); + blocks = md.blocks(); + } + if(gkfs::config::metadata::rename_support) { // Remove entries with negative blocks (rename) - if(md.blocks() == -1) { + if(blocks == -1) { continue; } } @@ -646,11 +756,10 @@ RocksDBBackend::get_dirents_filtered_impl( matched = false; } } - if(matched && filter_size != -1 && md.size() != (size_t) filter_size) { + if(matched && filter_size != -1 && size != (size_t) filter_size) { matched = false; } - if(matched && filter_ctime != -1 && - md.ctime() < (time_t) filter_ctime) { + if(matched && filter_ctime != -1 && ctime < (time_t) filter_ctime) { matched = false; } @@ -665,13 +774,13 @@ RocksDBBackend::get_dirents_filtered_impl( "", type, 0, 0)); // Dummy entry to keep track of size/pagination } else { - if(S_ISDIR(md.mode())) { + if(S_ISDIR(mode)) { type = 1; - } else if(S_ISLNK(md.mode())) { + } else if(S_ISLNK(mode)) { type = 2; } entries.emplace_back(std::forward_as_tuple( - std::move(relative_name), type, md.size(), md.ctime())); + std::move(relative_name), type, size, ctime)); } if(max_entries > 0 && entries.size() >= max_entries) { eof = false; diff --git a/src/daemon/daemon.cpp b/src/daemon/daemon.cpp index 450a42e55c1df94442487d3b49cfcfbabcc8c08b..e752a1c34f4ee98392b590db7ebf3ae137f1134d 100644 --- a/src/daemon/daemon.cpp +++ b/src/daemon/daemon.cpp @@ -161,6 +161,7 @@ register_server_rpcs(std::shared_ptr engine) { // Metadata RPCs engine->define(gkfs::rpc::tag::create, rpc_srv_create); + engine->define(gkfs::rpc::tag::batch_create, rpc_srv_batch_create); engine->define(gkfs::rpc::tag::stat, rpc_srv_stat); engine->define(gkfs::rpc::tag::remove_metadata, rpc_srv_remove_metadata); engine->define(gkfs::rpc::tag::remove_data, rpc_srv_remove_data); diff --git a/src/daemon/handler/srv_metadata.cpp b/src/daemon/handler/srv_metadata.cpp index c6081deed7eae9d6a399e5287a1e3bc12ec8778c..cf042e47fb817022acf87cebcfd940d44c73954e 100644 --- a/src/daemon/handler/srv_metadata.cpp +++ b/src/daemon/handler/srv_metadata.cpp @@ -109,6 +109,42 @@ rpc_srv_create(const tl::request& req, const gkfs::rpc::rpc_mk_node_in_t& in) { } } +void +rpc_srv_batch_create(const tl::request& req, + const gkfs::rpc::rpc_batch_mk_node_in_t& in) { + GKFS_DATA->spdlogger()->debug("{}() Got RPC with {} paths", __func__, + in.paths.size()); + gkfs::rpc::run_rpc_handler( + req, in, + [](const gkfs::rpc::rpc_batch_mk_node_in_t& in, + gkfs::rpc::rpc_batch_mk_node_out_t& out) { + out.errs.resize(in.paths.size()); + out.err = 0; + for(size_t i = 0; i < in.paths.size(); ++i) { + try { + gkfs::metadata::Metadata md(in.modes[i]); + gkfs::metadata::create(in.paths[i], md); + out.errs[i] = 0; + } catch(const gkfs::metadata::ExistsException& e) { + out.errs[i] = EEXIST; + } catch(const gkfs::metadata::NotFoundException& e) { + out.errs[i] = ENOENT; + } catch(const gkfs::metadata::DBException& e) { + out.errs[i] = EIO; + } catch(const std::exception& e) { + out.errs[i] = EBUSY; + } + } + }); + if(GKFS_DATA->enable_stats()) { + for(size_t i = 0; i < in.paths.size(); ++i) { + GKFS_DATA->stats()->add_value_iops( + gkfs::utils::Stats::IopsOp::iops_create); + } + } +} + /** * @brief Serves a stat request or returns an error to the * client if the object does not exist. diff --git a/tests/integration/directories/test_sfind.py b/tests/integration/directories/test_sfind.py index e5bd6433573d9fca913856b8266cc848b709de20..1fd17873f3d0cc01bc674ee365601a0048fd8c56 100644 --- a/tests/integration/directories/test_sfind.py +++ b/tests/integration/directories/test_sfind.py @@ -41,7 +41,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): pop_client_env = { "LIBGKFS_USE_DIRENTS_COMPRESSION": "OFF", "LIBGKFS_DENTRY_CACHE": "OFF", - "GKFS_LOG": "info" + "LIBGKFS_LOG": "info" } client = ShellClient(test_workspace) @@ -80,7 +80,7 @@ def test_sfind_permutations(test_workspace, request, conf, buff_size): "LIBGKFS_USE_DIRENTS_COMPRESSION": conf["compress"], "LIBGKFS_DENTRY_CACHE": conf["cache"], "LIBGKFS_DIRENTS_BUFF_SIZE": buff_size, - "GKFS_LOG": "info" + "LIBGKFS_LOG": "info" } test_env_str = "\n".join([f"export {k}={v}" for k,v in test_client_env.items()])