Use gkfs.io MD5 helpers in malleability perf tests (012106da) · Commits · hpc / gekkofs

tests/integration/harness/CMakeLists.txt

+3 −0

Original line number	Diff line number	Diff line
		@@ -56,6 +56,7 @@ add_executable(gkfs.io
		gkfs.io/lseek.cpp
		gkfs.io/write_validate.cpp
		gkfs.io/write_random.cpp
		gkfs.io/random_md5.cpp
		gkfs.io/truncate.cpp
		gkfs.io/util/file_compare.cpp
		gkfs.io/chdir.cpp
		@@ -74,6 +75,7 @@ add_executable(gkfs.io
		)

		include(load_nlohmann_json)
		find_package(OpenSSL REQUIRED)

		target_include_directories(gkfs.io PRIVATE
		${BOOST_PREPROCESSOR_INCLUDE_DIRS}
		@@ -84,6 +86,7 @@ target_link_libraries(gkfs.io
		fmt::fmt
		CLI11::CLI11
		std::filesystem
		OpenSSL::Crypto
		rt
		)

tests/integration/harness/gkfs.io/commands.hpp

+6 −0

Original line number	Diff line number	Diff line
		@@ -103,6 +103,12 @@ directory_validate_init(CLI::App& app);
		void
		write_random_init(CLI::App& app);

		void
		write_random_and_md5_init(CLI::App& app);

		void
		read_random_and_md5_init(CLI::App& app);

		void
		truncate_init(CLI::App& app);

tests/integration/harness/gkfs.io/main.cpp

+2 −0

Original line number	Diff line number	Diff line
		@@ -67,6 +67,8 @@ init_commands(CLI::App& app) {
		write_validate_init(app);
		directory_validate_init(app);
		write_random_init(app);
		write_random_and_md5_init(app);
		read_random_and_md5_init(app);
		truncate_init(app);
		access_init(app);
		statfs_init(app);

tests/integration/harness/gkfs.io/random_md5.cpp

0 → 100644

+263 −0

Original line number	Diff line number	Diff line
		/*
		Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain
		Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany

		SPDX-License-Identifier: GPL-3.0-or-later
		*/

		#include <CLI/CLI.hpp>
		#include <commands.hpp>
		#include <fmt/format.h>
		#include <nlohmann/json.hpp>
		#include <reflection.hpp>
		#include <serialize.hpp>

		#include <algorithm>
		#include <array>
		#include <cerrno>
		#include <cstdint>
		#include <cstring>
		#include <memory>
		#include <random>
		#include <string>
		#include <vector>

		#include <fcntl.h>
		#include <openssl/evp.h>
		#include <sys/stat.h>
		#include <sys/types.h>
		#include <unistd.h>

		using json = nlohmann::json;

		namespace {

		constexpr std::size_t buffer_size = 1024 * 1024;

		constexpr std::size_t md5_digest_length = 16;

		std::string
		md5_to_hex(const unsigned char* digest, unsigned int digest_len) {
		std::string hex;
		hex.reserve(digest_len * 2);
		for(unsigned int i = 0; i < digest_len; ++i) {
		hex += fmt::format("{:02x}", digest[i]);
		}
		return hex;
		}

		class md5_stream {
		public:
		md5_stream() : ctx_(EVP_MD_CTX_new()) {
		if(ctx_ == nullptr \|\| EVP_DigestInit_ex(ctx_, EVP_md5(), nullptr) != 1) {
		ok_ = false;
		}
		}

		~md5_stream() {
		if(ctx_ != nullptr) {
		EVP_MD_CTX_free(ctx_);
		}
		}

		bool
		update(const void* data, std::size_t size) {
		if(!ok_) {
		return false;
		}
		if(EVP_DigestUpdate(ctx_, data, size) != 1) {
		ok_ = false;
		}
		return ok_;
		}

		std::string
		final() {
		if(!ok_) {
		return "";
		}
		std::array<unsigned char, EVP_MAX_MD_SIZE> digest{};
		unsigned int digest_len = 0;
		if(EVP_DigestFinal_ex(ctx_, digest.data(), &digest_len) != 1) {
		return "";
		}
		if(digest_len != md5_digest_length) {
		return "";
		}
		return md5_to_hex(digest.data(), digest_len);
		}

		private:
		EVP_MD_CTX* ctx_;
		bool ok_{true};
		};

		void
		fill_random(std::vector<unsigned char>& buffer, std::mt19937_64& rng) {
		std::size_t pos = 0;
		while(pos < buffer.size()) {
		auto value = rng();
		for(std::size_t i = 0; i < sizeof(value) && pos < buffer.size(); ++i) {
		buffer[pos++] = static_cast<unsigned char>((value >> (i * 8U)) & 0xffU);
		}
		}
		}

		} // namespace

		struct random_md5_options {
		bool verbose{};
		std::string pathname{};
		::size_t count{};
		uint64_t seed{42};

		REFL_DECL_STRUCT(random_md5_options, REFL_DECL_MEMBER(bool, verbose),
		REFL_DECL_MEMBER(std::string, pathname),
		REFL_DECL_MEMBER(::size_t, count),
		REFL_DECL_MEMBER(uint64_t, seed));
		};

		struct random_md5_output {
		::ssize_t retval;
		int errnum;
		std::string md5;

		REFL_DECL_STRUCT(random_md5_output, REFL_DECL_MEMBER(::ssize_t, retval),
		REFL_DECL_MEMBER(int, errnum),
		REFL_DECL_MEMBER(std::string, md5));
		};

		void
		to_json(json& record, const random_md5_output& out) {
		record = serialize(out);
		}

		void
		write_random_and_md5_exec(const random_md5_options& opts) {
		auto fd = ::open(opts.pathname.c_str(), O_CREAT \| O_WRONLY \| O_TRUNC,
		S_IRWXU \| S_IRWXG \| S_IRWXO);
		if(fd == -1) {
		json out = random_md5_output{-1, errno, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}

		md5_stream md5_ctx;

		std::mt19937_64 rng(opts.seed);
		std::vector<unsigned char> buffer(buffer_size);
		std::size_t remaining = opts.count;
		::ssize_t written_total = 0;
		int err = 0;

		while(remaining > 0) {
		const auto step = std::min<std::size_t>(buffer.size(), remaining);
		if(buffer.size() != step) {
		buffer.resize(step);
		}
		fill_random(buffer, rng);
		if(!md5_ctx.update(buffer.data(), step)) {
		::close(fd);
		json out = random_md5_output{-1, EIO, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}

		std::size_t done = 0;
		while(done < step) {
		auto rv = ::write(fd, buffer.data() + done, step - done);
		if(rv < 0) {
		err = errno;
		::close(fd);
		json out = random_md5_output{-1, err, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}
		if(rv == 0) {
		err = EIO;
		::close(fd);
		json out = random_md5_output{-1, err, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}
		done += static_cast<std::size_t>(rv);
		written_total += rv;
		}
		remaining -= step;
		}

		if(::close(fd) != 0 && err == 0) {
		err = errno;
		}

		json out = random_md5_output{err == 0 ? written_total : -1, err,
		md5_ctx.final()};
		fmt::print("{}\n", out.dump(2));
		}

		void
		read_random_and_md5_exec(const random_md5_options& opts) {
		auto fd = ::open(opts.pathname.c_str(), O_RDONLY);
		if(fd == -1) {
		json out = random_md5_output{-1, errno, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}

		md5_stream md5_ctx;

		std::vector<unsigned char> buffer(buffer_size);
		::ssize_t read_total = 0;
		int err = 0;

		for(;;) {
		auto rv = ::read(fd, buffer.data(), buffer.size());
		if(rv < 0) {
		err = errno;
		::close(fd);
		json out = random_md5_output{-1, err, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}
		if(rv == 0) {
		break;
		}
		if(!md5_ctx.update(buffer.data(), static_cast<std::size_t>(rv))) {
		::close(fd);
		json out = random_md5_output{-1, EIO, ""};
		fmt::print("{}\n", out.dump(2));
		return;
		}
		read_total += rv;
		}

		if(::close(fd) != 0 && err == 0) {
		err = errno;
		}

		json out = random_md5_output{err == 0 ? read_total : -1, err,
		md5_ctx.final()};
		fmt::print("{}\n", out.dump(2));
		}

		void
		write_random_and_md5_init(CLI::App& app) {
		auto opts = std::make_shared<random_md5_options>();
		auto* cmd = app.add_subcommand("write_random_and_md5",
		"Create/truncate a file, write deterministic random data, and return MD5");
		cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output");
		cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name("");
		cmd->add_option("count", opts->count, "Number of bytes to write")->required()->type_name("");
		cmd->add_option("--seed", opts->seed, "Pseudo-random seed")->type_name("");
		cmd->callback([opts]() { write_random_and_md5_exec(*opts); });
		}

		void
		read_random_and_md5_init(CLI::App& app) {
		auto opts = std::make_shared<random_md5_options>();
		auto* cmd = app.add_subcommand("read_random_and_md5",
		"Read a full file and return MD5");
		cmd->add_flag("-v,--verbose", opts->verbose, "Produce human readable output");
		cmd->add_option("pathname", opts->pathname, "File name")->required()->type_name("");
		cmd->callback([opts]() { read_random_and_md5_exec(*opts); });
		}
		No newline at end of file

tests/integration/harness/io.py

+14 −0

Original line number	Diff line number	Diff line
		@@ -373,6 +373,18 @@ class WriteRandomOutputSchema(Schema):
		return namedtuple('WriteRandomReturn', ['retval', 'errno'])(**data)


		class RandomMd5OutputSchema(Schema):
		"""Schema to deserialize random write/read with MD5 helper output"""

		retval = fields.Integer(required=True)
		errno = Errno(data_key='errnum', required=True)
		md5 = fields.String(required=True)

		@post_load
		def make_object(self, data, **kwargs):
		return namedtuple('RandomMd5Return', ['retval', 'errno', 'md5'])(**data)


		class WriteSyncOutputSchema(Schema):
		"""Schema to deserialize the results of a write_sync() execution"""

		@@ -516,6 +528,8 @@ class IOParser:
		'statx' : StatxOutputSchema(),
		'lseek' : LseekOutputSchema(),
		'write_random': WriteRandomOutputSchema(),
		'write_random_and_md5': RandomMd5OutputSchema(),
		'read_random_and_md5': RandomMd5OutputSchema(),
		'write_validate' : WriteValidateOutputSchema(),
		'write_validate' : WriteValidateOutputSchema(),
		'write_sequential' : WriteValidateOutputSchema(),