tests/integration/harness/gkfs.io/main.cpp

@@ -39,6 +39,10 @@ init_commands(CLI::App& app) {
     #endif
     lseek_init(app);
     write_validate_init(app);
+    write_random_init(app);
+    truncate_init(app);
+    // util
+    file_compare_init(app);
 }
 
 

tests/integration/harness/gkfs.io/truncate.cpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+/* C++ includes */
+#include <CLI/CLI.hpp>
+#include <nlohmann/json.hpp>
+#include <memory>
+#include <fmt/format.h>
+#include <reflection.hpp>
+#include <serialize.hpp>
+
+/* C includes */
+#include <sys/types.h>
+#include <unistd.h>
+
+using json = nlohmann::json;
+
+struct truncate_options {
+    bool verbose{};
+    std::string path{};
+    ::off_t length{};
+
+    REFL_DECL_STRUCT(truncate_options,
+        REFL_DECL_MEMBER(bool, verbose),
+        REFL_DECL_MEMBER(std::string, path),
+        REFL_DECL_MEMBER(::off_t, length)
+    );
+};
+
+struct truncate_output {
+    int retval;
+    int errnum;
+
+    REFL_DECL_STRUCT(truncate_output,
+        REFL_DECL_MEMBER(int, retval),
+        REFL_DECL_MEMBER(int, errnum)
+    );
+};
+
+void
+to_json(json& record, 
+        const truncate_output& out) {
+    record = serialize(out);
+}
+
+void 
+truncate_exec(const truncate_options& opts) {
+
+    auto rv = ::truncate(opts.path.c_str(), opts.length);
+    if (rv == -1) {
+        if (opts.verbose) {
+            fmt::print("truncate(path=\"{}\", length={}) = {}, errno: {} [{}]\n",
+                       opts.path, opts.length, rv, errno, ::strerror(errno));
+            return;
+        }
+    }
+
+    json out = truncate_output{rv, errno};
+    fmt::print("{}\n", out.dump(2));
+}
+
+void
+truncate_init(CLI::App& app) {
+
+    // Create the option and subcommand objects
+    auto opts = std::make_shared<truncate_options>();
+    auto* cmd = app.add_subcommand(
+            "truncate",
+            "Execute the truncate() system call");
+
+    // Add options to cmd, binding them to opts
+    cmd->add_flag(
+            "-v,--verbose",
+            opts->verbose,
+            "Produce human writeable output"
+        );
+
+    cmd->add_option(
+            "path",
+            opts->path,
+            "Path to file"
+        )
+        ->required()
+        ->type_name("");
+
+    cmd->add_option(
+            "length",
+            opts->length,
+            "Truncate to a size precisely length bytes"
+        )
+        ->required()
+        ->type_name("");
+
+    cmd->callback([opts]() { 
+        truncate_exec(*opts);
+    });
+}
+
+

tests/integration/harness/gkfs.io/util/file_compare.cpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+/* C++ includes */
+#include <CLI/CLI.hpp>
+#include <nlohmann/json.hpp>
+#include <binary_buffer.hpp>
+#include <memory>
+#include <fmt/format.h>
+#include <reflection.hpp>
+#include <serialize.hpp>
+
+/* C includes */
+#include <sys/types.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+using json = nlohmann::json;
+
+struct file_compare_options {
+    bool verbose{};
+    std::string path_1{};
+    std::string path_2{};
+    size_t count{};
+
+    REFL_DECL_STRUCT(file_compare_options,
+                     REFL_DECL_MEMBER(bool, verbose),
+                     REFL_DECL_MEMBER(std::string, path_1),
+                     REFL_DECL_MEMBER(std::string, path_2),
+                     REFL_DECL_MEMBER(size_t, count)
+    );
+};
+
+struct file_compare_output {
+    int retval;
+    int errnum;
+
+    REFL_DECL_STRUCT(file_compare_output,
+                     REFL_DECL_MEMBER(int, retval),
+                     REFL_DECL_MEMBER(int, errnum)
+    );
+};
+
+void
+to_json(json& record,
+        const file_compare_output& out) {
+    record = serialize(out);
+}
+
+int open_file(const std::string& path, bool verbose) {
+    auto fd = ::open(path.c_str(), O_RDONLY);
+    if (fd == -1) {
+        if (verbose) {
+            fmt::print("open(pathname=\"{}\") = {}, errno: {} [{}]\n",
+                       path, fd, errno, ::strerror(errno));
+            return -1;
+        }
+        json out = file_compare_output{fd, errno};
+        fmt::print("{}\n", out.dump(2));
+        return -1;
+    }
+    return fd;
+}
+
+size_t read_file(io::buffer& buf, int fd, size_t count) {
+    ssize_t rv{};
+    size_t total{};
+    do {
+        rv = ::read(fd, buf.data(), count - total);
+        total += rv;
+    } while (rv > 0 && total < count);
+
+    if (rv < 0 && total != count) {
+        json out = file_compare_output{(int) rv, errno};
+        fmt::print("{}\n", out.dump(2));
+        return 0;
+    }
+    return total;
+}
+
+void
+file_compare_exec(const file_compare_options& opts) {
+
+    // Open both files
+    auto fd_1 = open_file(opts.path_1, opts.verbose);
+    if (fd_1 == -1) {
+        return;
+    }
+    auto fd_2 = open_file(opts.path_2, opts.verbose);
+    if (fd_2 == -1) {
+        return;
+    }
+
+    // read both files
+    io::buffer buf_1(opts.count);
+    auto rv = read_file(buf_1, fd_1, opts.count);
+    if (rv == 0)
+        return;
+
+    io::buffer buf_2(opts.count);
+    rv = read_file(buf_2, fd_2, opts.count);
+    if (rv == 0)
+        return;
+
+    // memcmp both files to check if they're equal
+    auto comp_rv = memcmp(buf_1.data(), buf_2.data(), opts.count);
+    if (comp_rv != 0) {
+        if (opts.verbose) {
+            fmt::print("memcmp(path_1='{}', path_2='{}', count='{}') = '{}'\n",
+                       opts.path_1, opts.path_2, opts.count, comp_rv);
+            return;
+        }
+    }
+
+    json out = file_compare_output{comp_rv, errno};
+    fmt::print("{}\n", out.dump(2));
+}
+
+void
+file_compare_init(CLI::App& app) {
+
+    // Create the option and subcommand objects
+    auto opts = std::make_shared<file_compare_options>();
+    auto* cmd = app.add_subcommand(
+            "file_compare",
+            "Execute the truncate() system call");
+
+    // Add options to cmd, binding them to opts
+    cmd->add_flag(
+            "-v,--verbose",
+            opts->verbose,
+            "Produce human writeable output"
+    );
+
+    cmd->add_option(
+                    "path_1",
+                    opts->path_1,
+                    "Path to first file"
+            )
+            ->required()
+            ->type_name("");
+
+    cmd->add_option(
+                    "path_2",
+                    opts->path_2,
+                    "Path to second file"
+            )
+            ->required()
+            ->type_name("");
+
+    cmd->add_option(
+                    "count",
+                    opts->count,
+                    "How many bytes to compare of each file"
+            )
+            ->required()
+            ->type_name("");
+
+    cmd->callback([opts]() {
+        file_compare_exec(*opts);
+    });
+}
\ No newline at end of file

tests/integration/harness/gkfs.io/write.cpp

@@ -66,7 +66,7 @@ write_exec(const write_options& opts) {
 
     if(fd == -1) {
         if(opts.verbose) {
-            fmt::print("write(pathname=\"{}\", buf=\"{}\" count={}) = {}, errno: {} [{}]\n", 
+            fmt::print("open(pathname=\"{}\", buf=\"{}\" count={}) = {}, errno: {} [{}]\n",
                     opts.pathname, opts.data, opts.count, fd, errno, ::strerror(errno));
             return;
         }

tests/integration/harness/gkfs.io/write_random.cpp

+/*
+  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain
+  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany
+
+  This software was partially supported by the
+  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+  This software was partially supported by the
+  ADA-FS project under the SPPEXA project funded by the DFG.
+
+  SPDX-License-Identifier: MIT
+*/
+
+/* C++ includes */
+#include <CLI/CLI.hpp>
+#include <nlohmann/json.hpp>
+#include <memory>
+#include <fmt/format.h>
+#include <commands.hpp>
+#include <reflection.hpp>
+#include <serialize.hpp>
+#include <binary_buffer.hpp>
+#include <random>
+#include <climits>
+
+/* C includes */
+#include <sys/types.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#ifndef CHAR_BIT
+#define CHAR_BIT 8
+#endif
+constexpr int seed = 42;
+
+using json = nlohmann::json;
+
+struct write_random_options {
+    bool verbose{};
+    std::string pathname{};
+    ::size_t count{};
+
+    REFL_DECL_STRUCT(write_random_options,
+                     REFL_DECL_MEMBER(bool, verbose),
+                     REFL_DECL_MEMBER(std::string, pathname),
+                     REFL_DECL_MEMBER(::size_t, count)
+    );
+};
+
+struct write_random_output {
+    ::ssize_t retval;
+    int errnum;
+
+    REFL_DECL_STRUCT(write_random_output,
+                     REFL_DECL_MEMBER(::size_t, retval),
+                     REFL_DECL_MEMBER(int, errnum)
+    );
+};
+
+void to_json(json& record,
+             const write_random_output& out) {
+    record = serialize(out);
+}
+
+/**
+ * Writes `count` random bytes to file
+ * @param opts
+ */
+void write_random_exec(const write_random_options& opts) {
+
+    int fd = ::open(opts.pathname.c_str(), O_WRONLY);
+
+    if (fd == -1) {
+        if (opts.verbose) {
+            fmt::print("open(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n",
+                       opts.pathname, opts.count, fd, errno, ::strerror(errno));
+            return;
+        }
+
+        json out = write_random_output{fd, errno};
+        fmt::print("{}\n", out.dump(2));
+        return;
+    }
+    // random number generator with seed
+    std::independent_bits_engine<std::default_random_engine, CHAR_BIT, unsigned char> engine{seed};
+    // create buffer for opts.count
+    std::vector<uint8_t> data(opts.count);
+    std::generate(begin(data), end(data), std::ref(engine));
+    // pass data to buffer
+    io::buffer buf(data);
+
+    int rv = ::write(fd, buf.data(), opts.count);
+
+    if (opts.verbose) {
+        fmt::print("write(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n",
+                   opts.pathname, opts.count, rv, errno, ::strerror(errno));
+        return;
+    }
+
+    json out = write_random_output{rv, errno};
+    fmt::print("{}\n", out.dump(2));
+}
+
+void write_random_init(CLI::App& app) {
+
+    // Create the option and subcommand objects
+    auto opts = std::make_shared<write_random_options>();
+    auto* cmd = app.add_subcommand(
+            "write_random",
+            "Execute the write() system call ");
+
+    // Add options to cmd, binding them to opts
+    cmd->add_flag(
+            "-v,--verbose",
+            opts->verbose,
+            "Produce human writeable output"
+    );
+
+    cmd->add_option(
+                    "pathname",
+                    opts->pathname,
+                    "File name"
+            )
+            ->required()
+            ->type_name("");
+
+    cmd->add_option(
+                    "count",
+                    opts->count,
+                    "Number of random bytes to write"
+            )
+            ->required()
+            ->type_name("");
+
+    cmd->callback([opts]() {
+        write_random_exec(*opts);
+    });
+}
\ No newline at end of file

tests/integration/harness/io.py

@@ -273,7 +273,7 @@ class StatOutputSchema(Schema):
 
 
 class StatxOutputSchema(Schema):
-    """Schema to deserialize the results of a stat() execution"""
+    """Schema to deserialize the results of a statx() execution"""
 
     retval = fields.Integer(required=True)
     statbuf = fields.Nested(StructStatxSchema, required=True)
@@ -285,7 +285,7 @@ class StatxOutputSchema(Schema):
 
 
 class LseekOutputSchema(Schema):
-    """Schema to deserialize the results of an open() execution"""
+    """Schema to deserialize the results of an lseek() execution"""
     retval = fields.Integer(required=True)
     errno = Errno(data_key='errnum', required=True)
 
@@ -304,6 +304,38 @@ class WriteValidateOutputSchema(Schema):
     def make_object(self, data, **kwargs):
         return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data)
 
+
+class WriteRandomOutputSchema(Schema):
+    """Schema to deserialize the results of a write() execution"""
+
+    retval = fields.Integer(required=True)
+    errno = Errno(data_key='errnum', required=True)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return namedtuple('WriteRandomReturn', ['retval', 'errno'])(**data)
+
+
+class TruncateOutputSchema(Schema):
+    """Schema to deserialize the results of an truncate() execution"""
+    retval = fields.Integer(required=True)
+    errno = Errno(data_key='errnum', required=True)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return namedtuple('TruncateReturn', ['retval', 'errno'])(**data)
+
+
+# UTIL
+class FileCompareOutputSchema(Schema):
+    """Schema to deserialize the results of comparing two files execution"""
+    retval = fields.Integer(required=True)
+    errno = Errno(data_key='errnum', required=True)
+
+    @post_load
+    def make_object(self, data, **kwargs):
+        return namedtuple('FileCompareReturn', ['retval', 'errno'])(**data)
+
 class IOParser:
 
     OutputSchemas = {
@@ -323,11 +355,15 @@ class IOParser:
         'stat'    : StatOutputSchema(),
         'statx'   : StatxOutputSchema(),
         'lseek'   : LseekOutputSchema(),
+        'write_random': WriteRandomOutputSchema(),
         'write_validate' : WriteValidateOutputSchema(),
+        'truncate': TruncateOutputSchema(),
+        # UTIL
+        'file_compare': FileCompareOutputSchema(),
     }
 
     def parse(self, command, output):
         if command in self.OutputSchemas:
             return self.OutputSchemas[command].loads(output)
         else:
-            raise ValueError(f"Unknown I/O command {cmd}")
+            raise ValueError(f"Unknown I/O command {command}")