diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 0b781976ef3a2174b4a3defa76f8bf3f35bd779b..bda2fc57a146d18b7a49df609df9d57f06a40175 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -84,6 +84,9 @@ ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); ssize_t gkfs_read(int fd, void* buf, size_t count); +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt); + +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset); int gkfs_opendir(const std::string& path); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e3897c1f414ce0230afe19a6cea9e607c1df83db..2f2a65463c0eaa95fe1128a95cfc3c91a0e8178e 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -42,6 +42,11 @@ int hook_read(unsigned int fd, void* buf, size_t count); int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos); +int hook_readv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt); + +int hook_preadv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h); + int hook_write(unsigned int fd, const char* buf, size_t count); int hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos); diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 396c5df5150985dad0f7490e4e00152e5c5b9af8..003cf2c4c2c5cd575ca87dd46b009be792b01cf1 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -521,6 +521,49 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { return ret; } +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { + + auto file = CTX->file_map()->get(fd); + auto pos = offset; // keep truck of current position + ssize_t read = 0; + ssize_t ret; + for (int i = 0; i < iovcnt; ++i) { + auto count = (iov + i)->iov_len; + if (count == 0) { + continue; + } + auto buf = (iov + i)->iov_base; + ret = gkfs_pread(file, reinterpret_cast(buf), count, pos); + if (ret == -1) { + break; + } + read += ret; + pos += ret; + + if (static_cast(ret) < count) { + break; + } + } + + if (read == 0) { + return -1; + } + return read; +} + +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { + + auto gkfs_fd = CTX->file_map()->get(fd); + auto pos = gkfs_fd->pos(); // retrieve the current offset + auto ret = gkfs_preadv(fd, iov, iovcnt, pos); + assert(ret != 0); + if (ret < 0) { + return -1; + } + gkfs_fd->pos(pos + ret); + return ret; +} + ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { auto gkfs_fd = CTX->file_map()->get(fd); return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset); diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index bb00e99ea4f45d68696b9686d7b8ad366bb816e5..b84f6f1d7fb04517815fa8cf3573a47e44afb4cb 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -209,6 +209,30 @@ int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) { return syscall_no_intercept(SYS_pread64, fd, buf, count, pos); } +int hook_readv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}", + __func__, fd, fmt::ptr(iov), iovcnt); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_readv(fd, iov, iovcnt)); + } + return syscall_no_intercept(SYS_readv, fd, iov, iovcnt); +} + +int hook_preadv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}, " + "pos_l: {}," "pos_h: {}", + __func__, fd, fmt::ptr(iov), iovcnt, pos_l, pos_h); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_preadv(fd, iov, iovcnt, pos_l)); + } + return syscall_no_intercept(SYS_preadv, fd, iov, iovcnt, pos_l); +} + int hook_write(unsigned int fd, const char* buf, size_t count) { LOG(DEBUG, "{}() called with fd: {}, buf: {}, count {}", @@ -253,7 +277,7 @@ int hook_pwritev(unsigned long fd, const struct iovec* iov, unsigned long iovcnt if (CTX->file_map()->exist(fd)) { return with_errno(gkfs::syscall::gkfs_pwritev(fd, iov, iovcnt, pos_l)); } - return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt); + return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt, pos_l); } int hook_unlinkat(int dirfd, const char* cpath, int flags) { @@ -793,4 +817,4 @@ int hook_fstatfs(unsigned int fd, struct statfs* buf) { } } // namespace hook -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 0cb43755d0f3dcc463760ca8cbbe8d3b6a149702..fe1a766e0588bc3e034ef846f338cbf9e9c1fe19 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -506,6 +506,20 @@ int hook(long syscall_number, static_cast(arg3)); break; + case SYS_readv: + *result = gkfs::hook::hook_readv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2)); + break; + + case SYS_preadv: + *result = gkfs::hook::hook_preadv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2), + static_cast(arg3), + static_cast(arg4)); + break; + case SYS_pwrite64: *result = gkfs::hook::hook_pwrite(static_cast(arg0), reinterpret_cast(arg1), diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 74645eef17458fb26b924e98aa344d4b0777695d..f8de8add70b23fc28cea7a2ace099b14e7d35839 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -23,6 +23,12 @@ gkfs_add_python_test( SOURCE status/test_status.py ) +gkfs_add_python_test( + NAME test_operations + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE operations/ +) gkfs_add_python_test( NAME test_shell @@ -57,6 +63,14 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) + install(DIRECTORY operations + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 89d33dde1076f3563e0bd9f6fe8cdb0008057c3b..2967cc3590617de3fe1c8f75cdc36daccb6ea044 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -12,12 +12,18 @@ add_executable(gkfs.io gkfs.io/open.cpp gkfs.io/opendir.cpp gkfs.io/read.cpp + gkfs.io/readv.cpp + gkfs.io/pread.cpp + gkfs.io/preadv.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp gkfs.io/rmdir.cpp gkfs.io/serialize.hpp gkfs.io/stat.cpp gkfs.io/write.cpp + gkfs.io/pwrite.cpp + gkfs.io/writev.cpp + gkfs.io/pwritev.cpp gkfs.io/statx.cpp ) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 97aa48564998f5891da27326c9f63e58e1abb0cb..9e0e75efa0abe7f49ebbe953cfeb9bcbb710e3e4 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -29,6 +29,15 @@ opendir_init(CLI::App& app); void read_init(CLI::App& app); +void +pread_init(CLI::App& app); + +void +readv_init(CLI::App& app); + +void +preadv_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -41,6 +50,15 @@ stat_init(CLI::App& app); void write_init(CLI::App& app); +void +pwrite_init(CLI::App& app); + +void +writev_init(CLI::App& app); + +void +pwritev_init(CLI::App& app); + void statx_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 5b5ad9544d0575a92c0b4ba68782baa7d236ed3b..4b8080af8e04f92cb5ef789bc1dce14b21ca7a9f 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -24,10 +24,16 @@ init_commands(CLI::App& app) { opendir_init(app); mkdir_init(app); read_init(app); + pread_init(app); + readv_init(app); + preadv_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); + pwrite_init(app); + writev_init(app); + pwritev_init(app); statx_init(app); } diff --git a/tests/integration/harness/gkfs.io/pread.cpp b/tests/integration/harness/gkfs.io/pread.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0fac6c6380d37797cbca2d513369b0c5fb0785df --- /dev/null +++ b/tests/integration/harness/gkfs.io/pread.cpp @@ -0,0 +1,140 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pread_options { + bool verbose; + std::string pathname; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pread_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pread_output { + ::ssize_t retval; + io::buffer buf; + int errnum; + + REFL_DECL_STRUCT(pread_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pread_output& out) { + record = serialize(out); +} + +void +pread_exec(const pread_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pread_output{fd, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.count); + + int rv = ::pread(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pread_output{rv, (rv != -1 ? buf : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pread_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pread", + "Execute the pread() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to read" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pread_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/preadv.cpp b/tests/integration/harness/gkfs.io/preadv.cpp new file mode 100644 index 0000000000000000000000000000000000000000..480e7bf3cbc1dac8a24040bcd51ae2e507b2e9d1 --- /dev/null +++ b/tests/integration/harness/gkfs.io/preadv.cpp @@ -0,0 +1,162 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct preadv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + ::size_t offset; + + REFL_DECL_STRUCT(preadv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct preadv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(preadv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const preadv_output& out) { + record = serialize(out); +} + +void +preadv_exec(const preadv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::preadv(fd, iov, 2, opts.offset); + + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +preadv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "preadv", + "Execute the preadv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + preadv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/pwrite.cpp b/tests/integration/harness/gkfs.io/pwrite.cpp new file mode 100644 index 0000000000000000000000000000000000000000..e098624b00c8ec69123e6b0ea8b514115adfa53d --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwrite.cpp @@ -0,0 +1,148 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwrite_options { + bool verbose; + std::string pathname; + std::string data; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwrite_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwrite_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwrite_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwrite_output& out) { + record = serialize(out); +} + +void +pwrite_exec(const pwrite_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", buf=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.data); + int rv = ::pwrite(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwrite_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwrite", + "Execute the pwrite() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data", + opts->data, + "Data to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwrite_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/gkfs.io/pwritev.cpp b/tests/integration/harness/gkfs.io/pwritev.cpp new file mode 100644 index 0000000000000000000000000000000000000000..76e2ec1900ec771cdf86698177fc534f0b774165 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwritev.cpp @@ -0,0 +1,168 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwritev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwritev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwritev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwritev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwritev_output& out) { + record = serialize(out); +} + +void +pwritev_exec(const pwritev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::pwritev(fd, iov, opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwritev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwritev", + "Execute the pwritev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwritev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/gkfs.io/readv.cpp b/tests/integration/harness/gkfs.io/readv.cpp new file mode 100644 index 0000000000000000000000000000000000000000..00ea9926f3dd953ca1e07a5e3a429bf162d8a918 --- /dev/null +++ b/tests/integration/harness/gkfs.io/readv.cpp @@ -0,0 +1,152 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct readv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + + REFL_DECL_STRUCT(readv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1) + ); +}; + +struct readv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(readv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const readv_output& out) { + record = serialize(out); +} + +void +readv_exec(const readv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, fd, errno, ::strerror(errno)); + return; + } + + json out = readv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::readv(fd, iov, 2); + + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, rv, errno, ::strerror(errno)); + return; + } + + json out = readv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +readv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "readv", + "Execute the readv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + readv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/writev.cpp b/tests/integration/harness/gkfs.io/writev.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d57507f305077d716e88bf705d6a94f8dc4be7b7 --- /dev/null +++ b/tests/integration/harness/gkfs.io/writev.cpp @@ -0,0 +1,158 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct writev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + + REFL_DECL_STRUCT(writev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count) + ); +}; + +struct writev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(writev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const writev_output& out) { + record = serialize(out); +} + +void +writev_exec(const writev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, fd, errno, ::strerror(errno)); + return; + } + + json out = writev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::writev(fd, iov, opts.count); + + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, rv, errno, ::strerror(errno)); + return; + } + + json out = writev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +writev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "writev", + "Execute the writev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + writev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index cf786150a92d5e8d6995672218b59312f77eb467..e1cb4415adac95b56933aae64703d631ce62de56 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -165,6 +165,41 @@ class ReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadReturn', ['buf', 'retval', 'errno'])(**data) +class PreadOutputSchema(Schema): + """Schema to deserialize the results of a pread() execution""" + + buf = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadReturn', ['buf', 'retval', 'errno'])(**data) + +class ReadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('ReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + +class PreadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -195,6 +230,36 @@ class WriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteReturn', ['retval', 'errno'])(**data) +class PwriteOutputSchema(Schema): + """Schema to deserialize the results of a pwrite() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWriteReturn', ['retval', 'errno'])(**data) + +class WritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('WritevReturn', ['retval', 'errno'])(**data) + +class PwritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWritevReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -225,9 +290,15 @@ class IOParser: 'open' : OpenOutputSchema(), 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), + 'pread' : PreadOutputSchema(), + 'readv' : ReadvOutputSchema(), + 'preadv' : PreadvOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), + 'pwrite' : PwriteOutputSchema(), + 'writev' : WritevOutputSchema(), + 'pwritev' : PwritevOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), } diff --git a/tests/integration/operations/README.md b/tests/integration/operations/README.md new file mode 100644 index 0000000000000000000000000000000000000000..45e29b5860bef86f7076a12a65e79442ae60c7e2 --- /dev/null +++ b/tests/integration/operations/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any operation-related +functionalities in GekkoFS. diff --git a/tests/integration/operations/test_read_operations.py b/tests/integration/operations/test_read_operations.py new file mode 100644 index 0000000000000000000000000000000000000000..22e65ff95f52105991d138974234c8095ec3c067 --- /dev/null +++ b/tests/integration/operations/test_read_operations.py @@ -0,0 +1,166 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_read(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pread(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file at offset 1024 + ret = gkfs_client.pread(file, len(buf), 1024) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_readv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.readv(file, len(buf_0), len(buf_1)) + + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_preadv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.preadv(file, len(buf_0), len(buf_1), 1024) + + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! \ No newline at end of file diff --git a/tests/integration/operations/test_write_operations.py b/tests/integration/operations/test_write_operations.py new file mode 100644 index 0000000000000000000000000000000000000000..c18987592ec5f3f80aefa9d767328b9c7765f91d --- /dev/null +++ b/tests/integration/operations/test_write_operations.py @@ -0,0 +1,97 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_write(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwrite(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + # write at the offset 1024 + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_writev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwritev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0!