From 3c32a79f336da500e61674886928984616879598 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Thu, 7 May 2020 10:35:55 -0300 Subject: [PATCH 01/15] This fixes #84 by implementing readv and preadv --- include/client/gkfs_functions.hpp | 3 +++ include/client/hooks.hpp | 5 ++++ src/client/gkfs_functions.cpp | 43 +++++++++++++++++++++++++++++++ src/client/hooks.cpp | 28 ++++++++++++++++++-- src/client/intercept.cpp | 16 +++++++++++- 5 files changed, 92 insertions(+), 3 deletions(-) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 7ae71ce99..674174c1c 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -76,6 +76,9 @@ ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); ssize_t gkfs_read(int fd, void* buf, size_t count); +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt); + +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset); int gkfs_opendir(const std::string& path); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index a0aacee42..d23127bc4 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -40,6 +40,11 @@ int hook_read(unsigned int fd, void* buf, size_t count); int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos); +int hook_readv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt); + +int hook_preadv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h); + int hook_write(unsigned int fd, const char* buf, size_t count); int hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos); diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 029304a77..0e1037a2b 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -485,6 +485,49 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { return ret; } +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { + + auto file = CTX->file_map()->get(fd); + auto pos = offset; // keep truck of current position + ssize_t read = 0; + ssize_t ret; + for (int i = 0; i < iovcnt; ++i) { + auto count = (iov + i)->iov_len; + if (count == 0) { + continue; + } + auto buf = (iov + i)->iov_base; + ret = gkfs_pread(file, reinterpret_cast(buf), count, pos); + if (ret == -1) { + break; + } + read += ret; + pos += ret; + + if (static_cast(ret) < count) { + break; + } + } + + if (read == 0) { + return -1; + } + return read; +} + +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { + + auto gkfs_fd = CTX->file_map()->get(fd); + auto pos = gkfs_fd->pos(); // retrieve the current offset + auto ret = gkfs_preadv(fd, iov, iovcnt, pos); + assert(ret != 0); + if (ret < 0) { + return -1; + } + gkfs_fd->pos(pos + ret); + return ret; +} + ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { auto gkfs_fd = CTX->file_map()->get(fd); return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset); diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index fa437d1ed..6c2539a76 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -176,6 +176,30 @@ int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) { return syscall_no_intercept(SYS_pread64, fd, buf, count, pos); } +int hook_readv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}", + __func__, fd, fmt::ptr(iov), iovcnt); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_readv(fd, iov, iovcnt)); + } + return syscall_no_intercept(SYS_readv, fd, iov, iovcnt); +} + +int hook_preadv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}, " + "pos_l: {}," "pos_h: {}", + __func__, fd, fmt::ptr(iov), iovcnt, pos_l, pos_h); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_preadv(fd, iov, iovcnt, pos_l)); + } + return syscall_no_intercept(SYS_preadv, fd, iov, iovcnt, pos_l); +} + int hook_write(unsigned int fd, const char* buf, size_t count) { LOG(DEBUG, "{}() called with fd: {}, buf: {}, count {}", @@ -220,7 +244,7 @@ int hook_pwritev(unsigned long fd, const struct iovec* iov, unsigned long iovcnt if (CTX->file_map()->exist(fd)) { return with_errno(gkfs::syscall::gkfs_pwritev(fd, iov, iovcnt, pos_l)); } - return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt); + return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt, pos_l); } int hook_unlinkat(int dirfd, const char* cpath, int flags) { @@ -760,4 +784,4 @@ int hook_fstatfs(unsigned int fd, struct statfs* buf) { } } // namespace hook -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 9905a0b5d..70434b4c1 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -496,6 +496,20 @@ int hook(long syscall_number, static_cast(arg3)); break; + case SYS_readv: + *result = gkfs::hook::hook_readv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2)); + break; + + case SYS_preadv: + *result = gkfs::hook::hook_preadv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2), + static_cast(arg3), + static_cast(arg4)); + break; + case SYS_pwrite64: *result = gkfs::hook::hook_pwrite(static_cast(arg0), reinterpret_cast(arg1), @@ -914,4 +928,4 @@ void stop_interception() { } } // namespace preload -} // namespace gkfs \ No newline at end of file +} // namespace gkfs -- GitLab From 1f313bd13da3decb29add45f7ca199208bc2f008 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 14:04:48 -0300 Subject: [PATCH 02/15] Implement tests for write, pwrite, read, and pread --- tests/integration/CMakeLists.txt | 7 + tests/integration/harness/CMakeLists.txt | 2 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + tests/integration/harness/gkfs.io/pread.cpp | 140 +++++++++++++++++ tests/integration/harness/gkfs.io/pwrite.cpp | 148 ++++++++++++++++++ tests/integration/harness/io.py | 23 +++ tests/integration/operations/README.md | 4 + .../integration/operations/test_operations.py | 129 +++++++++++++++ 9 files changed, 461 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/pread.cpp create mode 100644 tests/integration/harness/gkfs.io/pwrite.cpp create mode 100644 tests/integration/operations/README.md create mode 100644 tests/integration/operations/test_operations.py diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b5a50f5d6..ff707b649 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -16,6 +16,13 @@ gkfs_add_python_test( SOURCE directories/test_directories.py ) +gkfs_add_python_test( + NAME test_operations + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE operations/test_operations.py +) + gkfs_add_python_test( NAME test_shell PYTHON_VERSION 3.6 diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 282d7bb75..1b4c48a7a 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -12,12 +12,14 @@ add_executable(gkfs.io gkfs.io/open.cpp gkfs.io/opendir.cpp gkfs.io/read.cpp + gkfs.io/pread.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp gkfs.io/rmdir.cpp gkfs.io/serialize.hpp gkfs.io/stat.cpp gkfs.io/write.cpp + gkfs.io/pwrite.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index f91c696f5..358985092 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -29,6 +29,9 @@ opendir_init(CLI::App& app); void read_init(CLI::App& app); +void +pread_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -41,4 +44,7 @@ stat_init(CLI::App& app); void write_init(CLI::App& app); +void +pwrite_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 50f5b3210..c3b083d97 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -24,10 +24,12 @@ init_commands(CLI::App& app) { opendir_init(app); mkdir_init(app); read_init(app); + pread_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); + pwrite_init(app); } diff --git a/tests/integration/harness/gkfs.io/pread.cpp b/tests/integration/harness/gkfs.io/pread.cpp new file mode 100644 index 000000000..0fac6c638 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pread.cpp @@ -0,0 +1,140 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pread_options { + bool verbose; + std::string pathname; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pread_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pread_output { + ::ssize_t retval; + io::buffer buf; + int errnum; + + REFL_DECL_STRUCT(pread_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pread_output& out) { + record = serialize(out); +} + +void +pread_exec(const pread_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pread_output{fd, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.count); + + int rv = ::pread(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pread_output{rv, (rv != -1 ? buf : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pread_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pread", + "Execute the pread() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to read" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pread_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/pwrite.cpp b/tests/integration/harness/gkfs.io/pwrite.cpp new file mode 100644 index 000000000..e098624b0 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwrite.cpp @@ -0,0 +1,148 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwrite_options { + bool verbose; + std::string pathname; + std::string data; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwrite_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwrite_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwrite_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwrite_output& out) { + record = serialize(out); +} + +void +pwrite_exec(const pwrite_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", buf=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.data); + int rv = ::pwrite(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwrite_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwrite", + "Execute the pwrite() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data", + opts->data, + "Data to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwrite_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 798227c9c..bc24cbfad 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -120,6 +120,17 @@ class ReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadReturn', ['buf', 'retval', 'errno'])(**data) +class PReadOutputSchema(Schema): + """Schema to deserialize the results of a pread() execution""" + + buf = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadReturn', ['buf', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -150,6 +161,16 @@ class WriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteReturn', ['retval', 'errno'])(**data) +class PWriteOutputSchema(Schema): + """Schema to deserialize the results of a pwrite() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWriteReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -169,9 +190,11 @@ class IOParser: 'open' : OpenOutputSchema(), 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), + 'pread' : PReadOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), + 'pwrite' : PWriteOutputSchema(), 'stat' : StatOutputSchema(), } diff --git a/tests/integration/operations/README.md b/tests/integration/operations/README.md new file mode 100644 index 000000000..45e29b586 --- /dev/null +++ b/tests/integration/operations/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any operation-related +functionalities in GekkoFS. diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py new file mode 100644 index 000000000..4d0cfe181 --- /dev/null +++ b/tests/integration/operations/test_operations.py @@ -0,0 +1,129 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_write(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_read(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwrite(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + # write at the offset 1024 + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pread(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file at offset 1024 + ret = gkfs_client.pread(file, len(buf), 1024) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! -- GitLab From 344ad51dc3a0ba9312732bccc8ee010e1a24e1ee Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 14:24:22 -0300 Subject: [PATCH 03/15] Implement tests for write, pwrite, read, and pread --- tests/integration/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index ff707b649..2fe0c8162 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -48,6 +48,14 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) + install(DIRECTORY operations + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING -- GitLab From 09d5b59a11f8f5e285e47927fc2001eea55aacf5 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 21:55:50 -0300 Subject: [PATCH 04/15] Implement tests for writev and readv --- tests/integration/harness/CMakeLists.txt | 2 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + tests/integration/harness/gkfs.io/readv.cpp | 152 +++++++++++++++++ tests/integration/harness/gkfs.io/writev.cpp | 158 ++++++++++++++++++ tests/integration/harness/io.py | 24 +++ .../integration/operations/test_operations.py | 54 ++++++ 7 files changed, 398 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/readv.cpp create mode 100644 tests/integration/harness/gkfs.io/writev.cpp diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 1b4c48a7a..5ce643e10 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(gkfs.io gkfs.io/open.cpp gkfs.io/opendir.cpp gkfs.io/read.cpp + gkfs.io/readv.cpp gkfs.io/pread.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp @@ -20,6 +21,7 @@ add_executable(gkfs.io gkfs.io/stat.cpp gkfs.io/write.cpp gkfs.io/pwrite.cpp + gkfs.io/writev.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 358985092..9da6b0135 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -32,6 +32,9 @@ read_init(CLI::App& app); void pread_init(CLI::App& app); +void +readv_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -47,4 +50,7 @@ write_init(CLI::App& app); void pwrite_init(CLI::App& app); +void +writev_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index c3b083d97..5d9922f51 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -25,11 +25,13 @@ init_commands(CLI::App& app) { mkdir_init(app); read_init(app); pread_init(app); + readv_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); pwrite_init(app); + writev_init(app); } diff --git a/tests/integration/harness/gkfs.io/readv.cpp b/tests/integration/harness/gkfs.io/readv.cpp new file mode 100644 index 000000000..00ea9926f --- /dev/null +++ b/tests/integration/harness/gkfs.io/readv.cpp @@ -0,0 +1,152 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct readv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + + REFL_DECL_STRUCT(readv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1) + ); +}; + +struct readv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(readv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const readv_output& out) { + record = serialize(out); +} + +void +readv_exec(const readv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, fd, errno, ::strerror(errno)); + return; + } + + json out = readv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::readv(fd, iov, 2); + + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, rv, errno, ::strerror(errno)); + return; + } + + json out = readv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +readv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "readv", + "Execute the readv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + readv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/writev.cpp b/tests/integration/harness/gkfs.io/writev.cpp new file mode 100644 index 000000000..d57507f30 --- /dev/null +++ b/tests/integration/harness/gkfs.io/writev.cpp @@ -0,0 +1,158 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct writev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + + REFL_DECL_STRUCT(writev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count) + ); +}; + +struct writev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(writev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const writev_output& out) { + record = serialize(out); +} + +void +writev_exec(const writev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, fd, errno, ::strerror(errno)); + return; + } + + json out = writev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::writev(fd, iov, opts.count); + + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, rv, errno, ::strerror(errno)); + return; + } + + json out = writev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +writev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "writev", + "Execute the writev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + writev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index bc24cbfad..512079e35 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -131,6 +131,18 @@ class PReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('PReadReturn', ['buf', 'retval', 'errno'])(**data) +class ReadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('ReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -171,6 +183,16 @@ class PWriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('PWriteReturn', ['retval', 'errno'])(**data) +class WritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('WritevReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -191,10 +213,12 @@ class IOParser: 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), 'pread' : PReadOutputSchema(), + 'readv' : ReadvOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), 'pwrite' : PWriteOutputSchema(), + 'writev' : WritevOutputSchema(), 'stat' : StatOutputSchema(), } diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py index 4d0cfe181..c3a776937 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_operations.py @@ -127,3 +127,57 @@ def test_pread(gkfs_daemon, gkfs_client): assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! + +def test_writev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_readv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.readv(file, len(buf_0), len(buf_1)) + + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! \ No newline at end of file -- GitLab From dfb56c7e335972f4245a167e0d8748c2055e3067 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 23:36:38 -0300 Subject: [PATCH 05/15] Implement tests for pwritev and preadv --- tests/integration/harness/CMakeLists.txt | 2 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + tests/integration/harness/gkfs.io/preadv.cpp | 162 +++++++++++++++++ tests/integration/harness/gkfs.io/pwritev.cpp | 168 ++++++++++++++++++ tests/integration/harness/io.py | 36 +++- .../integration/operations/test_operations.py | 54 ++++++ 7 files changed, 424 insertions(+), 6 deletions(-) create mode 100644 tests/integration/harness/gkfs.io/preadv.cpp create mode 100644 tests/integration/harness/gkfs.io/pwritev.cpp diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 5ce643e10..83d5cb03f 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -14,6 +14,7 @@ add_executable(gkfs.io gkfs.io/read.cpp gkfs.io/readv.cpp gkfs.io/pread.cpp + gkfs.io/preadv.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp gkfs.io/rmdir.cpp @@ -22,6 +23,7 @@ add_executable(gkfs.io gkfs.io/write.cpp gkfs.io/pwrite.cpp gkfs.io/writev.cpp + gkfs.io/pwritev.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 9da6b0135..5695ec852 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -35,6 +35,9 @@ pread_init(CLI::App& app); void readv_init(CLI::App& app); +void +preadv_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -53,4 +56,7 @@ pwrite_init(CLI::App& app); void writev_init(CLI::App& app); +void +pwritev_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 5d9922f51..c1357f7c3 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -26,12 +26,14 @@ init_commands(CLI::App& app) { read_init(app); pread_init(app); readv_init(app); + preadv_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); pwrite_init(app); writev_init(app); + pwritev_init(app); } diff --git a/tests/integration/harness/gkfs.io/preadv.cpp b/tests/integration/harness/gkfs.io/preadv.cpp new file mode 100644 index 000000000..480e7bf3c --- /dev/null +++ b/tests/integration/harness/gkfs.io/preadv.cpp @@ -0,0 +1,162 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct preadv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + ::size_t offset; + + REFL_DECL_STRUCT(preadv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct preadv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(preadv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const preadv_output& out) { + record = serialize(out); +} + +void +preadv_exec(const preadv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::preadv(fd, iov, 2, opts.offset); + + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +preadv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "preadv", + "Execute the preadv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + preadv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/pwritev.cpp b/tests/integration/harness/gkfs.io/pwritev.cpp new file mode 100644 index 000000000..76e2ec190 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwritev.cpp @@ -0,0 +1,168 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwritev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwritev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwritev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwritev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwritev_output& out) { + record = serialize(out); +} + +void +pwritev_exec(const pwritev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::pwritev(fd, iov, opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwritev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwritev", + "Execute the pwritev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwritev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 512079e35..123117d79 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -120,7 +120,7 @@ class ReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadReturn', ['buf', 'retval', 'errno'])(**data) -class PReadOutputSchema(Schema): +class PreadOutputSchema(Schema): """Schema to deserialize the results of a pread() execution""" buf = ByteList(allow_none=True) @@ -143,6 +143,18 @@ class ReadvOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) +class PreadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -173,7 +185,7 @@ class WriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteReturn', ['retval', 'errno'])(**data) -class PWriteOutputSchema(Schema): +class PwriteOutputSchema(Schema): """Schema to deserialize the results of a pwrite() execution""" retval = fields.Integer(required=True) @@ -193,6 +205,16 @@ class WritevOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WritevReturn', ['retval', 'errno'])(**data) +class PwritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWritevReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -212,13 +234,15 @@ class IOParser: 'open' : OpenOutputSchema(), 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), - 'pread' : PReadOutputSchema(), - 'readv' : ReadvOutputSchema(), + 'pread' : PreadOutputSchema(), + 'readv' : ReadvOutputSchema(), + 'preadv' : PreadvOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), - 'pwrite' : PWriteOutputSchema(), - 'writev' : WritevOutputSchema(), + 'pwrite' : PwriteOutputSchema(), + 'writev' : WritevOutputSchema(), + 'pwritev' : PwritevOutputSchema(), 'stat' : StatOutputSchema(), } diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py index c3a776937..826b62c89 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_operations.py @@ -177,6 +177,60 @@ def test_readv(gkfs_daemon, gkfs_client): # read the file ret = gkfs_client.readv(file, len(buf_0), len(buf_1)) + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwritev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_preadv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.preadv(file, len(buf_0), len(buf_1), 1024) + assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes -- GitLab From 12dec59861b61eb6b1def00a1554f686e158c507 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 23:41:29 -0300 Subject: [PATCH 06/15] Refactor write and read tests --- tests/integration/CMakeLists.txt | 21 +++- ..._operations.py => test_read_operations.py} | 71 -------------- .../operations/test_write_operations.py | 97 +++++++++++++++++++ 3 files changed, 115 insertions(+), 74 deletions(-) rename tests/integration/operations/{test_operations.py => test_read_operations.py} (71%) create mode 100644 tests/integration/operations/test_write_operations.py diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 2fe0c8162..9932c235f 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -17,10 +17,17 @@ gkfs_add_python_test( ) gkfs_add_python_test( - NAME test_operations + NAME test_write_operations PYTHON_VERSION 3.6 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration - SOURCE operations/test_operations.py + SOURCE operations/test_write_operations.py +) + +gkfs_add_python_test( + NAME test_read_operations + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE operations/test_read_operations.py ) gkfs_add_python_test( @@ -48,7 +55,15 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) - install(DIRECTORY operations + install(DIRECTORY write_operations + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + + install(DIRECTORY read_operations DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING REGEX ".*\\.py" diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_read_operations.py similarity index 71% rename from tests/integration/operations/test_operations.py rename to tests/integration/operations/test_read_operations.py index 826b62c89..22e65ff95 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_read_operations.py @@ -25,23 +25,6 @@ from harness.logger import logger nonexisting = "nonexisting" -def test_write(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf = b'42' - ret = gkfs_client.write(file, buf, len(buf)) - - assert ret.retval == len(buf) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_read(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -76,24 +59,6 @@ def test_read(gkfs_daemon, gkfs_client): assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_pwrite(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf = b'42' - # write at the offset 1024 - ret = gkfs_client.pwrite(file, buf, len(buf), 1024) - - assert ret.retval == len(buf) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_pread(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -128,24 +93,6 @@ def test_pread(gkfs_daemon, gkfs_client): assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_writev(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf_0 = b'42' - buf_1 = b'24' - ret = gkfs_client.writev(file, buf_0, buf_1, 2) - - assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_readv(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -182,24 +129,6 @@ def test_readv(gkfs_daemon, gkfs_client): assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_pwritev(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf_0 = b'42' - buf_1 = b'24' - ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) - - assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_preadv(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" diff --git a/tests/integration/operations/test_write_operations.py b/tests/integration/operations/test_write_operations.py new file mode 100644 index 000000000..c18987592 --- /dev/null +++ b/tests/integration/operations/test_write_operations.py @@ -0,0 +1,97 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_write(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwrite(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + # write at the offset 1024 + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_writev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwritev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! -- GitLab From 52eb2bcc752435a503860a5c0449bcb1961c2dcd Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Tue, 12 May 2020 00:01:51 -0300 Subject: [PATCH 07/15] Refactor write and read tests --- tests/integration/CMakeLists.txt | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 9932c235f..870a3e026 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -17,17 +17,10 @@ gkfs_add_python_test( ) gkfs_add_python_test( - NAME test_write_operations + NAME test_operations PYTHON_VERSION 3.6 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration - SOURCE operations/test_write_operations.py -) - -gkfs_add_python_test( - NAME test_read_operations - PYTHON_VERSION 3.6 - WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration - SOURCE operations/test_read_operations.py + SOURCE operations/ ) gkfs_add_python_test( @@ -55,15 +48,7 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) - install(DIRECTORY write_operations - DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration - FILES_MATCHING - REGEX ".*\\.py" - PATTERN "__pycache__" EXCLUDE - PATTERN ".pytest_cache" EXCLUDE - ) - - install(DIRECTORY read_operations + install(DIRECTORY operations DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING REGEX ".*\\.py" -- GitLab From 38a8110cf8cecf423e863e900dbb4fe3314b0f2b Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Thu, 7 May 2020 10:35:55 -0300 Subject: [PATCH 08/15] This fixes #84 by implementing readv and preadv --- include/client/gkfs_functions.hpp | 3 +++ include/client/hooks.hpp | 5 ++++ src/client/gkfs_functions.cpp | 43 +++++++++++++++++++++++++++++++ src/client/hooks.cpp | 28 ++++++++++++++++++-- src/client/intercept.cpp | 14 ++++++++++ 5 files changed, 91 insertions(+), 2 deletions(-) diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 0b781976e..bda2fc57a 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -84,6 +84,9 @@ ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset); ssize_t gkfs_read(int fd, void* buf, size_t count); +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt); + +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset); int gkfs_opendir(const std::string& path); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index e3897c1f4..2f2a65463 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -42,6 +42,11 @@ int hook_read(unsigned int fd, void* buf, size_t count); int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos); +int hook_readv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt); + +int hook_preadv(unsigned long fd, const struct iovec * iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h); + int hook_write(unsigned int fd, const char* buf, size_t count); int hook_pwrite(unsigned int fd, const char* buf, size_t count, loff_t pos); diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 396c5df51..003cf2c4c 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -521,6 +521,49 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) { return ret; } +ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) { + + auto file = CTX->file_map()->get(fd); + auto pos = offset; // keep truck of current position + ssize_t read = 0; + ssize_t ret; + for (int i = 0; i < iovcnt; ++i) { + auto count = (iov + i)->iov_len; + if (count == 0) { + continue; + } + auto buf = (iov + i)->iov_base; + ret = gkfs_pread(file, reinterpret_cast(buf), count, pos); + if (ret == -1) { + break; + } + read += ret; + pos += ret; + + if (static_cast(ret) < count) { + break; + } + } + + if (read == 0) { + return -1; + } + return read; +} + +ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) { + + auto gkfs_fd = CTX->file_map()->get(fd); + auto pos = gkfs_fd->pos(); // retrieve the current offset + auto ret = gkfs_preadv(fd, iov, iovcnt, pos); + assert(ret != 0); + if (ret < 0) { + return -1; + } + gkfs_fd->pos(pos + ret); + return ret; +} + ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) { auto gkfs_fd = CTX->file_map()->get(fd); return gkfs_pread(gkfs_fd, reinterpret_cast(buf), count, offset); diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index bb00e99ea..b84f6f1d7 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -209,6 +209,30 @@ int hook_pread(unsigned int fd, char* buf, size_t count, loff_t pos) { return syscall_no_intercept(SYS_pread64, fd, buf, count, pos); } +int hook_readv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}", + __func__, fd, fmt::ptr(iov), iovcnt); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_readv(fd, iov, iovcnt)); + } + return syscall_no_intercept(SYS_readv, fd, iov, iovcnt); +} + +int hook_preadv(unsigned long fd, const struct iovec* iov, unsigned long iovcnt, + unsigned long pos_l, unsigned long pos_h) { + + LOG(DEBUG, "{}() called with fd: {}, iov: {}, iovcnt: {}, " + "pos_l: {}," "pos_h: {}", + __func__, fd, fmt::ptr(iov), iovcnt, pos_l, pos_h); + + if (CTX->file_map()->exist(fd)) { + return with_errno(gkfs::syscall::gkfs_preadv(fd, iov, iovcnt, pos_l)); + } + return syscall_no_intercept(SYS_preadv, fd, iov, iovcnt, pos_l); +} + int hook_write(unsigned int fd, const char* buf, size_t count) { LOG(DEBUG, "{}() called with fd: {}, buf: {}, count {}", @@ -253,7 +277,7 @@ int hook_pwritev(unsigned long fd, const struct iovec* iov, unsigned long iovcnt if (CTX->file_map()->exist(fd)) { return with_errno(gkfs::syscall::gkfs_pwritev(fd, iov, iovcnt, pos_l)); } - return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt); + return syscall_no_intercept(SYS_pwritev, fd, iov, iovcnt, pos_l); } int hook_unlinkat(int dirfd, const char* cpath, int flags) { @@ -793,4 +817,4 @@ int hook_fstatfs(unsigned int fd, struct statfs* buf) { } } // namespace hook -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 0cb43755d..fe1a766e0 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -506,6 +506,20 @@ int hook(long syscall_number, static_cast(arg3)); break; + case SYS_readv: + *result = gkfs::hook::hook_readv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2)); + break; + + case SYS_preadv: + *result = gkfs::hook::hook_preadv(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2), + static_cast(arg3), + static_cast(arg4)); + break; + case SYS_pwrite64: *result = gkfs::hook::hook_pwrite(static_cast(arg0), reinterpret_cast(arg1), -- GitLab From eb935ed649ac9aa69f999c31c59f512183175bd4 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 14:04:48 -0300 Subject: [PATCH 09/15] Implement tests for write, pwrite, read, and pread --- tests/integration/CMakeLists.txt | 6 + tests/integration/harness/CMakeLists.txt | 2 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + tests/integration/harness/gkfs.io/pread.cpp | 140 +++++++++++++++++ tests/integration/harness/gkfs.io/pwrite.cpp | 148 ++++++++++++++++++ tests/integration/harness/io.py | 23 +++ tests/integration/operations/README.md | 4 + .../integration/operations/test_operations.py | 129 +++++++++++++++ 9 files changed, 460 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/pread.cpp create mode 100644 tests/integration/harness/gkfs.io/pwrite.cpp create mode 100644 tests/integration/operations/README.md create mode 100644 tests/integration/operations/test_operations.py diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 74645eef1..e4d8071dd 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -23,6 +23,12 @@ gkfs_add_python_test( SOURCE status/test_status.py ) +gkfs_add_python_test( + NAME test_operations + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE operations/test_operations.py +) gkfs_add_python_test( NAME test_shell diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 89d33dde1..d3b1c5b05 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -12,12 +12,14 @@ add_executable(gkfs.io gkfs.io/open.cpp gkfs.io/opendir.cpp gkfs.io/read.cpp + gkfs.io/pread.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp gkfs.io/rmdir.cpp gkfs.io/serialize.hpp gkfs.io/stat.cpp gkfs.io/write.cpp + gkfs.io/pwrite.cpp gkfs.io/statx.cpp ) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 97aa48564..e096836d4 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -29,6 +29,9 @@ opendir_init(CLI::App& app); void read_init(CLI::App& app); +void +pread_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -41,6 +44,9 @@ stat_init(CLI::App& app); void write_init(CLI::App& app); +void +pwrite_init(CLI::App& app); + void statx_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 5b5ad9544..4279a85f3 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -24,10 +24,12 @@ init_commands(CLI::App& app) { opendir_init(app); mkdir_init(app); read_init(app); + pread_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); + pwrite_init(app); statx_init(app); } diff --git a/tests/integration/harness/gkfs.io/pread.cpp b/tests/integration/harness/gkfs.io/pread.cpp new file mode 100644 index 000000000..0fac6c638 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pread.cpp @@ -0,0 +1,140 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pread_options { + bool verbose; + std::string pathname; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pread_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pread_output { + ::ssize_t retval; + io::buffer buf; + int errnum; + + REFL_DECL_STRUCT(pread_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pread_output& out) { + record = serialize(out); +} + +void +pread_exec(const pread_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pread_output{fd, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.count); + + int rv = ::pread(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pread(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pread_output{rv, (rv != -1 ? buf : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pread_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pread", + "Execute the pread() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to read" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pread_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/pwrite.cpp b/tests/integration/harness/gkfs.io/pwrite.cpp new file mode 100644 index 000000000..e098624b0 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwrite.cpp @@ -0,0 +1,148 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwrite_options { + bool verbose; + std::string pathname; + std::string data; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwrite_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwrite_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwrite_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwrite_output& out) { + record = serialize(out); +} + +void +pwrite_exec(const pwrite_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", buf=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf(opts.data); + int rv = ::pwrite(fd, buf.data(), opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwrite(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwrite_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwrite_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwrite", + "Execute the pwrite() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data", + opts->data, + "Data to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwrite_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index cf786150a..7e558cd57 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -165,6 +165,17 @@ class ReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadReturn', ['buf', 'retval', 'errno'])(**data) +class PReadOutputSchema(Schema): + """Schema to deserialize the results of a pread() execution""" + + buf = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadReturn', ['buf', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -195,6 +206,16 @@ class WriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteReturn', ['retval', 'errno'])(**data) +class PWriteOutputSchema(Schema): + """Schema to deserialize the results of a pwrite() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWriteReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -225,9 +246,11 @@ class IOParser: 'open' : OpenOutputSchema(), 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), + 'pread' : PReadOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), + 'pwrite' : PWriteOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), } diff --git a/tests/integration/operations/README.md b/tests/integration/operations/README.md new file mode 100644 index 000000000..45e29b586 --- /dev/null +++ b/tests/integration/operations/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any operation-related +functionalities in GekkoFS. diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py new file mode 100644 index 000000000..4d0cfe181 --- /dev/null +++ b/tests/integration/operations/test_operations.py @@ -0,0 +1,129 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_write(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_read(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.read(file, len(buf)) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwrite(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + # write at the offset 1024 + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pread(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf = b'42' + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file at offset 1024 + ret = gkfs_client.pread(file, len(buf), 1024) + + assert ret.buf == buf + assert ret.retval == len(buf) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! -- GitLab From 73679cca12b5df93a43edcf851e8d53f5cf9313c Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 14:24:22 -0300 Subject: [PATCH 10/15] Implement tests for write, pwrite, read, and pread --- tests/integration/CMakeLists.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index e4d8071dd..ffef5064c 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -63,6 +63,14 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) + install(DIRECTORY operations + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING -- GitLab From 1975846288d1a1c9bb365725e60156a510b2955a Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 21:55:50 -0300 Subject: [PATCH 11/15] Implement tests for writev and readv --- tests/integration/harness/CMakeLists.txt | 2 + .../integration/harness/gkfs.io/commands.hpp | 6 + tests/integration/harness/gkfs.io/main.cpp | 2 + tests/integration/harness/gkfs.io/readv.cpp | 152 +++++++++++++++++ tests/integration/harness/gkfs.io/writev.cpp | 158 ++++++++++++++++++ tests/integration/harness/io.py | 24 +++ .../integration/operations/test_operations.py | 54 ++++++ 7 files changed, 398 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/readv.cpp create mode 100644 tests/integration/harness/gkfs.io/writev.cpp diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index d3b1c5b05..b8d03d668 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -12,6 +12,7 @@ add_executable(gkfs.io gkfs.io/open.cpp gkfs.io/opendir.cpp gkfs.io/read.cpp + gkfs.io/readv.cpp gkfs.io/pread.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp @@ -20,6 +21,7 @@ add_executable(gkfs.io gkfs.io/stat.cpp gkfs.io/write.cpp gkfs.io/pwrite.cpp + gkfs.io/writev.cpp gkfs.io/statx.cpp ) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index e096836d4..7ee7fcb33 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -32,6 +32,9 @@ read_init(CLI::App& app); void pread_init(CLI::App& app); +void +readv_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -47,6 +50,9 @@ write_init(CLI::App& app); void pwrite_init(CLI::App& app); +void +writev_init(CLI::App& app); + void statx_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 4279a85f3..32307488e 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -25,11 +25,13 @@ init_commands(CLI::App& app) { mkdir_init(app); read_init(app); pread_init(app); + readv_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); pwrite_init(app); + writev_init(app); statx_init(app); } diff --git a/tests/integration/harness/gkfs.io/readv.cpp b/tests/integration/harness/gkfs.io/readv.cpp new file mode 100644 index 000000000..00ea9926f --- /dev/null +++ b/tests/integration/harness/gkfs.io/readv.cpp @@ -0,0 +1,152 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct readv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + + REFL_DECL_STRUCT(readv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1) + ); +}; + +struct readv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(readv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const readv_output& out) { + record = serialize(out); +} + +void +readv_exec(const readv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, fd, errno, ::strerror(errno)); + return; + } + + json out = readv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::readv(fd, iov, 2); + + if(opts.verbose) { + fmt::print("readv(pathname=\"{}\", count_0={}, count_1={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, rv, errno, ::strerror(errno)); + return; + } + + json out = readv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +readv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "readv", + "Execute the readv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + readv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/writev.cpp b/tests/integration/harness/gkfs.io/writev.cpp new file mode 100644 index 000000000..d57507f30 --- /dev/null +++ b/tests/integration/harness/gkfs.io/writev.cpp @@ -0,0 +1,158 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct writev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + + REFL_DECL_STRUCT(writev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count) + ); +}; + +struct writev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(writev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const writev_output& out) { + record = serialize(out); +} + +void +writev_exec(const writev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, fd, errno, ::strerror(errno)); + return; + } + + json out = writev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::writev(fd, iov, opts.count); + + if(opts.verbose) { + fmt::print("writev(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, rv, errno, ::strerror(errno)); + return; + } + + json out = writev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +writev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "writev", + "Execute the writev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + writev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 7e558cd57..ce65ed33d 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -176,6 +176,18 @@ class PReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('PReadReturn', ['buf', 'retval', 'errno'])(**data) +class ReadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('ReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -216,6 +228,16 @@ class PWriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('PWriteReturn', ['retval', 'errno'])(**data) +class WritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('WritevReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -247,10 +269,12 @@ class IOParser: 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), 'pread' : PReadOutputSchema(), + 'readv' : ReadvOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), 'pwrite' : PWriteOutputSchema(), + 'writev' : WritevOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), } diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py index 4d0cfe181..c3a776937 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_operations.py @@ -127,3 +127,57 @@ def test_pread(gkfs_daemon, gkfs_client): assert ret.buf == buf assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! + +def test_writev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_readv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.readv(file, len(buf_0), len(buf_1)) + + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! \ No newline at end of file -- GitLab From 4cd68e2e6b7775ed141af6f38a1e52d03741f0f7 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 23:36:38 -0300 Subject: [PATCH 12/15] Implement tests for pwritev and preadv --- tests/integration/harness/gkfs.io/preadv.cpp | 162 +++++++++++++++++ tests/integration/harness/gkfs.io/pwritev.cpp | 168 ++++++++++++++++++ tests/integration/harness/io.py | 36 +++- .../integration/operations/test_operations.py | 54 ++++++ 4 files changed, 414 insertions(+), 6 deletions(-) create mode 100644 tests/integration/harness/gkfs.io/preadv.cpp create mode 100644 tests/integration/harness/gkfs.io/pwritev.cpp diff --git a/tests/integration/harness/gkfs.io/preadv.cpp b/tests/integration/harness/gkfs.io/preadv.cpp new file mode 100644 index 000000000..480e7bf3c --- /dev/null +++ b/tests/integration/harness/gkfs.io/preadv.cpp @@ -0,0 +1,162 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct preadv_options { + bool verbose; + std::string pathname; + ::size_t count_0; + ::size_t count_1; + ::size_t offset; + + REFL_DECL_STRUCT(preadv_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count_0), + REFL_DECL_MEMBER(::size_t, count_1), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct preadv_output { + ::ssize_t retval; + io::buffer buf_0; + io::buffer buf_1; + int errnum; + + REFL_DECL_STRUCT(preadv_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(void*, buf_0), + REFL_DECL_MEMBER(void*, buf_1), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const preadv_output& out) { + record = serialize(out); +} + +void +preadv_exec(const preadv_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{fd, nullptr, nullptr, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.count_0); + io::buffer buf_1(opts.count_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = opts.count_0; + iov[1].iov_len = opts.count_1; + + int rv = ::preadv(fd, iov, 2, opts.offset); + + if(opts.verbose) { + fmt::print("preadv(pathname=\"{}\", count_0={}, count_1={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count_0, opts.count_1, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = preadv_output{rv, (rv != -1 ? buf_0 : nullptr), (rv != -1 ? buf_1 : nullptr), errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +preadv_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "preadv", + "Execute the preadv() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_0", + opts->count_0, + "Number of bytes to read to buffer 0" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count_1", + opts->count_1, + "Number of bytes to read to buffer 1" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + preadv_exec(*opts); + }); +} + diff --git a/tests/integration/harness/gkfs.io/pwritev.cpp b/tests/integration/harness/gkfs.io/pwritev.cpp new file mode 100644 index 000000000..76e2ec190 --- /dev/null +++ b/tests/integration/harness/gkfs.io/pwritev.cpp @@ -0,0 +1,168 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +struct pwritev_options { + bool verbose; + std::string pathname; + std::string data_0, data_1; + ::size_t count; + ::size_t offset; + + REFL_DECL_STRUCT(pwritev_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(std::string, data_0), + REFL_DECL_MEMBER(std::string, data_1), + REFL_DECL_MEMBER(::size_t, count), + REFL_DECL_MEMBER(::size_t, offset) + ); +}; + +struct pwritev_output { + ::ssize_t retval; + int errnum; + + REFL_DECL_STRUCT(pwritev_output, + REFL_DECL_MEMBER(::size_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const pwritev_output& out) { + record = serialize(out); +} + +void +pwritev_exec(const pwritev_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", buf_0=\"{}\" buf_1=\"{}\" count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.data_0, opts.data_1, opts.count, opts.offset, fd, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + io::buffer buf_0(opts.data_0); + io::buffer buf_1(opts.data_1); + + struct iovec iov[2]; + + iov[0].iov_base = buf_0.data(); + iov[1].iov_base = buf_1.data(); + + iov[0].iov_len = buf_0.size(); + iov[1].iov_len = buf_1.size(); + + int rv = ::pwritev(fd, iov, opts.count, opts.offset); + + if(opts.verbose) { + fmt::print("pwritev(pathname=\"{}\", count={}, offset={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, opts.offset, rv, errno, ::strerror(errno)); + return; + } + + json out = pwritev_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +pwritev_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "pwritev", + "Execute the pwritev() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_0", + opts->data_0, + "Data 0 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "data_1", + opts->data_1, + "Data 1 to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to write" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "Offset to read" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + pwritev_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index ce65ed33d..e1cb4415a 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -165,7 +165,7 @@ class ReadOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadReturn', ['buf', 'retval', 'errno'])(**data) -class PReadOutputSchema(Schema): +class PreadOutputSchema(Schema): """Schema to deserialize the results of a pread() execution""" buf = ByteList(allow_none=True) @@ -188,6 +188,18 @@ class ReadvOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('ReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) +class PreadvOutputSchema(Schema): + """Schema to deserialize the results of a read() execution""" + + buf_0 = ByteList(allow_none=True) + buf_1 = ByteList(allow_none=True) + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PReadvReturn', ['buf_0', 'buf_1', 'retval', 'errno'])(**data) + class ReaddirOutputSchema(Schema): """Schema to deserialize the results of a readdir() execution""" @@ -218,7 +230,7 @@ class WriteOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteReturn', ['retval', 'errno'])(**data) -class PWriteOutputSchema(Schema): +class PwriteOutputSchema(Schema): """Schema to deserialize the results of a pwrite() execution""" retval = fields.Integer(required=True) @@ -238,6 +250,16 @@ class WritevOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WritevReturn', ['retval', 'errno'])(**data) +class PwritevOutputSchema(Schema): + """Schema to deserialize the results of a writev() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('PWritevReturn', ['retval', 'errno'])(**data) + class StatOutputSchema(Schema): """Schema to deserialize the results of a stat() execution""" @@ -268,13 +290,15 @@ class IOParser: 'open' : OpenOutputSchema(), 'opendir' : OpendirOutputSchema(), 'read' : ReadOutputSchema(), - 'pread' : PReadOutputSchema(), - 'readv' : ReadvOutputSchema(), + 'pread' : PreadOutputSchema(), + 'readv' : ReadvOutputSchema(), + 'preadv' : PreadvOutputSchema(), 'readdir' : ReaddirOutputSchema(), 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), - 'pwrite' : PWriteOutputSchema(), - 'writev' : WritevOutputSchema(), + 'pwrite' : PwriteOutputSchema(), + 'writev' : WritevOutputSchema(), + 'pwritev' : PwritevOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), } diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_operations.py index c3a776937..826b62c89 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_operations.py @@ -177,6 +177,60 @@ def test_readv(gkfs_daemon, gkfs_client): # read the file ret = gkfs_client.readv(file, len(buf_0), len(buf_1)) + assert ret.buf_0 == buf_0 + assert ret.buf_1 == buf_1 + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwritev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_preadv(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + # create a file in gekkofs + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # write a buffer we know + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + + # open the file to read + ret = gkfs_client.open(file, + os.O_RDONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + # read the file + ret = gkfs_client.preadv(file, len(buf_0), len(buf_1), 1024) + assert ret.buf_0 == buf_0 assert ret.buf_1 == buf_1 assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes -- GitLab From 476ca4961db176588df2b93a3313d6f569ece5ac Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Mon, 11 May 2020 23:41:29 -0300 Subject: [PATCH 13/15] Refactor write and read tests --- ..._operations.py => test_read_operations.py} | 71 -------------- .../operations/test_write_operations.py | 97 +++++++++++++++++++ 2 files changed, 97 insertions(+), 71 deletions(-) rename tests/integration/operations/{test_operations.py => test_read_operations.py} (71%) create mode 100644 tests/integration/operations/test_write_operations.py diff --git a/tests/integration/operations/test_operations.py b/tests/integration/operations/test_read_operations.py similarity index 71% rename from tests/integration/operations/test_operations.py rename to tests/integration/operations/test_read_operations.py index 826b62c89..22e65ff95 100644 --- a/tests/integration/operations/test_operations.py +++ b/tests/integration/operations/test_read_operations.py @@ -25,23 +25,6 @@ from harness.logger import logger nonexisting = "nonexisting" -def test_write(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf = b'42' - ret = gkfs_client.write(file, buf, len(buf)) - - assert ret.retval == len(buf) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_read(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -76,24 +59,6 @@ def test_read(gkfs_daemon, gkfs_client): assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_pwrite(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf = b'42' - # write at the offset 1024 - ret = gkfs_client.pwrite(file, buf, len(buf), 1024) - - assert ret.retval == len(buf) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_pread(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -128,24 +93,6 @@ def test_pread(gkfs_daemon, gkfs_client): assert ret.retval == len(buf) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_writev(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf_0 = b'42' - buf_1 = b'24' - ret = gkfs_client.writev(file, buf_0, buf_1, 2) - - assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_readv(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" @@ -182,24 +129,6 @@ def test_readv(gkfs_daemon, gkfs_client): assert ret.retval == len(buf_0) + len(buf_1) # Return the number of read bytes assert ret.errno == 115 #FIXME: Should be 0! -def test_pwritev(gkfs_daemon, gkfs_client): - - file = gkfs_daemon.mountdir / "file" - - ret = gkfs_client.open(file, - os.O_CREAT | os.O_WRONLY, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval == 10000 - assert ret.errno == 115 #FIXME: Should be 0! - - buf_0 = b'42' - buf_1 = b'24' - ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) - - assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes - assert ret.errno == 115 #FIXME: Should be 0! - def test_preadv(gkfs_daemon, gkfs_client): file = gkfs_daemon.mountdir / "file" diff --git a/tests/integration/operations/test_write_operations.py b/tests/integration/operations/test_write_operations.py new file mode 100644 index 000000000..c18987592 --- /dev/null +++ b/tests/integration/operations/test_write_operations.py @@ -0,0 +1,97 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + +def test_write(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + ret = gkfs_client.write(file, buf, len(buf)) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwrite(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf = b'42' + # write at the offset 1024 + ret = gkfs_client.pwrite(file, buf, len(buf), 1024) + + assert ret.retval == len(buf) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_writev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.writev(file, buf_0, buf_1, 2) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! + +def test_pwritev(gkfs_daemon, gkfs_client): + + file = gkfs_daemon.mountdir / "file" + + ret = gkfs_client.open(file, + os.O_CREAT | os.O_WRONLY, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 10000 + assert ret.errno == 115 #FIXME: Should be 0! + + buf_0 = b'42' + buf_1 = b'24' + ret = gkfs_client.pwritev(file, buf_0, buf_1, 2, 1024) + + assert ret.retval == len(buf_0) + len(buf_1) # Return the number of written bytes + assert ret.errno == 115 #FIXME: Should be 0! -- GitLab From 10fd3038d09f4ea7a90c92e29cf5ad41bbc48471 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Fri, 15 May 2020 23:51:33 -0300 Subject: [PATCH 14/15] Fix missing file in CMakeLists --- tests/integration/CMakeLists.txt | 2 +- tests/integration/harness/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index ffef5064c..f8de8add7 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -27,7 +27,7 @@ gkfs_add_python_test( NAME test_operations PYTHON_VERSION 3.6 WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration - SOURCE operations/test_operations.py + SOURCE operations/ ) gkfs_add_python_test( diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index b8d03d668..2967cc359 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -14,6 +14,7 @@ add_executable(gkfs.io gkfs.io/read.cpp gkfs.io/readv.cpp gkfs.io/pread.cpp + gkfs.io/preadv.cpp gkfs.io/readdir.cpp gkfs.io/reflection.hpp gkfs.io/rmdir.cpp @@ -22,6 +23,7 @@ add_executable(gkfs.io gkfs.io/write.cpp gkfs.io/pwrite.cpp gkfs.io/writev.cpp + gkfs.io/pwritev.cpp gkfs.io/statx.cpp ) -- GitLab From 7740c8fdbcdb60f50aa8319041e45d3e4d4b3659 Mon Sep 17 00:00:00 2001 From: Jean Luca Bez Date: Sat, 16 May 2020 00:06:06 -0300 Subject: [PATCH 15/15] Missing files on merge --- tests/integration/harness/gkfs.io/commands.hpp | 6 ++++++ tests/integration/harness/gkfs.io/main.cpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 7ee7fcb33..9e0e75efa 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -35,6 +35,9 @@ pread_init(CLI::App& app); void readv_init(CLI::App& app); +void +preadv_init(CLI::App& app); + void readdir_init(CLI::App& app); @@ -53,6 +56,9 @@ pwrite_init(CLI::App& app); void writev_init(CLI::App& app); +void +pwritev_init(CLI::App& app); + void statx_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 32307488e..4b8080af8 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -26,12 +26,14 @@ init_commands(CLI::App& app) { read_init(app); pread_init(app); readv_init(app); + preadv_init(app); readdir_init(app); rmdir_init(app); stat_init(app); write_init(app); pwrite_init(app); writev_init(app); + pwritev_init(app); statx_init(app); } -- GitLab