From 1933a035bacf85c6d0554d21d830e3bc6cfd5161 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 13 May 2020 16:28:17 +0200 Subject: [PATCH 1/8] Adding lseek test --- src/client/gkfs_functions.cpp | 6 + tests/integration/CMakeLists.txt | 6 + tests/integration/harness/CMakeLists.txt | 1 + .../integration/harness/gkfs.io/commands.hpp | 3 + tests/integration/harness/gkfs.io/lseek.cpp | 148 ++++++++++++++++++ tests/integration/harness/gkfs.io/main.cpp | 1 + tests/integration/harness/io.py | 11 ++ tests/integration/position/README.md | 4 + tests/integration/position/test_lseek.py | 124 +++++++++++++++ 9 files changed, 304 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/lseek.cpp create mode 100644 tests/integration/position/README.md create mode 100644 tests/integration/position/test_lseek.py diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 396c5df51..e6f2578ad 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -319,10 +319,16 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi case SEEK_END: { off64_t file_size; auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } + + if (offset < 0 and file_size < -offset) { + errno = EINVAL; + return -1; + } gkfs_fd->pos(file_size + offset); break; } diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index d289ccfe4..83fc7b6a2 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -23,6 +23,12 @@ gkfs_add_python_test( SOURCE status/test_status.py ) +gkfs_add_python_test( + NAME test_lseek + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE position/test_lseek.py +) gkfs_add_python_test( NAME test_shell diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 89d33dde1..cd48604fe 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(gkfs.io gkfs.io/stat.cpp gkfs.io/write.cpp gkfs.io/statx.cpp + gkfs.io/lseek.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 97aa48564..0f68a39b1 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -44,4 +44,7 @@ write_init(CLI::App& app); void statx_init(CLI::App& app); +void +lseek_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/lseek.cpp b/tests/integration/harness/gkfs.io/lseek.cpp new file mode 100644 index 000000000..f779c44c9 --- /dev/null +++ b/tests/integration/harness/gkfs.io/lseek.cpp @@ -0,0 +1,148 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct lseek_options { + bool verbose; + std::string pathname; + ::off_t offset; + int whence; + + REFL_DECL_STRUCT(lseek_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::off_t, offset), + REFL_DECL_MEMBER(int, whence) + ); +}; + +struct lseek_output { + ::off_t retval; + int errnum; + + REFL_DECL_STRUCT(lseek_output, + REFL_DECL_MEMBER(::off_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const lseek_output& out) { + record = serialize(out); +} + +std::string +whence2str(int whence) { + switch (whence) { + case SEEK_SET : return "SEEK_SET"; + case SEEK_CUR : return "SEEK_CUR"; + case SEEK_END : return "SEEK_END"; + default : return "UNKNOWN"; + } + return "UNKNOWN"; +} + +void +lseek_exec(const lseek_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("open(pathname=\"{}\") = {}, errno: {} [{}]\n", + opts.pathname, fd, errno, ::strerror(errno)); + return; + } + + json out = lseek_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + int rv = ::lseek(fd, opts.offset, opts.whence); + + if(opts.verbose) { + fmt::print("lseek(pathname=\"{}\", offset='{}', whence='{}') = {}, errno: {} [{}]\n", + opts.pathname, opts.offset, whence2str(opts.whence), rv, errno, ::strerror(errno)); + return; + } + + json out = lseek_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +lseek_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "lseek", + "Execute the lseek() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "offset used" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "whence", + opts->whence, + "Whence the action is done" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + lseek_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 5b5ad9544..704f78dd1 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -29,6 +29,7 @@ init_commands(CLI::App& app) { stat_init(app); write_init(app); statx_init(app); + lseek_init(app); } diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index cf786150a..f7f6b2367 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -218,6 +218,16 @@ class StatxOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('StatxReturn', ['retval', 'statbuf', 'errno'])(**data) + +class LseekOutputSchema(Schema): + """Schema to deserialize the results of an open() execution""" + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('LseekReturn', ['retval', 'errno'])(**data) + class IOParser: OutputSchemas = { @@ -230,6 +240,7 @@ class IOParser: 'write' : WriteOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), + 'lseek' : LseekOutputSchema(), } def parse(self, command, output): diff --git a/tests/integration/position/README.md b/tests/integration/position/README.md new file mode 100644 index 000000000..ad502acfe --- /dev/null +++ b/tests/integration/position/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any position-related +functionalities in GekkoFS. diff --git a/tests/integration/position/test_lseek.py b/tests/integration/position/test_lseek.py new file mode 100644 index 000000000..30f8c9afc --- /dev/null +++ b/tests/integration/position/test_lseek.py @@ -0,0 +1,124 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + + +#TESTING LSEEK + + +#SEEK_SET: int +#SEEK_CUR: int +#SEEK_END: int + +# 1. LSeek on empty file +# 2. LSeek on non-empty file + + + +#@pytest.mark.xfail(reason="invalid errno returned on success") +def test_lseek(gkfs_daemon, gkfs_client): + """Test several statx commands""" + topdir = gkfs_daemon.mountdir / "top" + longer = Path(topdir.parent, topdir.name + "_plus") + file_a = topdir / "file_a" + + # create topdir + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + + # test statx on existing dir + ret = gkfs_client.statx(0, topdir, 0, 0) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + assert stat.S_ISDIR(ret.statbuf.stx_mode) + + ret = gkfs_client.open(file_a, + os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval != -1 + + # LSeek on empty file + + ret = gkfs_client.lseek(file_a, 0, os.SEEK_SET) + + assert ret.retval == 0 + + ret = gkfs_client.lseek(file_a, -1, os.SEEK_SET) + assert ret.retval == -1 + assert ret.errno == 115 #FIXME: Should be 22 + + # We can go further an empty file + + ret = gkfs_client.lseek(file_a, 5, os.SEEK_SET) + assert ret.retval == 5 + assert ret.errno == 115 #FIXME: Should be 0 + + # Size needs to be 0 + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0) + assert (ret.statbuf.stx_size == 0) + + + # next commands write at the end of the file (pos0), as the filedescriptor is not shared + buf = b'42' + ret = gkfs_client.write(file_a, buf, 2) + + assert ret.retval == 2 + + # Size should be 2 + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (ret.statbuf.stx_size == 2) + + ret = gkfs_client.lseek(file_a, 0, os.SEEK_END) + assert ret.retval == 2 #FAILS + assert ret.errno == 115 #FIXME: Should be 0 + + ret = gkfs_client.lseek(file_a, -2, os.SEEK_END) + assert ret.retval == 0 #FAILS + assert ret.errno == 115 #FIXME: Should be 0 + + ret = gkfs_client.lseek(file_a, -3, os.SEEK_END) + assert ret.retval == -1 #FAILS + assert ret.errno == 115 #FIXME: Should be 22 + + + + + + + return + + -- GitLab From c00dfe90a17d8af80c959246034c2fb91ee97359 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 14 May 2020 12:09:21 +0200 Subject: [PATCH 2/8] Enable position tests at install tests --- tests/integration/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b6276da4c..01b5f6f0f 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -63,6 +63,15 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) + install(DIRECTORY position + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING -- GitLab From 076d8e7c8cf57ac12cd6a178fd74324ac53e51bb Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 14 May 2020 16:07:05 +0200 Subject: [PATCH 3/8] Solved lseek bug, still investigating --- src/client/gkfs_functions.cpp | 15 +++++++++++---- tests/integration/position/test_lseek.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index e6f2578ad..0c2bf663e 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -318,13 +318,20 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi break; case SEEK_END: { off64_t file_size; - auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); - - if (err < 0) { + auto md = gkfs::util::get_metadata(gkfs_fd->path(), true); + if (!md) { + errno = EINVAL; + return -1; + } + + //auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + + /* if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } - + */ + file_size = md.get()->size(); if (offset < 0 and file_size < -offset) { errno = EINVAL; return -1; diff --git a/tests/integration/position/test_lseek.py b/tests/integration/position/test_lseek.py index 30f8c9afc..51b9b05c6 100644 --- a/tests/integration/position/test_lseek.py +++ b/tests/integration/position/test_lseek.py @@ -112,7 +112,7 @@ def test_lseek(gkfs_daemon, gkfs_client): ret = gkfs_client.lseek(file_a, -3, os.SEEK_END) assert ret.retval == -1 #FAILS - assert ret.errno == 115 #FIXME: Should be 22 + assert ret.errno == 22 -- GitLab From 6337fcd36615118dcd1bf3fce76b58a4bc6659b2 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 15 May 2020 08:38:39 +0200 Subject: [PATCH 4/8] [LSeek] Solved get_metadata_size bug --- include/client/rpc/rpc_types.hpp | 2 +- src/client/gkfs_functions.cpp | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index 95babcb0c..85aec4586 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -1012,7 +1012,7 @@ struct get_metadentry_size { // Mercury callback to serialize output arguments constexpr static const auto mercury_out_proc_cb = - HG_GEN_PROC_NAME(rpc_err_out_t); + HG_GEN_PROC_NAME(rpc_get_metadentry_size_out_t); class input { diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 0c2bf663e..feff529bc 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -318,20 +318,13 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi break; case SEEK_END: { off64_t file_size; - auto md = gkfs::util::get_metadata(gkfs_fd->path(), true); - if (!md) { - errno = EINVAL; - return -1; - } - - //auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); - /* if (err < 0) { + if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } - */ - file_size = md.get()->size(); + if (offset < 0 and file_size < -offset) { errno = EINVAL; return -1; -- GitLab From d6be612d72db9304aae81662d42cd8f17dc2a3bf Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 13 May 2020 16:28:17 +0200 Subject: [PATCH 5/8] Adding lseek test --- src/client/gkfs_functions.cpp | 6 + tests/integration/CMakeLists.txt | 7 + tests/integration/harness/CMakeLists.txt | 1 + .../integration/harness/gkfs.io/commands.hpp | 3 + tests/integration/harness/gkfs.io/lseek.cpp | 148 ++++++++++++++++++ tests/integration/harness/gkfs.io/main.cpp | 1 + tests/integration/harness/io.py | 11 ++ tests/integration/position/README.md | 4 + tests/integration/position/test_lseek.py | 124 +++++++++++++++ 9 files changed, 305 insertions(+) create mode 100644 tests/integration/harness/gkfs.io/lseek.cpp create mode 100644 tests/integration/position/README.md create mode 100644 tests/integration/position/test_lseek.py diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 003cf2c4c..43dc46ae0 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -319,10 +319,16 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi case SEEK_END: { off64_t file_size; auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } + + if (offset < 0 and file_size < -offset) { + errno = EINVAL; + return -1; + } gkfs_fd->pos(file_size + offset); break; } diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index f8de8add7..d83653755 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -30,6 +30,13 @@ gkfs_add_python_test( SOURCE operations/ ) +gkfs_add_python_test( + NAME test_lseek + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE position/test_lseek.py +) + gkfs_add_python_test( NAME test_shell PYTHON_VERSION 3.6 diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 2967cc359..7a8090e79 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -25,6 +25,7 @@ add_executable(gkfs.io gkfs.io/writev.cpp gkfs.io/pwritev.cpp gkfs.io/statx.cpp + gkfs.io/lseek.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 9e0e75efa..edda1c289 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -62,4 +62,7 @@ pwritev_init(CLI::App& app); void statx_init(CLI::App& app); +void +lseek_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/lseek.cpp b/tests/integration/harness/gkfs.io/lseek.cpp new file mode 100644 index 000000000..f779c44c9 --- /dev/null +++ b/tests/integration/harness/gkfs.io/lseek.cpp @@ -0,0 +1,148 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct lseek_options { + bool verbose; + std::string pathname; + ::off_t offset; + int whence; + + REFL_DECL_STRUCT(lseek_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::off_t, offset), + REFL_DECL_MEMBER(int, whence) + ); +}; + +struct lseek_output { + ::off_t retval; + int errnum; + + REFL_DECL_STRUCT(lseek_output, + REFL_DECL_MEMBER(::off_t, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const lseek_output& out) { + record = serialize(out); +} + +std::string +whence2str(int whence) { + switch (whence) { + case SEEK_SET : return "SEEK_SET"; + case SEEK_CUR : return "SEEK_CUR"; + case SEEK_END : return "SEEK_END"; + default : return "UNKNOWN"; + } + return "UNKNOWN"; +} + +void +lseek_exec(const lseek_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_RDONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("open(pathname=\"{}\") = {}, errno: {} [{}]\n", + opts.pathname, fd, errno, ::strerror(errno)); + return; + } + + json out = lseek_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + int rv = ::lseek(fd, opts.offset, opts.whence); + + if(opts.verbose) { + fmt::print("lseek(pathname=\"{}\", offset='{}', whence='{}') = {}, errno: {} [{}]\n", + opts.pathname, opts.offset, whence2str(opts.whence), rv, errno, ::strerror(errno)); + return; + } + + json out = lseek_output{rv, errno}; + fmt::print("{}\n", out.dump(2)); +} + +void +lseek_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "lseek", + "Execute the lseek() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "offset", + opts->offset, + "offset used" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "whence", + opts->whence, + "Whence the action is done" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + lseek_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 4b8080af8..0cde949db 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -35,6 +35,7 @@ init_commands(CLI::App& app) { writev_init(app); pwritev_init(app); statx_init(app); + lseek_init(app); } diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index e1cb4415a..f50d3d9f4 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -283,6 +283,16 @@ class StatxOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('StatxReturn', ['retval', 'statbuf', 'errno'])(**data) + +class LseekOutputSchema(Schema): + """Schema to deserialize the results of an open() execution""" + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('LseekReturn', ['retval', 'errno'])(**data) + class IOParser: OutputSchemas = { @@ -301,6 +311,7 @@ class IOParser: 'pwritev' : PwritevOutputSchema(), 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), + 'lseek' : LseekOutputSchema(), } def parse(self, command, output): diff --git a/tests/integration/position/README.md b/tests/integration/position/README.md new file mode 100644 index 000000000..ad502acfe --- /dev/null +++ b/tests/integration/position/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any position-related +functionalities in GekkoFS. diff --git a/tests/integration/position/test_lseek.py b/tests/integration/position/test_lseek.py new file mode 100644 index 000000000..30f8c9afc --- /dev/null +++ b/tests/integration/position/test_lseek.py @@ -0,0 +1,124 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + + +#TESTING LSEEK + + +#SEEK_SET: int +#SEEK_CUR: int +#SEEK_END: int + +# 1. LSeek on empty file +# 2. LSeek on non-empty file + + + +#@pytest.mark.xfail(reason="invalid errno returned on success") +def test_lseek(gkfs_daemon, gkfs_client): + """Test several statx commands""" + topdir = gkfs_daemon.mountdir / "top" + longer = Path(topdir.parent, topdir.name + "_plus") + file_a = topdir / "file_a" + + # create topdir + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + + # test statx on existing dir + ret = gkfs_client.statx(0, topdir, 0, 0) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + assert stat.S_ISDIR(ret.statbuf.stx_mode) + + ret = gkfs_client.open(file_a, + os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval != -1 + + # LSeek on empty file + + ret = gkfs_client.lseek(file_a, 0, os.SEEK_SET) + + assert ret.retval == 0 + + ret = gkfs_client.lseek(file_a, -1, os.SEEK_SET) + assert ret.retval == -1 + assert ret.errno == 115 #FIXME: Should be 22 + + # We can go further an empty file + + ret = gkfs_client.lseek(file_a, 5, os.SEEK_SET) + assert ret.retval == 5 + assert ret.errno == 115 #FIXME: Should be 0 + + # Size needs to be 0 + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0) + assert (ret.statbuf.stx_size == 0) + + + # next commands write at the end of the file (pos0), as the filedescriptor is not shared + buf = b'42' + ret = gkfs_client.write(file_a, buf, 2) + + assert ret.retval == 2 + + # Size should be 2 + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (ret.statbuf.stx_size == 2) + + ret = gkfs_client.lseek(file_a, 0, os.SEEK_END) + assert ret.retval == 2 #FAILS + assert ret.errno == 115 #FIXME: Should be 0 + + ret = gkfs_client.lseek(file_a, -2, os.SEEK_END) + assert ret.retval == 0 #FAILS + assert ret.errno == 115 #FIXME: Should be 0 + + ret = gkfs_client.lseek(file_a, -3, os.SEEK_END) + assert ret.retval == -1 #FAILS + assert ret.errno == 115 #FIXME: Should be 22 + + + + + + + return + + -- GitLab From 6e9dffbcb0cd4276a1342717e7dbe8d200f2f245 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 14 May 2020 12:09:21 +0200 Subject: [PATCH 6/8] Enable position tests at install tests --- tests/integration/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index d83653755..3ba045901 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -78,6 +78,15 @@ if(GKFS_INSTALL_TESTS) PATTERN ".pytest_cache" EXCLUDE ) + install(DIRECTORY position + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING -- GitLab From b8a775c161e573bc0a0122b25663d2bb6aafa8c2 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Thu, 14 May 2020 16:07:05 +0200 Subject: [PATCH 7/8] Solved lseek bug, still investigating --- src/client/gkfs_functions.cpp | 15 +++++++++++---- tests/integration/position/test_lseek.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 43dc46ae0..49f86719b 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -318,13 +318,20 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi break; case SEEK_END: { off64_t file_size; - auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); - - if (err < 0) { + auto md = gkfs::util::get_metadata(gkfs_fd->path(), true); + if (!md) { + errno = EINVAL; + return -1; + } + + //auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + + /* if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } - + */ + file_size = md.get()->size(); if (offset < 0 and file_size < -offset) { errno = EINVAL; return -1; diff --git a/tests/integration/position/test_lseek.py b/tests/integration/position/test_lseek.py index 30f8c9afc..51b9b05c6 100644 --- a/tests/integration/position/test_lseek.py +++ b/tests/integration/position/test_lseek.py @@ -112,7 +112,7 @@ def test_lseek(gkfs_daemon, gkfs_client): ret = gkfs_client.lseek(file_a, -3, os.SEEK_END) assert ret.retval == -1 #FAILS - assert ret.errno == 115 #FIXME: Should be 22 + assert ret.errno == 22 -- GitLab From 655cb66c9d559b5d1d93156700b88ce7394ae6c8 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 15 May 2020 08:38:39 +0200 Subject: [PATCH 8/8] [LSeek] Solved get_metadata_size bug --- include/client/rpc/rpc_types.hpp | 2 +- src/client/gkfs_functions.cpp | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/include/client/rpc/rpc_types.hpp b/include/client/rpc/rpc_types.hpp index 95babcb0c..85aec4586 100644 --- a/include/client/rpc/rpc_types.hpp +++ b/include/client/rpc/rpc_types.hpp @@ -1012,7 +1012,7 @@ struct get_metadentry_size { // Mercury callback to serialize output arguments constexpr static const auto mercury_out_proc_cb = - HG_GEN_PROC_NAME(rpc_err_out_t); + HG_GEN_PROC_NAME(rpc_get_metadentry_size_out_t); class input { diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 49f86719b..29854a518 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -318,20 +318,13 @@ off_t gkfs_lseek(shared_ptr gkfs_fd, off_t offset, unsi break; case SEEK_END: { off64_t file_size; - auto md = gkfs::util::get_metadata(gkfs_fd->path(), true); - if (!md) { - errno = EINVAL; - return -1; - } - - //auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); + auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size); - /* if (err < 0) { + if (err < 0) { errno = err; // Negative numbers are explicitly for error codes return -1; } - */ - file_size = md.get()->size(); + if (offset < 0 and file_size < -offset) { errno = EINVAL; return -1; -- GitLab