diff --git a/include/client/gkfs_functions.hpp b/include/client/gkfs_functions.hpp index 7ae71ce99300cb5f32188b1b14445a6de368e330..0b781976ef3a2174b4a3defa76f8bf3f35bd779b 100644 --- a/include/client/gkfs_functions.hpp +++ b/include/client/gkfs_functions.hpp @@ -31,10 +31,18 @@ int gkfs_create(const std::string& path, mode_t mode); int gkfs_remove(const std::string& path); +// Implementation of access, +// Follow links is true by default int gkfs_access(const std::string& path, int mask, bool follow_links = true); +// Implementation of stat, +// Follow links is true by default int gkfs_stat(const std::string& path, struct stat* buf, bool follow_links = true); +// Implementation of statx, it uses the normal stat and maps the information to the statx structure +// Follow links is true by default +int gkfs_statx(int dirfd, const std::string& path, int flags, unsigned int mask,struct statx* buf, bool follow_links = true ); + int gkfs_statfs(struct statfs* buf); int gkfs_statvfs(struct statvfs* buf); diff --git a/include/client/hooks.hpp b/include/client/hooks.hpp index a0aacee421d46c30ee1060d40f54dff6c341aabf..e3897c1f414ce0230afe19a6cea9e607c1df83db 100644 --- a/include/client/hooks.hpp +++ b/include/client/hooks.hpp @@ -30,6 +30,8 @@ int hook_close(int fd); int hook_stat(const char* path, struct stat* buf); +int hook_statx(int dirfd, const char* path, int flags, unsigned int mask,struct statx* buf); + int hook_lstat(const char* path, struct stat* buf); int hook_fstat(unsigned int fd, struct stat* buf); diff --git a/src/client/gkfs_functions.cpp b/src/client/gkfs_functions.cpp index 029304a7726f2474a81cd60fe492bdd854703dde..396c5df5150985dad0f7490e4e00152e5c5b9af8 100644 --- a/src/client/gkfs_functions.cpp +++ b/src/client/gkfs_functions.cpp @@ -59,7 +59,6 @@ struct linux_dirent64 { char d_name[1]; // originally `char d_name[0]` in kernel, but ISO C++ forbids zero-size array 'd_name' }; - namespace { int check_parent_dir(const std::string& path) { @@ -233,6 +232,43 @@ int gkfs_stat(const string& path, struct stat* buf, bool follow_links) { return 0; } + +int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, struct statx* buf, bool follow_links) { + auto md = gkfs::util::get_metadata(path, follow_links); + if (!md) { + return -1; + } + + struct stat tmp{}; + + gkfs::util::metadata_to_stat(path, *md, tmp); + + buf->stx_mask = 0; + buf->stx_blksize = tmp.st_blksize; + buf->stx_attributes = 0; + buf->stx_nlink = tmp.st_nlink; + buf->stx_uid = tmp.st_uid; + buf->stx_gid = tmp.st_gid; + buf->stx_mode = tmp.st_mode; + buf->stx_ino = tmp.st_ino; + buf->stx_size = tmp.st_size; + buf->stx_blocks = tmp.st_blocks; + buf->stx_attributes_mask = 0; + + buf->stx_atime.tv_sec = tmp.st_atim.tv_sec; + buf->stx_atime.tv_nsec = tmp.st_atim.tv_nsec; + + buf->stx_mtime.tv_sec = tmp.st_mtim.tv_sec; + buf->stx_mtime.tv_nsec = tmp.st_mtim.tv_nsec; + + buf->stx_ctime.tv_sec = tmp.st_ctim.tv_sec; + buf->stx_ctime.tv_nsec = tmp.st_ctim.tv_nsec; + + buf->stx_btime = buf->stx_atime; + + return 0; +} + int gkfs_statfs(struct statfs* buf) { auto blk_stat = gkfs::rpc::forward_get_chunk_stat(); buf->f_type = 0; diff --git a/src/client/hooks.cpp b/src/client/hooks.cpp index fa437d1ede9fb044896a706305642ab79fc21b48..bb00e99ea4f45d68696b9686d7b8ad366bb816e5 100644 --- a/src/client/hooks.cpp +++ b/src/client/hooks.cpp @@ -98,6 +98,39 @@ int hook_stat(const char* path, struct stat* buf) { return syscall_no_intercept(SYS_stat, rel_path.c_str(), buf); } + +int hook_statx(int dirfd, const char* path, int flags, unsigned int mask, struct ::statx* buf) { + + LOG(DEBUG, "{}() called with dirfd: '{}', path: \"{}\", flags: '{}', mask: '{}', buf: '{}'", + __func__, dirfd, path, flags, mask, fmt::ptr(buf)); + + std::string resolved; + auto rstatus = CTX->relativize_fd_path(dirfd, path, resolved); + switch (rstatus) { + case gkfs::preload::RelativizeStatus::fd_unknown: + return syscall_no_intercept(SYS_statx, dirfd, path, flags, mask, buf); + + case gkfs::preload::RelativizeStatus::external: + return syscall_no_intercept(SYS_statx, dirfd, resolved.c_str(), flags, mask, buf); + + case gkfs::preload::RelativizeStatus::fd_not_a_dir: + return -ENOTDIR; + + case gkfs::preload::RelativizeStatus::internal: + return with_errno(gkfs::syscall::gkfs_statx(dirfd, resolved.c_str() , flags, mask, buf)); + + default: + LOG(ERROR, "{}() relativize status unknown: {}", __func__); + return -EINVAL; + + } + + return syscall_no_intercept(SYS_statx, dirfd, path, flags, mask, buf); +} + + + + int hook_lstat(const char* path, struct stat* buf) { LOG(DEBUG, "{}() called with path: \"{}\", buf: {}", diff --git a/src/client/intercept.cpp b/src/client/intercept.cpp index 9905a0b5db5282a5e61b7e738aa7b50664bd8f3f..0cb43755d0f3dcc463760ca8cbbe8d3b6a149702 100644 --- a/src/client/intercept.cpp +++ b/src/client/intercept.cpp @@ -466,6 +466,16 @@ int hook(long syscall_number, reinterpret_cast(arg1)); break; +#ifdef SYS_statx + case SYS_statx: + *result = gkfs::hook::hook_statx(static_cast(arg0), + reinterpret_cast(arg1), + static_cast(arg2), + static_cast(arg3), + reinterpret_cast(arg4)); + break; +#endif + case SYS_lstat: *result = gkfs::hook::hook_lstat(reinterpret_cast(arg0), reinterpret_cast(arg1)); @@ -914,4 +924,4 @@ void stop_interception() { } } // namespace preload -} // namespace gkfs \ No newline at end of file +} // namespace gkfs diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b5a50f5d60c362864efdb803ba0a48a04f4f7442..d289ccfe44bb7e61023529650f60aa1c4ca7f752 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -16,6 +16,14 @@ gkfs_add_python_test( SOURCE directories/test_directories.py ) +gkfs_add_python_test( + NAME test_status + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE status/test_status.py +) + + gkfs_add_python_test( NAME test_shell PYTHON_VERSION 3.6 diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 282d7bb75878d18d9cd8a3a9907c45d0d0782289..89d33dde1076f3563e0bd9f6fe8cdb0008057c3b 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -18,6 +18,7 @@ add_executable(gkfs.io gkfs.io/serialize.hpp gkfs.io/stat.cpp gkfs.io/write.cpp + gkfs.io/statx.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index f91c696f5f4b7b8b4c5afbd02dc789fa15915be4..97aa48564998f5891da27326c9f63e58e1abb0cb 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -41,4 +41,7 @@ stat_init(CLI::App& app); void write_init(CLI::App& app); +void +statx_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 50f5b3210b7dadcb2cd2b2e51696d6abb2fd0338..5b5ad9544d0575a92c0b4ba68782baa7d236ed3b 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -28,6 +28,7 @@ init_commands(CLI::App& app) { rmdir_init(app); stat_init(app); write_init(app); + statx_init(app); } diff --git a/tests/integration/harness/gkfs.io/serialize.hpp b/tests/integration/harness/gkfs.io/serialize.hpp index 739b89ee82517d787aac4d2eda8e57d77ba03c4a..ceb22db803c0baa550a4875dbf6944324d37e425 100644 --- a/tests/integration/harness/gkfs.io/serialize.hpp +++ b/tests/integration/harness/gkfs.io/serialize.hpp @@ -105,6 +105,20 @@ struct adl_serializer { } }; +// ADL specialization for struct ::statx_timestamp type +template <> +struct adl_serializer { + static void to_json(json& j, const struct ::statx_timestamp opt) { + + j = json { + { "tv_sec", opt.tv_sec }, + { "tv_nsec", opt.tv_nsec } + }; + } +}; + + + // ADL specialization for struct ::dirent type template <> struct adl_serializer { @@ -148,6 +162,37 @@ struct adl_serializer { } }; +// ADL specialization for struct ::statx type +template <> +struct adl_serializer { + static void to_json(json& j, const struct ::statx opt) { + + j = json { + { "stx_mask", opt.stx_mask }, + { "stx_blksize", opt.stx_blksize }, + { "stx_attributes", opt.stx_attributes}, + { "stx_nlink", opt.stx_nlink }, + { "stx_uid", opt.stx_uid }, + { "stx_gid", opt.stx_gid }, + { "stx_mode", opt.stx_mode }, + { "stx_ino", opt.stx_ino }, + { "stx_size", opt.stx_size }, + { "stx_blocks", opt.stx_blocks }, + { "stx_attributes_mask", opt.stx_attributes_mask}, + { "stx_atime", opt.stx_atime }, + { "stx_btime", opt.stx_btime }, + { "stx_ctime", opt.stx_ctime }, + { "stx_mtime", opt.stx_mtime }, + + { "stx_rdev_major", opt.stx_rdev_major }, + { "stx_rdev_minor", opt.stx_rdev_minor }, + { "stx_dev_major", opt.stx_dev_major }, + { "stx_dev_minor", opt.stx_dev_minor } + + }; + } +}; + } // namespace nlohmann namespace fmt { diff --git a/tests/integration/harness/gkfs.io/statx.cpp b/tests/integration/harness/gkfs.io/statx.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0ffe481ba7eabd6688a7642af8779adfd36f4783 --- /dev/null +++ b/tests/integration/harness/gkfs.io/statx.cpp @@ -0,0 +1,138 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include /* Definition of AT_* constants */ + +using json = nlohmann::json; + +/* int statx(int dirfd, const char *pathname, int flags, + unsigned int mask, struct statx *statxbuf); +*/ + +struct statx_options { + bool verbose; + int dirfd; + std::string pathname; + int flags; + unsigned int mask; + + REFL_DECL_STRUCT(statx_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(int, dirfd), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(int, flags), + REFL_DECL_MEMBER(unsigned int, mask) + ); +}; + +struct statx_output { + int retval; + int errnum; + struct ::statx statbuf; + + REFL_DECL_STRUCT(statx_output, + REFL_DECL_MEMBER(int, retval), + REFL_DECL_MEMBER(int, errnum), + REFL_DECL_MEMBER(struct ::statx, statbuf) + ); +}; + +void +to_json(json& record, + const statx_output& out) { + record = serialize(out); +} + +void +statx_exec(const statx_options& opts) { + + struct ::statx statbuf; + + int rv = ::statx(opts.dirfd, opts.pathname.c_str(), opts.flags, opts.mask, &statbuf); + + if(opts.verbose) { + fmt::print("statx(dirfd={}, pathname=\"{}\", flags={}, mask={}) = {}, errno: {} [{}]\n", + opts.dirfd, opts.pathname, opts.flags, opts.mask, rv, errno, ::strerror(errno)); + return; + } + + json out = statx_output{rv, errno, statbuf}; + fmt::print("{}\n", out.dump(2)); +} + +void +statx_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "statx", + "Execute the statx() system call"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human readable output" + ); + + cmd->add_option( + "dirfd", + opts->dirfd, + "file descritor" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "flags", + opts->flags, + "Flags" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "mask", + opts->mask, + "Mask" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + statx_exec(*opts); + }); +} + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 798227c9c58cee6af16e6fefc7774d55470c708e..cf786150a92d5e8d6995672218b59312f77eb467 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -67,6 +67,51 @@ class StructStatSchema(Schema): 'st_gid', 'st_rdev', 'st_size', 'st_blksize', 'st_blocks', 'st_atim', 'st_mtim', 'st_ctim'])(**data) + +class StructStatxTimestampSchema(Schema): + """Schema that deserializes a struct timespec""" + tv_sec = fields.Integer(required=True) + tv_nsec = fields.Integer(required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('StructStatxTimestampSchema', + ['tv_sec', 'tv_nsec'])(**data) + +class StructStatxSchema(Schema): + """Schema that deserializes a struct stat""" + + stx_mask = fields.Integer(required=True) + stx_blksize = fields.Integer(required=True) + stx_attributes = fields.Integer(required=True) + stx_nlink = fields.Integer(required=True) + stx_uid = fields.Integer(required=True) + stx_gid = fields.Integer(required=True) + stx_mode = fields.Integer(required=True) + stx_ino = fields.Integer(required=True) + stx_size = fields.Integer(required=True) + stx_blocks = fields.Integer(required=True) + stx_attributes_mask = fields.Integer(required=True) + + stx_atime = fields.Nested(StructStatxTimestampSchema) + stx_btime = fields.Nested(StructStatxTimestampSchema) + stx_ctime = fields.Nested(StructStatxTimestampSchema) + stx_mtime = fields.Nested(StructStatxTimestampSchema) + + stx_rdev_major = fields.Integer(required=True) + stx_rdev_minor = fields.Integer(required=True) + stx_dev_major = fields.Integer(required=True) + stx_dev_minor = fields.Integer(required=True) + + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('StructStatx', + ['stx_mask', 'stx_blksize', 'stx_attributes', 'stx_nlink', 'stx_uid', + 'stx_gid', 'stx_mode', 'stx_ino', 'stx_size', 'stx_blocks', 'stx_attributes_mask', + 'stx_atime', 'stx_btime', 'stx_ctime', 'stx_mtime', 'stx_rdev_major', + 'stx_rdev_minor', 'stx_dev_major', 'stx_dev_minor'])(**data) + class DirentStruct(Schema): """Schema that deserializes a struct dirent""" @@ -162,6 +207,17 @@ class StatOutputSchema(Schema): return namedtuple('StatReturn', ['retval', 'statbuf', 'errno'])(**data) +class StatxOutputSchema(Schema): + """Schema to deserialize the results of a stat() execution""" + + retval = fields.Integer(required=True) + statbuf = fields.Nested(StructStatxSchema, required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('StatxReturn', ['retval', 'statbuf', 'errno'])(**data) + class IOParser: OutputSchemas = { @@ -173,6 +229,7 @@ class IOParser: 'rmdir' : RmdirOutputSchema(), 'write' : WriteOutputSchema(), 'stat' : StatOutputSchema(), + 'statx' : StatxOutputSchema(), } def parse(self, command, output): diff --git a/tests/integration/status/README.md b/tests/integration/status/README.md new file mode 100644 index 0000000000000000000000000000000000000000..209ec79541a0a2ed113c87ea126b26ca8d431575 --- /dev/null +++ b/tests/integration/status/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any status-related +functionalities in GekkoFS. diff --git a/tests/integration/status/test_status.py b/tests/integration/status/test_status.py new file mode 100644 index 0000000000000000000000000000000000000000..d3684a241d288e0e2ce085c1430ca5b27487213d --- /dev/null +++ b/tests/integration/status/test_status.py @@ -0,0 +1,84 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + + + +#@pytest.mark.xfail(reason="invalid errno returned on success") +def test_statx(gkfs_daemon, gkfs_client): + """Test several statx commands""" + topdir = gkfs_daemon.mountdir / "top" + longer = Path(topdir.parent, topdir.name + "_plus") + dir_a = topdir / "dir_a" + dir_b = topdir / "dir_b" + file_a = topdir / "file_a" + subdir_a = dir_a / "subdir_a" + + # create topdir + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + + # test statx on existing dir + ret = gkfs_client.statx(0, topdir, 0, 0) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + assert stat.S_ISDIR(ret.statbuf.stx_mode) + + ret = gkfs_client.open(file_a, + os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval != -1 + + + # test statx on existing file + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0) + assert (ret.statbuf.stx_size == 0) + + + buf = b'42' + ret = gkfs_client.write(file_a, buf, 2) + + assert ret.retval == 2 + + # test statx on existing file + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (ret.statbuf.stx_size == 2) + + + return + +