Commit 655b1f8b authored by Alberto Miranda's avatar Alberto Miranda
Browse files

Merge branch 'rnou/216-increase-testing-coverage' into 'master'

Resolve "increase testing coverage"

This MR increases testing coverage significantly by including additional tests and implementing basic syscall testing.

It also fixes the following issues:
* Include `<array>` to compile with GCC 12.1
* Solved SIGSEV in fchdir (When debug is enabled)

Closes #216

See merge request !141
parents fcf7fb60 b1b3e519
Pipeline #2793 passed with stages
in 15 minutes and 21 seconds
......@@ -49,6 +49,12 @@ gkfs:
interruptible: true
needs: []
script:
# Change config.hpp with sed to enable extra features
- sed -i 's/constexpr auto use_atime = false;/constexpr auto use_atime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_ctime = false;/constexpr auto use_ctime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_mtime = false;/constexpr auto use_mtime = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_link_cnt = false;/constexpr auto use_link_cnt = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- sed -i 's/constexpr auto use_blocks = false;/constexpr auto use_blocks = true;/g' "${CI_PROJECT_DIR}/include/config.hpp"
- mkdir -p ${BUILD_PATH} && cd ${BUILD_PATH}
- cmake
-Wdev
......@@ -131,7 +137,7 @@ gkfs:integration:
needs: ['gkfs']
parallel:
matrix:
- SUBTEST: [ data, directories, operations, position, shell, status ]
- SUBTEST: [ data, status, syscalls, directories, operations, position, shell ]
script:
## run tests
......@@ -227,6 +233,7 @@ gkfs:unit:
- ctest -j $(nproc) -L unit::all --output-junit report.xml
## capture coverage information
- cd ${BUILD_PATH}
- ${CI_SCRIPTS_DIR}/coverage.sh
--verbose
--capture unit
......
......@@ -7,6 +7,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
### New
- Additional tests to increase code coverage ([!141](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141)).
- GKFS_ENABLE_UNUSED_FUNCTIONS added to disable code to increase code coverage.
### Changed
- Support parallelism for path resolution tests ([!145](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/145)).
- Support parallelism for symlink tests ([!147](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/147)).
......@@ -14,7 +16,11 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
### Fixed
- Using `unlink` now fails if it is a directory unless the `AT_REMOVEDIR` flag is used (POSIX compliance) ([!139](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/139)).
- Support glibc-2.34 or newer with syscall_intercept [!146](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/146)).
- Additional `#ifdef` to remove unused code ([!141](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141))
### Removed
### Fixed
- Using `unlink` now fails if it is a directory unless the `AT_REMOVEDIR` flag is used (POSIX compliance) ([!139](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/139)).
- fchdir generate a SIGSEV in debug mode (due to log) ([!141](https://storage.bsc.es/gitlab/hpc/gekkofs/-/merge_requests/141))
## [0.9.1] - 2022-04-29
### New
......
......@@ -178,6 +178,8 @@ if (GKFS_ENABLE_AGIOS)
find_package(AGIOS REQUIRED)
endif ()
option(GKFS_ENABLE_UNUSED_FUNCTIONS "Enable unused functions compilation" OFF)
option(GKFS_ENABLE_PARALLAX "Enable Parallax db backend" OFF)
option(GKFS_ENABLE_ROCKSDB "Enable ROCKSDB backend" ON)
......
......@@ -34,6 +34,7 @@
#include <mutex>
#include <memory>
#include <atomic>
#include <array>
namespace gkfs::filemap {
......
......@@ -44,8 +44,10 @@ bool_to_merc_bool(bool state);
std::string
get_my_hostname(bool short_hostname = false);
#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS
std::string
get_host_by_name(const std::string& hostname);
#endif
} // namespace gkfs::rpc
......
......@@ -411,6 +411,7 @@ gkfs_statfs(struct statfs* buf) {
return 0;
}
#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS
/**
* gkfs wrapper for statvfs() system calls
* errno may be set
......@@ -444,6 +445,7 @@ gkfs_statvfs(struct statvfs* buf) {
ST_NOATIME | ST_NODIRATIME | ST_NOSUID | ST_NODEV | ST_SYNCHRONOUS;
return 0;
}
#endif
/**
* gkfs wrapper for lseek() system calls with available file descriptor
......@@ -1094,7 +1096,7 @@ gkfs_getdents64(unsigned int fd, struct linux_dirent64* dirp,
#ifdef HAS_SYMLINKS
#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS
/**
* gkfs wrapper for make symlink() system calls
* errno may be set
......@@ -1177,7 +1179,7 @@ gkfs_readlink(const std::string& path, char* buf, int bufsize) {
std::strcpy(buf + CTX->mountdir().size(), md->target_path().c_str());
return path_size;
}
#endif
#endif
} // namespace gkfs::syscall
......
......@@ -712,8 +712,8 @@ hook_fchdir(unsigned int fd) {
auto open_dir = CTX->file_map()->get_dir(fd);
if(open_dir == nullptr) {
// Cast did not succeeded: open_file is a regular file
LOG(ERROR, "{}() file descriptor refers to a normal file: '{}'",
__func__, open_dir->path());
LOG(ERROR, "{}() file descriptor refers to a normal file",
__func__);
return -EBADF;
}
......
......@@ -73,7 +73,7 @@ get_my_hostname(bool short_hostname) {
return ""s;
}
#ifdef GKFS_ENABLE_UNUSED_FUNCTIONS
string
get_host_by_name(const string& hostname) {
int err = 0;
......@@ -102,5 +102,6 @@ get_host_by_name(const string& hostname) {
freeaddrinfo(addr);
return addr_str;
}
#endif
} // namespace gkfs::rpc
\ No newline at end of file
......@@ -120,6 +120,13 @@ gkfs_add_python_test(
SOURCE data/
)
gkfs_add_python_test(
NAME test_syscalls
PYTHON_VERSION 3.6
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
SOURCE syscalls/
)
if(GKFS_INSTALL_TESTS)
install(DIRECTORY harness
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
......@@ -182,6 +189,14 @@ if(GKFS_INSTALL_TESTS)
PATTERN ".pytest_cache" EXCLUDE
)
install(DIRECTORY syscalls
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
FILES_MATCHING
REGEX ".*\\.py"
PATTERN "__pycache__" EXCLUDE
PATTERN ".pytest_cache" EXCLUDE
)
install(DIRECTORY shell
DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration
FILES_MATCHING
......
......@@ -30,6 +30,7 @@
# from pathlib import Path
import os
import stat
import errno
# @pytest.mark.xfail(reason="invalid errno returned on success")
......@@ -121,3 +122,35 @@ def test_truncate(gkfs_daemon, gkfs_client):
ret = gkfs_client.file_compare(truncfile, truncfile_verify_2, trunc_size)
assert ret.retval == 0
# Tests failure cases
def test_fail_truncate(gkfs_daemon, gkfs_client):
truncfile = gkfs_daemon.mountdir / "trunc_file2"
# open and create test file
ret = gkfs_client.open(truncfile, os.O_CREAT | os.O_WRONLY, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
assert ret.retval != -1
buf_length = 256
ret = gkfs_client.write_random(truncfile, buf_length)
assert ret.retval == buf_length
ret = gkfs_client.truncate(truncfile, buf_length)
assert ret.retval == 0
# Size should not change
ret = gkfs_client.stat(truncfile)
assert ret.statbuf.st_size == buf_length
# Truncate to a negative size
ret = gkfs_client.truncate(truncfile, -1)
assert ret.retval == -1
assert ret.errno == errno.EINVAL
# Truncate to a size greater than the file size
ret = gkfs_client.truncate(truncfile, buf_length + 1)
assert ret.retval == -1
assert ret.errno == errno.EINVAL
......@@ -189,6 +189,15 @@ def test_mkdir(gkfs_daemon, gkfs_client):
assert d.d_name == e[0]
assert d.d_type == e[1]
# Try to open a file as a dir
ret = gkfs_client.opendir(file_a)
assert ret.dirp is None
assert ret.errno == errno.ENOTDIR
# Try to remove a non empty dir
ret = gkfs_client.rmdir(topdir)
assert ret.retval == -1
assert ret.errno == errno.ENOTEMPTY
return
......
......@@ -63,6 +63,10 @@ add_executable(gkfs.io
gkfs.io/symlink.cpp
gkfs.io/directory_validate.cpp
gkfs.io/unlink.cpp
gkfs.io/access.cpp
gkfs.io/statfs.cpp
gkfs.io/dup_validate.cpp
gkfs.io/syscall_coverage.cpp
)
include(FetchContent)
......
/*
Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
This file is part of GekkoFS.
GekkoFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
GekkoFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.
SPDX-License-Identifier: GPL-3.0-or-later
*/
/* C++ includes */
#include <CLI11/CLI11.hpp>
#include <nlohmann/json.hpp>
#include <memory>
#include <fmt/format.h>
#include <reflection.hpp>
#include <serialize.hpp>
/* C includes */
#include <sys/types.h>
#include <unistd.h>
using json = nlohmann::json;
struct access_options {
bool verbose{};
std::string path{};
int mask{};
REFL_DECL_STRUCT(access_options, REFL_DECL_MEMBER(bool, verbose),
REFL_DECL_MEMBER(std::string, path),
REFL_DECL_MEMBER(int, mask));
};
struct access_output {
int retval;
int errnum;
REFL_DECL_STRUCT(access_output, REFL_DECL_MEMBER(int, retval),
REFL_DECL_MEMBER(int, errnum));
};
void
to_json(json& record, const access_output& out) {
record = serialize(out);
}
void
access_exec(const access_options& opts) {
auto rv = ::access(opts.path.c_str(), opts.mask);
if(rv == -1) {
if(opts.verbose) {
fmt::print(
"access(path=\"{}\", mask={}) = {}, errno: {} [{}]\n",
opts.path, opts.mask, rv, errno, ::strerror(errno));
return;
}
}
json out = access_output{rv, errno};
fmt::print("{}\n", out.dump(2));
}
void
access_init(CLI::App& app) {
// Create the option and subcommand objects
auto opts = std::make_shared<access_options>();
auto* cmd = app.add_subcommand("access",
"Execute the access() system call");
// Add options to cmd, binding them to opts
cmd->add_flag("-v,--verbose", opts->verbose,
"Produce human writeable output");
cmd->add_option("path", opts->path, "Path to file")
->required()
->type_name("");
cmd->add_option("mask", opts->mask,
"Mask to apply to the access check")
->required()
->type_name("");
cmd->callback([opts]() { access_exec(*opts); });
}
......@@ -96,6 +96,12 @@ write_random_init(CLI::App& app);
void
truncate_init(CLI::App& app);
void
access_init(CLI::App& app);
void
statfs_init(CLI::App& app);
// UTIL
void
file_compare_init(CLI::App& app);
......@@ -111,4 +117,10 @@ symlink_init(CLI::App& app);
void
unlink_init(CLI::App& app);
void
dup_validate_init(CLI::App& app);
void
syscall_coverage_init(CLI::App& app);
#endif // IO_COMMANDS_HPP
/*
Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
This file is part of GekkoFS.
GekkoFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
GekkoFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.
SPDX-License-Identifier: GPL-3.0-or-later
*/
/* C++ includes */
#include <CLI11/CLI11.hpp>
#include <nlohmann/json.hpp>
#include <memory>
#include <fmt/format.h>
#include <commands.hpp>
#include <reflection.hpp>
#include <serialize.hpp>
#include <binary_buffer.hpp>
/* C includes */
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
using json = nlohmann::json;
struct dup_validate_options {
bool verbose{};
std::string pathname;
REFL_DECL_STRUCT(dup_validate_options, REFL_DECL_MEMBER(bool, verbose),
REFL_DECL_MEMBER(std::string, pathname));
};
struct dup_validate_output {
int retval;
int errnum;
REFL_DECL_STRUCT(dup_validate_output, REFL_DECL_MEMBER(int, retval),
REFL_DECL_MEMBER(int, errnum));
};
void
to_json(json& record, const dup_validate_output& out) {
record = serialize(out);
}
void
dup_validate_exec(const dup_validate_options& opts) {
int fd = ::open(opts.pathname.c_str(), O_WRONLY);
if(fd == -1) {
if(opts.verbose) {
fmt::print(
"dup_validate(pathname=\"{}\") = {}, errno: {} [{}]\n",
opts.pathname, fd, errno, ::strerror(errno));
return;
}
json out = dup_validate_output{fd, errno};
fmt::print("{}\n", out.dump(2));
return;
}
auto rv = ::dup(fd);
auto rv2 = ::dup2(fd, rv);
if(opts.verbose) {
fmt::print(
"dup_validate(pathname=\"{}\") = {}, errno: {} [{}]\n",
opts.pathname, rv, errno, ::strerror(errno));
return;
}
if(rv < 0 || rv2 < 0) {
json out = dup_validate_output{(int) rv, errno};
fmt::print("{}\n", out.dump(2));
return;
}
rv = 0;
errno = 0;
json out = dup_validate_output{(int) rv, errno};
fmt::print("{}\n", out.dump(2));
return;
}
void
dup_validate_init(CLI::App& app) {
// Create the option and subcommand objects
auto opts = std::make_shared<dup_validate_options>();
auto* cmd = app.add_subcommand(
"dup_validate",
"Execute dup, dup2, and dup3 returns 0 if the file descriptor is valid");
// Add options to cmd, binding them to opts
cmd->add_flag("-v,--verbose", opts->verbose,
"Produce human writeable output");
cmd->add_option("pathname", opts->pathname, "File name")
->required()
->type_name("");
cmd->callback([opts]() { dup_validate_exec(*opts); });
}
......@@ -57,12 +57,16 @@ init_commands(CLI::App& app) {
directory_validate_init(app);
write_random_init(app);
truncate_init(app);
access_init(app);
statfs_init(app);
// utils
file_compare_init(app);
chdir_init(app);
getcwd_validate_init(app);
symlink_init(app);
unlink_init(app);
dup_validate_init(app);
syscall_coverage_init(app);
}
......
......@@ -34,6 +34,8 @@
extern "C" {
#include <sys/stat.h>
#include <sys/vfs.h>
#include <sys/statvfs.h>
}
template <typename T>
......@@ -171,6 +173,21 @@ struct adl_serializer<struct ::stat> {
}
};
// ADL specialization for struct ::statfs (not exhaustive) type
template <>
struct adl_serializer<struct ::statfs> {
static void
to_json(json& j, const struct ::statfs opt) {
j = json{{"f_type", opt.f_type},
{"f_bsize", opt.f_bsize},
{"f_blocks", opt.f_blocks},
{"f_bfree", opt.f_bfree},
{"f_bavail", opt.f_bavail},
{"f_files", opt.f_files},
{"f_ffree", opt.f_ffree}};
}
};
#ifdef STATX_TYPE
// ADL specialization for struct ::statx type
template <>
......
/*
Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
This software was partially supported by the
EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
This software was partially supported by the
ADA-FS project under the SPPEXA project funded by the DFG.
This file is part of GekkoFS.
GekkoFS is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
GekkoFS is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with GekkoFS. If not, see <https://www.gnu.org/licenses/>.
SPDX-License-Identifier: GPL-3.0-or-later
*/
/* C++ includes */
#include <CLI11/CLI11.hpp>
#include <nlohmann/json.hpp>
#include <memory>
#include <fmt/format.h>
#include <commands.hpp>
#include <reflection.hpp>
#include <serialize.hpp>
/* C includes */
#include <sys/types.h>
#include <sys/vfs.h>
#include <unistd.h>
using json = nlohmann::json;
struct statfs_options {
bool verbose{};
std::string pathname