diff --git a/tests/integration/data/test_data_integrity.py b/tests/integration/data/test_data_integrity.py index cb5587ebf5fd04b5c89790f7aa882e50d5cd1cc8..50e07d29a5beaaf106b4064fcced609c82da6ca5 100644 --- a/tests/integration/data/test_data_integrity.py +++ b/tests/integration/data/test_data_integrity.py @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -32,23 +32,31 @@ import errno import stat import os import ctypes -import sh import sys import pytest import string import random from harness.logger import logger -nonexisting = "nonexisting" -chunksize_start = 128192 -chunksize_end = 2097153 -step = 4096*9 def generate_random_data(size): return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)]) +def check_write(gkfs_client, i, file): + buf = bytes(generate_random_data(i), sys.stdout.encoding) + + ret = gkfs_client.write(file, buf, i) + assert ret.retval == i + ret = gkfs_client.stat(file) + + assert ret.retval == 0 + assert (ret.statbuf.st_size == i) + + ret = gkfs_client.read(file, i) + assert ret.retval== i + assert ret.buf == buf #@pytest.mark.xfail(reason="invalid errno returned on success") def test_data_integrity(gkfs_daemon, gkfs_client): @@ -89,29 +97,36 @@ def test_data_integrity(gkfs_daemon, gkfs_client): # Read data # Compare buffer - - for i in range (1, 512, 64): - buf = bytes(generate_random_data(i), sys.stdout.encoding) - - ret = gkfs_client.write(file_a, buf, i) + ret = gkfs_client.write_validate(file_a, 1) + assert ret.retval == 1 - assert ret.retval == i - ret = gkfs_client.stat(file_a) + ret = gkfs_client.write_validate(file_a, 256) + assert ret.retval == 1 - assert ret.retval == 0 - assert (ret.statbuf.st_size == i) + ret = gkfs_client.write_validate(file_a, 512) + assert ret.retval == 1 - ret = gkfs_client.read(file_a, i) - assert ret.retval== i - assert ret.buf == buf + # Step 2 - Compare bigger sizes exceeding typical chunksize and not aligned + ret = gkfs_client.write_validate(file_a, 128192) + assert ret.retval == 1 + # < 1 chunk + ret = gkfs_client.write_validate(file_a, 400000) + assert ret.retval == 1 - # Step 2 - Compare bigger sizes exceeding typical chunksize - for i in range (chunksize_start, chunksize_end, step): - ret = gkfs_client.write_validate(file_a, i) - assert ret.retval == 1 + # > 1 chunk < 2 chunks + ret = gkfs_client.write_validate(file_a, 600000) + assert ret.retval == 1 + # > 1 chunk < 2 chunks + ret = gkfs_client.write_validate(file_a, 900000) + assert ret.retval == 1 - return + # > 2 chunks + ret = gkfs_client.write_validate(file_a, 1100000) + assert ret.retval == 1 + # > 4 chunks + ret = gkfs_client.write_validate(file_a, 2097153) + assert ret.retval == 1 diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 90cfcee482de4ea5bc2da082bd9c66e708c56f74..066b36962f9c77b5fdc59365dedce2b1a72e1711 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -32,7 +32,6 @@ import errno import stat import os import ctypes -import sh import sys import pytest from harness.logger import logger @@ -198,9 +197,8 @@ def test_finedir(gkfs_daemon, gkfs_client): """Tests several corner cases for directories scan""" topdir = gkfs_daemon.mountdir / "finetop" - longer = Path(topdir.parent, topdir.name + "_fine") file_a = topdir / "file_" - file_b = topdir / "dir_b" + # create topdir ret = gkfs_client.mkdir( @@ -216,34 +214,19 @@ def test_finedir(gkfs_daemon, gkfs_client): # populate top directory - for files in range (1,11): - ret = gkfs_client.open( - str(file_a) + str(files), - os.O_CREAT, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - ret = gkfs_client.readdir(topdir) + for files in range (1,4): + ret = gkfs_client.directory_validate( + topdir, 1) + assert ret.retval == files - assert len(ret.dirents) == files - for files in range(11, 20): - ret = gkfs_client.open( - str(file_a) + str(files), - os.O_CREAT, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - ret = gkfs_client.readdir(topdir) - - assert len(ret.dirents) == 19 + ret = gkfs_client.directory_validate( + topdir, 1000) + assert ret.retval == 1000+3 def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): topdir = gkfs_daemon.mountdir / "test_extended" - longer = Path(topdir.parent, topdir.name + "_plus") dir_a = topdir / "dir_a" dir_b = topdir / "dir_b" file_a = topdir / "file_a" @@ -307,11 +290,3 @@ def test_opendir(gkfs_daemon, gkfs_client, directory_path): assert ret.dirp is None assert ret.errno == errno.ENOENT -# def test_stat(gkfs_daemon): -# pass -# -# def test_rmdir(gkfs_daemon): -# pass -# -# def test_closedir(gkfs_daemon): -# pass diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 762580c8684f1644e8a5b7e73aa62c082692eb27..a2aea938e94d55245b97656c8460d92fe82eadd2 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -61,6 +61,7 @@ add_executable(gkfs.io gkfs.io/chdir.cpp gkfs.io/getcwd_validate.cpp gkfs.io/symlink.cpp + gkfs.io/directory_validate.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 39b74e07fa8b5bf69c89eeb1a6351378b561d413..1425e7d9fea22ac5efafc88e359620daf20fccbf 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -87,6 +87,9 @@ lseek_init(CLI::App& app); void write_validate_init(CLI::App& app); +void +directory_validate_init(CLI::App& app); + void write_random_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/directory_validate.cpp b/tests/integration/harness/gkfs.io/directory_validate.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2d03c11b4c2fffd8fc3cfc2db9fde39827d10ec2 --- /dev/null +++ b/tests/integration/harness/gkfs.io/directory_validate.cpp @@ -0,0 +1,218 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +// Creates a file in the path, and does a readdir comparing the count size (may be accumulated with previous operations) +struct directory_validate_options { + bool verbose{}; + std::string pathname; + ::size_t count; + + REFL_DECL_STRUCT(directory_validate_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count)); +}; + +struct directory_validate_output { + int retval; + int errnum; + + REFL_DECL_STRUCT(directory_validate_output, REFL_DECL_MEMBER(int, retval), + REFL_DECL_MEMBER(int, errnum)); +}; + +void +to_json(json& record, const directory_validate_output& out) { + record = serialize(out); +} + + +/** + * returns the number of elements existing in the path + * @param opts + * @param dir Path checked + * @returns The number of elements in the directory + */ +int +number_of_elements(const directory_validate_options& opts, const std::string dir) { + int num_elements = 0; + + ::DIR* dirp = ::opendir(dir.c_str()); + + if(dirp == NULL) { + if(opts.verbose) { + fmt::print("readdir(pathname=\"{}\") = {}, errno: {} [{}]\n", + dir, "NULL", errno, ::strerror(errno)); + return -3; + } + + std::cout << "Error create directory" << std::endl; + json out = directory_validate_output{-3, errno}; + fmt::print("{}\n", out.dump(2)); + + return -3; + } + + struct ::dirent* entry; + + while((entry = ::readdir(dirp)) != NULL) { + num_elements++; + } + + return num_elements; +} + + +/** + * Creates `count` files, with a suffix starting at the number of elements existing in the path + * @param opts + * @param path Path where the file is created + * @param count Number of files to create + * @returns The number of elements created plus the number of elements already in the directory (calculated, not checked) + */ +int +create_n_files (const directory_validate_options& opts, const std::string path, int count) { + + // Read Directory and get number of entries + int num_elements = number_of_elements(opts, path); + + for (int i = num_elements; i < count+num_elements; i++) { + std::string filename = path+"/file_auto_"+std::to_string(i); + int fd = ::creat(filename.c_str(), S_IRWXU); + if(fd == -1) { + if(opts.verbose) { + fmt::print( + "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + filename, i, fd, errno, ::strerror(errno)); + return -2; + } + json out = directory_validate_output{-2, errno}; + fmt::print("{}\n", out.dump(2)); + + return -2; + } + + ::close(fd); + } + return num_elements+count; +} + +/** + * Creates `count` files, and returns the number of elements in the path + * If count == 0, the path is the filename to be created. Parent directory is checked + * @param opts + */ +void +directory_validate_exec(const directory_validate_options& opts) { + + if (opts.count == 0) { + int fd = ::creat(opts.pathname.c_str(), S_IRWXU); + + if(fd == -1) { + if(opts.verbose) { + fmt::print( + "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, fd, errno, ::strerror(errno)); + return; + } + json out = directory_validate_output{-2, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + ::close(fd); + } + else { + int created = create_n_files(opts, opts.pathname, opts.count); + if (created <= 0) return; + } + + // Do a readdir + std::string dir = opts.pathname; + + if (opts.count == 0) + dir = ::dirname((char*)opts.pathname.c_str()); + + auto num_elements = number_of_elements(opts, dir); + + if(opts.verbose) { + fmt::print("readdir(pathname=\"{}\") = [\n{}],\nerrno: {} [{}]\n", + opts.pathname, num_elements, errno, + ::strerror(errno)); + return; + } + + errno = 0; + json out = directory_validate_output{num_elements, errno}; + fmt::print("{}\n", out.dump(2)); + return; + +} + +void +directory_validate_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "directory_validate", + "Create count files in the directory and execute a direntry system call and returns the number of elements"); + + // Add options to cmd, binding them to opts + cmd->add_flag("-v,--verbose", opts->verbose, + "Produce human writeable output"); + + cmd->add_option("pathname", opts->pathname, "directory to check or filename to create (if count is 0), elements will be checked in the parent dir") + ->required() + ->type_name(""); + + cmd->add_option("count", opts->count, "Number of files to create. If 0, it creates only the entry in the pathname.") + ->required() + ->type_name(""); + + cmd->callback([opts]() { directory_validate_exec(*opts); }); +} diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index ee79fc70bc43b59d7a97b9c6c8289e2d1c8bd794..2233c0d81ecf43e1bdaace9432957b354d3b96a4 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -54,6 +54,7 @@ init_commands(CLI::App& app) { #endif lseek_init(app); write_validate_init(app); + directory_validate_init(app); write_random_init(app); truncate_init(app); // utils diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 37d2a8c13d2bbd85eee3f1c9303a3779d40c364e..6522cdad9394601fd5bd3ebbcc95dbbe45a14700 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -319,6 +319,15 @@ class WriteValidateOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data) +class DirectoryValidateOutputSchema(Schema): + """Schema to deserialize the results of a write() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('DirectoryValidateReturn', ['retval', 'errno'])(**data) class WriteRandomOutputSchema(Schema): """Schema to deserialize the results of a write() execution""" @@ -409,6 +418,7 @@ class IOParser: 'write_random': WriteRandomOutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'truncate': TruncateOutputSchema(), + 'directory_validate' : DirectoryValidateOutputSchema(), # UTIL 'file_compare': FileCompareOutputSchema(), 'chdir' : ChdirOutputSchema(),