From 667800a85aa0f8d4c09b2ff1800c4ccb7aa84b74 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Wed, 2 Feb 2022 12:45:43 +0100 Subject: [PATCH 1/2] Reduce test_integration and directory tests --- tests/integration/data/test_data_integrity.py | 63 +++++--- .../directories/test_directories.py | 31 +--- tests/integration/harness/CMakeLists.txt | 1 + .../integration/harness/gkfs.io/commands.hpp | 3 + .../harness/gkfs.io/directory_validate.cpp | 153 ++++++++++++++++++ tests/integration/harness/gkfs.io/main.cpp | 1 + tests/integration/harness/io.py | 10 ++ 7 files changed, 213 insertions(+), 49 deletions(-) create mode 100644 tests/integration/harness/gkfs.io/directory_validate.cpp diff --git a/tests/integration/data/test_data_integrity.py b/tests/integration/data/test_data_integrity.py index cb5587ebf..50e07d29a 100644 --- a/tests/integration/data/test_data_integrity.py +++ b/tests/integration/data/test_data_integrity.py @@ -1,6 +1,6 @@ ################################################################################ -# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain # -# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany # +# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany # # # # This software was partially supported by the # # EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # @@ -32,23 +32,31 @@ import errno import stat import os import ctypes -import sh import sys import pytest import string import random from harness.logger import logger -nonexisting = "nonexisting" -chunksize_start = 128192 -chunksize_end = 2097153 -step = 4096*9 def generate_random_data(size): return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)]) +def check_write(gkfs_client, i, file): + buf = bytes(generate_random_data(i), sys.stdout.encoding) + + ret = gkfs_client.write(file, buf, i) + assert ret.retval == i + ret = gkfs_client.stat(file) + + assert ret.retval == 0 + assert (ret.statbuf.st_size == i) + + ret = gkfs_client.read(file, i) + assert ret.retval== i + assert ret.buf == buf #@pytest.mark.xfail(reason="invalid errno returned on success") def test_data_integrity(gkfs_daemon, gkfs_client): @@ -89,29 +97,36 @@ def test_data_integrity(gkfs_daemon, gkfs_client): # Read data # Compare buffer - - for i in range (1, 512, 64): - buf = bytes(generate_random_data(i), sys.stdout.encoding) - - ret = gkfs_client.write(file_a, buf, i) + ret = gkfs_client.write_validate(file_a, 1) + assert ret.retval == 1 - assert ret.retval == i - ret = gkfs_client.stat(file_a) + ret = gkfs_client.write_validate(file_a, 256) + assert ret.retval == 1 - assert ret.retval == 0 - assert (ret.statbuf.st_size == i) + ret = gkfs_client.write_validate(file_a, 512) + assert ret.retval == 1 - ret = gkfs_client.read(file_a, i) - assert ret.retval== i - assert ret.buf == buf + # Step 2 - Compare bigger sizes exceeding typical chunksize and not aligned + ret = gkfs_client.write_validate(file_a, 128192) + assert ret.retval == 1 + # < 1 chunk + ret = gkfs_client.write_validate(file_a, 400000) + assert ret.retval == 1 - # Step 2 - Compare bigger sizes exceeding typical chunksize - for i in range (chunksize_start, chunksize_end, step): - ret = gkfs_client.write_validate(file_a, i) - assert ret.retval == 1 + # > 1 chunk < 2 chunks + ret = gkfs_client.write_validate(file_a, 600000) + assert ret.retval == 1 + # > 1 chunk < 2 chunks + ret = gkfs_client.write_validate(file_a, 900000) + assert ret.retval == 1 - return + # > 2 chunks + ret = gkfs_client.write_validate(file_a, 1100000) + assert ret.retval == 1 + # > 4 chunks + ret = gkfs_client.write_validate(file_a, 2097153) + assert ret.retval == 1 diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 90cfcee48..10bd74346 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -32,7 +32,6 @@ import errno import stat import os import ctypes -import sh import sys import pytest from harness.logger import logger @@ -198,9 +197,8 @@ def test_finedir(gkfs_daemon, gkfs_client): """Tests several corner cases for directories scan""" topdir = gkfs_daemon.mountdir / "finetop" - longer = Path(topdir.parent, topdir.name + "_fine") file_a = topdir / "file_" - file_b = topdir / "dir_b" + # create topdir ret = gkfs_client.mkdir( @@ -216,29 +214,12 @@ def test_finedir(gkfs_daemon, gkfs_client): # populate top directory - for files in range (1,11): - ret = gkfs_client.open( - str(file_a) + str(files), - os.O_CREAT, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - ret = gkfs_client.readdir(topdir) - - assert len(ret.dirents) == files + for files in range (1,4): + file_name = str(file_a) + str(files) + ret = gkfs_client.directory_validate( + Path(file_name), files) + assert ret.retval == files - for files in range(11, 20): - ret = gkfs_client.open( - str(file_a) + str(files), - os.O_CREAT, - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - - assert ret.retval != -1 - - ret = gkfs_client.readdir(topdir) - - assert len(ret.dirents) == 19 def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 762580c86..a2aea938e 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -61,6 +61,7 @@ add_executable(gkfs.io gkfs.io/chdir.cpp gkfs.io/getcwd_validate.cpp gkfs.io/symlink.cpp + gkfs.io/directory_validate.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 39b74e07f..1425e7d9f 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -87,6 +87,9 @@ lseek_init(CLI::App& app); void write_validate_init(CLI::App& app); +void +directory_validate_init(CLI::App& app); + void write_random_init(CLI::App& app); diff --git a/tests/integration/harness/gkfs.io/directory_validate.cpp b/tests/integration/harness/gkfs.io/directory_validate.cpp new file mode 100644 index 000000000..26aa33ef2 --- /dev/null +++ b/tests/integration/harness/gkfs.io/directory_validate.cpp @@ -0,0 +1,153 @@ +/* + Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + This file is part of GekkoFS. + + GekkoFS is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + GekkoFS is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GekkoFS. If not, see . + + SPDX-License-Identifier: GPL-3.0-or-later +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include +#include + +using json = nlohmann::json; + +// Creates a file in the path, and does a readdir comparing the count size (may be accumulated with previous operations) +struct directory_validate_options { + bool verbose{}; + std::string pathname; + ::size_t count; + + REFL_DECL_STRUCT(directory_validate_options, REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count)); +}; + +struct directory_validate_output { + int retval; + int errnum; + + REFL_DECL_STRUCT(directory_validate_output, REFL_DECL_MEMBER(int, retval), + REFL_DECL_MEMBER(int, errnum)); +}; + +void +to_json(json& record, const directory_validate_output& out) { + record = serialize(out); +} + +void +directory_validate_exec(const directory_validate_options& opts) { + + int fd = ::creat(opts.pathname.c_str(), S_IRWXU); + + if(fd == -1) { + if(1) { + fmt::print( + "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, fd, errno, ::strerror(errno)); + return; + } + json out = directory_validate_output{-2, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + ::close(fd); + + // Do a readdir + std::string dir = ::dirname((char*)opts.pathname.c_str()); + ::DIR* dirp = ::opendir(dir.c_str()); + + if(dirp == NULL) { + if(opts.verbose) { + fmt::print("readdir(pathname=\"{}\") = {}, errno: {} [{}]\n", + opts.pathname, "NULL", errno, ::strerror(errno)); + return; + } + + std::cout << "Error create directory" << std::endl; + json out = directory_validate_output{-3, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + std::vector entries; + struct ::dirent* entry; + + while((entry = ::readdir(dirp)) != NULL) { + entries.push_back(*entry); + } + + if(opts.verbose) { + fmt::print("readdir(pathname=\"{}\") = [\n{}],\nerrno: {} [{}]\n", + opts.pathname, fmt::join(entries, ",\n"), errno, + ::strerror(errno)); + return; + } + + errno = 0; + json out = directory_validate_output{(int) entries.size(), errno}; + fmt::print("{}\n", out.dump(2)); + return; + +} + +void +directory_validate_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "directory_validate", + "Create a file and execute a direntry system call and count the number of elements"); + + // Add options to cmd, binding them to opts + cmd->add_flag("-v,--verbose", opts->verbose, + "Produce human writeable output"); + + cmd->add_option("pathname", opts->pathname, "file name, entries will be checked in last directory") + ->required() + ->type_name(""); + + cmd->add_option("count", opts->count, "Number of files to check") + ->required() + ->type_name(""); + + cmd->callback([opts]() { directory_validate_exec(*opts); }); +} diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index ee79fc70b..2233c0d81 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -54,6 +54,7 @@ init_commands(CLI::App& app) { #endif lseek_init(app); write_validate_init(app); + directory_validate_init(app); write_random_init(app); truncate_init(app); // utils diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index 37d2a8c13..6522cdad9 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -319,6 +319,15 @@ class WriteValidateOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data) +class DirectoryValidateOutputSchema(Schema): + """Schema to deserialize the results of a write() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('DirectoryValidateReturn', ['retval', 'errno'])(**data) class WriteRandomOutputSchema(Schema): """Schema to deserialize the results of a write() execution""" @@ -409,6 +418,7 @@ class IOParser: 'write_random': WriteRandomOutputSchema(), 'write_validate' : WriteValidateOutputSchema(), 'truncate': TruncateOutputSchema(), + 'directory_validate' : DirectoryValidateOutputSchema(), # UTIL 'file_compare': FileCompareOutputSchema(), 'chdir' : ChdirOutputSchema(), -- GitLab From 382f0fd19036a3d8677fd439c2c3ab831d99ae8e Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 4 Feb 2022 10:07:14 +0100 Subject: [PATCH 2/2] Improved test_directories, check 1000 files --- .../directories/test_directories.py | 16 +-- .../harness/gkfs.io/directory_validate.cpp | 125 +++++++++++++----- 2 files changed, 100 insertions(+), 41 deletions(-) diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py index 10bd74346..066b36962 100644 --- a/tests/integration/directories/test_directories.py +++ b/tests/integration/directories/test_directories.py @@ -215,16 +215,18 @@ def test_finedir(gkfs_daemon, gkfs_client): # populate top directory for files in range (1,4): - file_name = str(file_a) + str(files) ret = gkfs_client.directory_validate( - Path(file_name), files) + topdir, 1) assert ret.retval == files + ret = gkfs_client.directory_validate( + topdir, 1000) + assert ret.retval == 1000+3 + def test_extended(gkfs_daemon, gkfs_shell, gkfs_client): topdir = gkfs_daemon.mountdir / "test_extended" - longer = Path(topdir.parent, topdir.name + "_plus") dir_a = topdir / "dir_a" dir_b = topdir / "dir_b" file_a = topdir / "file_a" @@ -288,11 +290,3 @@ def test_opendir(gkfs_daemon, gkfs_client, directory_path): assert ret.dirp is None assert ret.errno == errno.ENOENT -# def test_stat(gkfs_daemon): -# pass -# -# def test_rmdir(gkfs_daemon): -# pass -# -# def test_closedir(gkfs_daemon): -# pass diff --git a/tests/integration/harness/gkfs.io/directory_validate.cpp b/tests/integration/harness/gkfs.io/directory_validate.cpp index 26aa33ef2..2d03c11b4 100644 --- a/tests/integration/harness/gkfs.io/directory_validate.cpp +++ b/tests/integration/harness/gkfs.io/directory_validate.cpp @@ -69,60 +69,125 @@ to_json(json& record, const directory_validate_output& out) { record = serialize(out); } -void -directory_validate_exec(const directory_validate_options& opts) { - - int fd = ::creat(opts.pathname.c_str(), S_IRWXU); - - if(fd == -1) { - if(1) { - fmt::print( - "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", - opts.pathname, opts.count, fd, errno, ::strerror(errno)); - return; - } - json out = directory_validate_output{-2, errno}; - fmt::print("{}\n", out.dump(2)); - - return; - } - ::close(fd); +/** + * returns the number of elements existing in the path + * @param opts + * @param dir Path checked + * @returns The number of elements in the directory + */ +int +number_of_elements(const directory_validate_options& opts, const std::string dir) { + int num_elements = 0; - // Do a readdir - std::string dir = ::dirname((char*)opts.pathname.c_str()); ::DIR* dirp = ::opendir(dir.c_str()); if(dirp == NULL) { if(opts.verbose) { fmt::print("readdir(pathname=\"{}\") = {}, errno: {} [{}]\n", - opts.pathname, "NULL", errno, ::strerror(errno)); - return; + dir, "NULL", errno, ::strerror(errno)); + return -3; } std::cout << "Error create directory" << std::endl; json out = directory_validate_output{-3, errno}; fmt::print("{}\n", out.dump(2)); - return; + return -3; } - std::vector entries; struct ::dirent* entry; while((entry = ::readdir(dirp)) != NULL) { - entries.push_back(*entry); + num_elements++; } + return num_elements; +} + + +/** + * Creates `count` files, with a suffix starting at the number of elements existing in the path + * @param opts + * @param path Path where the file is created + * @param count Number of files to create + * @returns The number of elements created plus the number of elements already in the directory (calculated, not checked) + */ +int +create_n_files (const directory_validate_options& opts, const std::string path, int count) { + + // Read Directory and get number of entries + int num_elements = number_of_elements(opts, path); + + for (int i = num_elements; i < count+num_elements; i++) { + std::string filename = path+"/file_auto_"+std::to_string(i); + int fd = ::creat(filename.c_str(), S_IRWXU); + if(fd == -1) { + if(opts.verbose) { + fmt::print( + "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + filename, i, fd, errno, ::strerror(errno)); + return -2; + } + json out = directory_validate_output{-2, errno}; + fmt::print("{}\n", out.dump(2)); + + return -2; + } + + ::close(fd); + } + return num_elements+count; +} + +/** + * Creates `count` files, and returns the number of elements in the path + * If count == 0, the path is the filename to be created. Parent directory is checked + * @param opts + */ +void +directory_validate_exec(const directory_validate_options& opts) { + + if (opts.count == 0) { + int fd = ::creat(opts.pathname.c_str(), S_IRWXU); + + if(fd == -1) { + if(opts.verbose) { + fmt::print( + "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, fd, errno, ::strerror(errno)); + return; + } + json out = directory_validate_output{-2, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + ::close(fd); + } + else { + int created = create_n_files(opts, opts.pathname, opts.count); + if (created <= 0) return; + } + + // Do a readdir + std::string dir = opts.pathname; + + if (opts.count == 0) + dir = ::dirname((char*)opts.pathname.c_str()); + + auto num_elements = number_of_elements(opts, dir); + if(opts.verbose) { fmt::print("readdir(pathname=\"{}\") = [\n{}],\nerrno: {} [{}]\n", - opts.pathname, fmt::join(entries, ",\n"), errno, + opts.pathname, num_elements, errno, ::strerror(errno)); return; } errno = 0; - json out = directory_validate_output{(int) entries.size(), errno}; + json out = directory_validate_output{num_elements, errno}; fmt::print("{}\n", out.dump(2)); return; @@ -135,17 +200,17 @@ directory_validate_init(CLI::App& app) { auto opts = std::make_shared(); auto* cmd = app.add_subcommand( "directory_validate", - "Create a file and execute a direntry system call and count the number of elements"); + "Create count files in the directory and execute a direntry system call and returns the number of elements"); // Add options to cmd, binding them to opts cmd->add_flag("-v,--verbose", opts->verbose, "Produce human writeable output"); - cmd->add_option("pathname", opts->pathname, "file name, entries will be checked in last directory") + cmd->add_option("pathname", opts->pathname, "directory to check or filename to create (if count is 0), elements will be checked in the parent dir") ->required() ->type_name(""); - cmd->add_option("count", opts->count, "Number of files to check") + cmd->add_option("count", opts->count, "Number of files to create. If 0, it creates only the entry in the pathname.") ->required() ->type_name(""); -- GitLab