diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b3a084766ed05440474c7f33c55816a03136ad90..f29f079e9bbb7081e3e16505ec179e41b3b3e924 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -46,6 +46,15 @@ gkfs_add_python_test( SOURCE shell/ ) + +gkfs_add_python_test( + NAME test_data + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE data/ +) + + if(GKFS_INSTALL_TESTS) install(DIRECTORY harness DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration @@ -89,6 +98,14 @@ if(GKFS_INSTALL_TESTS) ) + install(DIRECTORY data + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING diff --git a/tests/integration/data/README.md b/tests/integration/data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8bde8d433aea166a34b828be9663c81f899d8b8b --- /dev/null +++ b/tests/integration/data/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any data-related +functionalities in GekkoFS. diff --git a/tests/integration/data/test_data_integrity.py b/tests/integration/data/test_data_integrity.py new file mode 100644 index 0000000000000000000000000000000000000000..67b88611c75bd414e64d99e4028a5e1b78204e76 --- /dev/null +++ b/tests/integration/data/test_data_integrity.py @@ -0,0 +1,105 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +import string +import random +from harness.logger import logger + +nonexisting = "nonexisting" +chunksize_start = 128192 +chunksize_end = 2097153 +step = 4096*9 + +def generate_random_data(size): + return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)]) + + + + +#@pytest.mark.xfail(reason="invalid errno returned on success") +def test_data_integrity(gkfs_daemon, gkfs_client): + """Test several data write-read commands and check that the data is correct""" + topdir = gkfs_daemon.mountdir / "top" + file_a = topdir / "file_a" + + # create topdir + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + + # test statx on existing dir + ret = gkfs_client.statx(0, topdir, 0, 0) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + assert stat.S_ISDIR(ret.statbuf.stx_mode) + + ret = gkfs_client.open(file_a, + os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval != -1 + + + # test statx on existing file + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0) + assert (ret.statbuf.stx_size == 0) + + + # Step 1 - small sizes + + # Generate writes + # Read data + # Compare buffer + + + for i in range (1, 512, 64): + buf = bytes(generate_random_data(i), sys.stdout.encoding) + + ret = gkfs_client.write(file_a, buf, i) + + assert ret.retval == i + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (ret.statbuf.stx_size == i) + + ret = gkfs_client.read(file_a, i) + assert ret.retval== i + assert ret.buf == buf + + + # Step 2 - Compare bigger sizes exceeding typical chunksize + for i in range (chunksize_start, chunksize_end, step): + ret = gkfs_client.write_validate(file_a, i) + assert ret.retval == 1 + + + return + + diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 7a8090e79cce731df891167a6ee2535cd8c03725..13e6d5d956deaaf81a02def3238661de19dddef4 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(gkfs.io gkfs.io/pwritev.cpp gkfs.io/statx.cpp gkfs.io/lseek.cpp + gkfs.io/write_validate.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 7d111c60943651794822a1ce321394bbe0c85e54..0fc3041d852aee4254e3ea2b2134a888c138a6a0 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -67,4 +67,7 @@ statx_init(CLI::App& app); void lseek_init(CLI::App& app); +void +write_validate_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index e946d1b116e9185f3eeed82c3ea7e25d7a5caebe..426df6a7e12e7dd801c42c8015ff9ad0af6b5406 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -38,6 +38,7 @@ init_commands(CLI::App& app) { statx_init(app); #endif lseek_init(app); + write_validate_init(app); } diff --git a/tests/integration/harness/gkfs.io/write_validate.cpp b/tests/integration/harness/gkfs.io/write_validate.cpp new file mode 100644 index 0000000000000000000000000000000000000000..54e9d10bab49ff14515bd8921e605da4f51e722f --- /dev/null +++ b/tests/integration/harness/gkfs.io/write_validate.cpp @@ -0,0 +1,169 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct write_validate_options { + bool verbose; + std::string pathname; + ::size_t count; + + REFL_DECL_STRUCT(write_validate_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count) + ); +}; + +struct write_validate_output { + int retval; + int errnum; + + REFL_DECL_STRUCT(write_validate_output, + REFL_DECL_MEMBER(int, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const write_validate_output& out) { + record = serialize(out); +} + +void +write_validate_exec(const write_validate_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, fd, errno, ::strerror(errno)); + return; + } + + json out = write_validate_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + + std::string data = ""; + for (::size_t i = 0 ; i < opts.count; i++) + { + data += char((i%10)+'0'); + } + + io::buffer buf(data); + + auto rv = ::write(fd, buf.data(), opts.count); + + if(opts.verbose) { + fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, rv, errno, ::strerror(errno)); + return; + } + + if (rv < 0 or ::size_t(rv) != opts.count) { + json out = write_validate_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + + + io::buffer bufread(opts.count); + + size_t total = 0; + do{ + rv = ::read(fd, bufread.data(), opts.count-total); + total += rv; + } while (rv > 0 and total < opts.count); + + if (rv < 0 and total != opts.count) { + json out = write_validate_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + + if ( memcmp(buf.data(),bufread.data(),opts.count) ) { + rv = 1; + errno = 0; + json out = write_validate_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + else { + rv = 2; + errno = EINVAL; + json out = write_validate_output{(int)-1, errno}; + fmt::print("{}\n", out.dump(2)); + } + +} + +void +write_validate_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "write_validate", + "Execute the write()-read() system call and compare the content of the buffer"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to test" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + write_validate_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index f50d3d9f45ec44a3b8f40923154e773e35410ded..d5acf60bf5a79314b05551113cc3bf2a12e1f1ca 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -293,6 +293,17 @@ class LseekOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('LseekReturn', ['retval', 'errno'])(**data) + +class WriteValidateOutputSchema(Schema): + """Schema to deserialize the results of a write() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data) + class IOParser: OutputSchemas = { @@ -312,6 +323,7 @@ class IOParser: 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), 'lseek' : LseekOutputSchema(), + 'write_validate' : WriteValidateOutputSchema(), } def parse(self, command, output):