From 8895c7025bf7c3101f6a62a445c5a3781d950323 Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Fri, 15 May 2020 17:24:56 +0200 Subject: [PATCH 1/4] [IO Test] Added. Missing write-read compare for larger sizes --- tests/integration/CMakeLists.txt | 17 ++++++ tests/integration/data/README.md | 4 ++ tests/integration/data/test_io.py | 99 +++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 tests/integration/data/README.md create mode 100644 tests/integration/data/test_io.py diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index b3a084766..f29f079e9 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -46,6 +46,15 @@ gkfs_add_python_test( SOURCE shell/ ) + +gkfs_add_python_test( + NAME test_data + PYTHON_VERSION 3.6 + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration + SOURCE data/ +) + + if(GKFS_INSTALL_TESTS) install(DIRECTORY harness DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration @@ -89,6 +98,14 @@ if(GKFS_INSTALL_TESTS) ) + install(DIRECTORY data + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration + FILES_MATCHING + REGEX ".*\\.py" + PATTERN "__pycache__" EXCLUDE + PATTERN ".pytest_cache" EXCLUDE + ) + install(DIRECTORY shell DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration FILES_MATCHING diff --git a/tests/integration/data/README.md b/tests/integration/data/README.md new file mode 100644 index 000000000..8bde8d433 --- /dev/null +++ b/tests/integration/data/README.md @@ -0,0 +1,4 @@ +# README + +This directory contains functional tests for any data-related +functionalities in GekkoFS. diff --git a/tests/integration/data/test_io.py b/tests/integration/data/test_io.py new file mode 100644 index 000000000..423bed20f --- /dev/null +++ b/tests/integration/data/test_io.py @@ -0,0 +1,99 @@ +################################################################################ +# Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain # +# Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany # +# # +# This software was partially supported by the # +# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). # +# # +# This software was partially supported by the # +# ADA-FS project under the SPPEXA project funded by the DFG. # +# # +# SPDX-License-Identifier: MIT # +################################################################################ + +import harness +from pathlib import Path +import errno +import stat +import os +import ctypes +import sh +import sys +import pytest +from harness.logger import logger + +nonexisting = "nonexisting" + + + + +#@pytest.mark.xfail(reason="invalid errno returned on success") +def test_io(gkfs_daemon, gkfs_client): + """Test several statx commands""" + topdir = gkfs_daemon.mountdir / "top" + file_a = topdir / "file_a" + + # create topdir + ret = gkfs_client.mkdir( + topdir, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + + # test statx on existing dir + ret = gkfs_client.statx(0, topdir, 0, 0) + + assert ret.retval == 0 + assert ret.errno == 115 #FIXME: Should be 0! + assert stat.S_ISDIR(ret.statbuf.stx_mode) + + ret = gkfs_client.open(file_a, + os.O_CREAT, + stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + + assert ret.retval != -1 + + + # test statx on existing file + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0) + assert (ret.statbuf.stx_size == 0) + + + # Phase 1 + # from 1 to n data + # Generate writes + # Read data + # Compare buffer + # delete data + + # from 1 to 2M +1 + + for i in range (1, 512, 64): + buf = b'' + print (i) + for k in range (0,i): + value = str(k%10) + buf += bytes(value, sys.stdout.encoding) + + ret = gkfs_client.write(file_a, buf, i) + + assert ret.retval == i + ret = gkfs_client.statx(0, file_a, 0, 0) + + assert ret.retval == 0 + assert (ret.statbuf.stx_size == i) + + ret = gkfs_client.read(file_a, i) + assert ret.retval== i + assert ret.buf == buf + + + + + return + + -- GitLab From e7fe0fdaf540d80e4f3e45b018d5006a627b1bfd Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 18 May 2020 15:35:36 +0200 Subject: [PATCH 2/4] Added write-read with data comparison --- tests/integration/data/test_io.py | 4 +- tests/integration/harness/CMakeLists.txt | 1 + .../integration/harness/gkfs.io/commands.hpp | 3 + tests/integration/harness/gkfs.io/main.cpp | 1 + .../harness/gkfs.io/write_read.cpp | 169 ++++++++++++++++++ tests/integration/harness/io.py | 12 ++ 6 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 tests/integration/harness/gkfs.io/write_read.cpp diff --git a/tests/integration/data/test_io.py b/tests/integration/data/test_io.py index 423bed20f..340ea0304 100644 --- a/tests/integration/data/test_io.py +++ b/tests/integration/data/test_io.py @@ -74,7 +74,6 @@ def test_io(gkfs_daemon, gkfs_client): for i in range (1, 512, 64): buf = b'' - print (i) for k in range (0,i): value = str(k%10) buf += bytes(value, sys.stdout.encoding) @@ -92,6 +91,9 @@ def test_io(gkfs_daemon, gkfs_client): assert ret.buf == buf + for i in range (128192, 2097153, 4096*3): + ret = gkfs_client.write_read(file_a, i) + assert ret.retval == 1 return diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 7a8090e79..20f31e59b 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -26,6 +26,7 @@ add_executable(gkfs.io gkfs.io/pwritev.cpp gkfs.io/statx.cpp gkfs.io/lseek.cpp + gkfs.io/write_read.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index 7d111c609..b36216f63 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -67,4 +67,7 @@ statx_init(CLI::App& app); void lseek_init(CLI::App& app); +void +write_read_init(CLI::App& app); + #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index e946d1b11..7aae3d3b3 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -38,6 +38,7 @@ init_commands(CLI::App& app) { statx_init(app); #endif lseek_init(app); + write_read_init(app); } diff --git a/tests/integration/harness/gkfs.io/write_read.cpp b/tests/integration/harness/gkfs.io/write_read.cpp new file mode 100644 index 000000000..070c8e8a2 --- /dev/null +++ b/tests/integration/harness/gkfs.io/write_read.cpp @@ -0,0 +1,169 @@ +/* + Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain + Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany + + This software was partially supported by the + EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). + + This software was partially supported by the + ADA-FS project under the SPPEXA project funded by the DFG. + + SPDX-License-Identifier: MIT +*/ + +/* C++ includes */ +#include +#include +#include +#include +#include +#include +#include +#include + +/* C includes */ +#include +#include +#include +#include + +using json = nlohmann::json; + +struct write_read_options { + bool verbose; + std::string pathname; + ::size_t count; + + REFL_DECL_STRUCT(write_read_options, + REFL_DECL_MEMBER(bool, verbose), + REFL_DECL_MEMBER(std::string, pathname), + REFL_DECL_MEMBER(::size_t, count) + ); +}; + +struct write_read_output { + int retval; + int errnum; + + REFL_DECL_STRUCT(write_read_output, + REFL_DECL_MEMBER(int, retval), + REFL_DECL_MEMBER(int, errnum) + ); +}; + +void +to_json(json& record, + const write_read_output& out) { + record = serialize(out); +} + +void +write_read_exec(const write_read_options& opts) { + + int fd = ::open(opts.pathname.c_str(), O_WRONLY); + + if(fd == -1) { + if(opts.verbose) { + fmt::print("write(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, fd, errno, ::strerror(errno)); + return; + } + + json out = write_read_output{fd, errno}; + fmt::print("{}\n", out.dump(2)); + + return; + } + + + std::string data = ""; + for (::size_t i = 0 ; i < opts.count; i++) + { + data += char((i%10)+'0'); + } + + io::buffer buf(data); + + auto rv = ::write(fd, buf.data(), opts.count); + + if(opts.verbose) { + fmt::print("write_read(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + opts.pathname, opts.count, rv, errno, ::strerror(errno)); + return; + } + + if (rv < 0 or ::size_t(rv) != opts.count) { + json out = write_read_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + + + io::buffer bufread(opts.count); + + size_t total = 0; + do{ + rv = ::read(fd, bufread.data(), opts.count-total); + total += rv; + } while (rv > 0 and total < opts.count); + + if (rv < 0 and total != opts.count) { + json out = write_read_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + + if ( memcmp(buf.data(),bufread.data(),opts.count) ) { + rv = 1; + errno = 0; + json out = write_read_output{(int)rv, errno}; + fmt::print("{}\n", out.dump(2)); + return; + } + else { + rv = 2; + errno = EINVAL; + json out = write_read_output{(int)-1, errno}; + fmt::print("{}\n", out.dump(2)); + } + +} + +void +write_read_init(CLI::App& app) { + + // Create the option and subcommand objects + auto opts = std::make_shared(); + auto* cmd = app.add_subcommand( + "write_read", + "Execute the write()-read() system call and compare the content of the buffer"); + + // Add options to cmd, binding them to opts + cmd->add_flag( + "-v,--verbose", + opts->verbose, + "Produce human writeable output" + ); + + cmd->add_option( + "pathname", + opts->pathname, + "Directory name" + ) + ->required() + ->type_name(""); + + cmd->add_option( + "count", + opts->count, + "Number of bytes to test" + ) + ->required() + ->type_name(""); + + cmd->callback([opts]() { + write_read_exec(*opts); + }); +} + + diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index f50d3d9f4..c0667be52 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -293,6 +293,17 @@ class LseekOutputSchema(Schema): def make_object(self, data, **kwargs): return namedtuple('LseekReturn', ['retval', 'errno'])(**data) + +class WriteReadOutputSchema(Schema): + """Schema to deserialize the results of a write() execution""" + + retval = fields.Integer(required=True) + errno = Errno(data_key='errnum', required=True) + + @post_load + def make_object(self, data, **kwargs): + return namedtuple('WriteReadReturn', ['retval', 'errno'])(**data) + class IOParser: OutputSchemas = { @@ -312,6 +323,7 @@ class IOParser: 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), 'lseek' : LseekOutputSchema(), + 'write_read' : WriteReadOutputSchema(), } def parse(self, command, output): -- GitLab From 8e0da9614a5fa7c2652fa8c465f506002138ae5f Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Mon, 18 May 2020 16:02:50 +0200 Subject: [PATCH 3/4] reduced test stress --- tests/integration/data/test_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/data/test_io.py b/tests/integration/data/test_io.py index 340ea0304..541c1c279 100644 --- a/tests/integration/data/test_io.py +++ b/tests/integration/data/test_io.py @@ -91,7 +91,7 @@ def test_io(gkfs_daemon, gkfs_client): assert ret.buf == buf - for i in range (128192, 2097153, 4096*3): + for i in range (128192, 2097153, 4096*9): ret = gkfs_client.write_read(file_a, i) assert ret.retval == 1 -- GitLab From c70a106d9c973c88de6ea289d17c20c9b830960d Mon Sep 17 00:00:00 2001 From: Ramon Nou Date: Tue, 19 May 2020 14:37:30 +0200 Subject: [PATCH 4/4] Modified write-read to write validate and random string generation --- .../{test_io.py => test_data_integrity.py} | 28 ++++++++------- tests/integration/harness/CMakeLists.txt | 2 +- .../integration/harness/gkfs.io/commands.hpp | 2 +- tests/integration/harness/gkfs.io/main.cpp | 2 +- .../{write_read.cpp => write_validate.cpp} | 34 +++++++++---------- tests/integration/harness/io.py | 6 ++-- 6 files changed, 39 insertions(+), 35 deletions(-) rename tests/integration/data/{test_io.py => test_data_integrity.py} (80%) rename tests/integration/harness/gkfs.io/{write_read.cpp => write_validate.cpp} (78%) diff --git a/tests/integration/data/test_io.py b/tests/integration/data/test_data_integrity.py similarity index 80% rename from tests/integration/data/test_io.py rename to tests/integration/data/test_data_integrity.py index 541c1c279..67b88611c 100644 --- a/tests/integration/data/test_io.py +++ b/tests/integration/data/test_data_integrity.py @@ -20,16 +20,24 @@ import ctypes import sh import sys import pytest +import string +import random from harness.logger import logger nonexisting = "nonexisting" +chunksize_start = 128192 +chunksize_end = 2097153 +step = 4096*9 + +def generate_random_data(size): + return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)]) #@pytest.mark.xfail(reason="invalid errno returned on success") -def test_io(gkfs_daemon, gkfs_client): - """Test several statx commands""" +def test_data_integrity(gkfs_daemon, gkfs_client): + """Test several data write-read commands and check that the data is correct""" topdir = gkfs_daemon.mountdir / "top" file_a = topdir / "file_a" @@ -63,20 +71,15 @@ def test_io(gkfs_daemon, gkfs_client): assert (ret.statbuf.stx_size == 0) - # Phase 1 - # from 1 to n data + # Step 1 - small sizes + # Generate writes # Read data # Compare buffer - # delete data - # from 1 to 2M +1 for i in range (1, 512, 64): - buf = b'' - for k in range (0,i): - value = str(k%10) - buf += bytes(value, sys.stdout.encoding) + buf = bytes(generate_random_data(i), sys.stdout.encoding) ret = gkfs_client.write(file_a, buf, i) @@ -91,8 +94,9 @@ def test_io(gkfs_daemon, gkfs_client): assert ret.buf == buf - for i in range (128192, 2097153, 4096*9): - ret = gkfs_client.write_read(file_a, i) + # Step 2 - Compare bigger sizes exceeding typical chunksize + for i in range (chunksize_start, chunksize_end, step): + ret = gkfs_client.write_validate(file_a, i) assert ret.retval == 1 diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt index 20f31e59b..13e6d5d95 100644 --- a/tests/integration/harness/CMakeLists.txt +++ b/tests/integration/harness/CMakeLists.txt @@ -26,7 +26,7 @@ add_executable(gkfs.io gkfs.io/pwritev.cpp gkfs.io/statx.cpp gkfs.io/lseek.cpp - gkfs.io/write_read.cpp + gkfs.io/write_validate.cpp ) include(FetchContent) diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp index b36216f63..0fc3041d8 100644 --- a/tests/integration/harness/gkfs.io/commands.hpp +++ b/tests/integration/harness/gkfs.io/commands.hpp @@ -68,6 +68,6 @@ void lseek_init(CLI::App& app); void -write_read_init(CLI::App& app); +write_validate_init(CLI::App& app); #endif // IO_COMMANDS_HPP diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp index 7aae3d3b3..426df6a7e 100644 --- a/tests/integration/harness/gkfs.io/main.cpp +++ b/tests/integration/harness/gkfs.io/main.cpp @@ -38,7 +38,7 @@ init_commands(CLI::App& app) { statx_init(app); #endif lseek_init(app); - write_read_init(app); + write_validate_init(app); } diff --git a/tests/integration/harness/gkfs.io/write_read.cpp b/tests/integration/harness/gkfs.io/write_validate.cpp similarity index 78% rename from tests/integration/harness/gkfs.io/write_read.cpp rename to tests/integration/harness/gkfs.io/write_validate.cpp index 070c8e8a2..54e9d10ba 100644 --- a/tests/integration/harness/gkfs.io/write_read.cpp +++ b/tests/integration/harness/gkfs.io/write_validate.cpp @@ -29,23 +29,23 @@ using json = nlohmann::json; -struct write_read_options { +struct write_validate_options { bool verbose; std::string pathname; ::size_t count; - REFL_DECL_STRUCT(write_read_options, + REFL_DECL_STRUCT(write_validate_options, REFL_DECL_MEMBER(bool, verbose), REFL_DECL_MEMBER(std::string, pathname), REFL_DECL_MEMBER(::size_t, count) ); }; -struct write_read_output { +struct write_validate_output { int retval; int errnum; - REFL_DECL_STRUCT(write_read_output, + REFL_DECL_STRUCT(write_validate_output, REFL_DECL_MEMBER(int, retval), REFL_DECL_MEMBER(int, errnum) ); @@ -53,23 +53,23 @@ struct write_read_output { void to_json(json& record, - const write_read_output& out) { + const write_validate_output& out) { record = serialize(out); } void -write_read_exec(const write_read_options& opts) { +write_validate_exec(const write_validate_options& opts) { int fd = ::open(opts.pathname.c_str(), O_WRONLY); if(fd == -1) { if(opts.verbose) { - fmt::print("write(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", opts.pathname, opts.count, fd, errno, ::strerror(errno)); return; } - json out = write_read_output{fd, errno}; + json out = write_validate_output{fd, errno}; fmt::print("{}\n", out.dump(2)); return; @@ -87,13 +87,13 @@ write_read_exec(const write_read_options& opts) { auto rv = ::write(fd, buf.data(), opts.count); if(opts.verbose) { - fmt::print("write_read(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", + fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", opts.pathname, opts.count, rv, errno, ::strerror(errno)); return; } if (rv < 0 or ::size_t(rv) != opts.count) { - json out = write_read_output{(int)rv, errno}; + json out = write_validate_output{(int)rv, errno}; fmt::print("{}\n", out.dump(2)); return; } @@ -108,7 +108,7 @@ write_read_exec(const write_read_options& opts) { } while (rv > 0 and total < opts.count); if (rv < 0 and total != opts.count) { - json out = write_read_output{(int)rv, errno}; + json out = write_validate_output{(int)rv, errno}; fmt::print("{}\n", out.dump(2)); return; } @@ -116,26 +116,26 @@ write_read_exec(const write_read_options& opts) { if ( memcmp(buf.data(),bufread.data(),opts.count) ) { rv = 1; errno = 0; - json out = write_read_output{(int)rv, errno}; + json out = write_validate_output{(int)rv, errno}; fmt::print("{}\n", out.dump(2)); return; } else { rv = 2; errno = EINVAL; - json out = write_read_output{(int)-1, errno}; + json out = write_validate_output{(int)-1, errno}; fmt::print("{}\n", out.dump(2)); } } void -write_read_init(CLI::App& app) { +write_validate_init(CLI::App& app) { // Create the option and subcommand objects - auto opts = std::make_shared(); + auto opts = std::make_shared(); auto* cmd = app.add_subcommand( - "write_read", + "write_validate", "Execute the write()-read() system call and compare the content of the buffer"); // Add options to cmd, binding them to opts @@ -162,7 +162,7 @@ write_read_init(CLI::App& app) { ->type_name(""); cmd->callback([opts]() { - write_read_exec(*opts); + write_validate_exec(*opts); }); } diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py index c0667be52..d5acf60bf 100644 --- a/tests/integration/harness/io.py +++ b/tests/integration/harness/io.py @@ -294,7 +294,7 @@ class LseekOutputSchema(Schema): return namedtuple('LseekReturn', ['retval', 'errno'])(**data) -class WriteReadOutputSchema(Schema): +class WriteValidateOutputSchema(Schema): """Schema to deserialize the results of a write() execution""" retval = fields.Integer(required=True) @@ -302,7 +302,7 @@ class WriteReadOutputSchema(Schema): @post_load def make_object(self, data, **kwargs): - return namedtuple('WriteReadReturn', ['retval', 'errno'])(**data) + return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data) class IOParser: @@ -323,7 +323,7 @@ class IOParser: 'stat' : StatOutputSchema(), 'statx' : StatxOutputSchema(), 'lseek' : LseekOutputSchema(), - 'write_read' : WriteReadOutputSchema(), + 'write_validate' : WriteValidateOutputSchema(), } def parse(self, command, output): -- GitLab