diff --git a/tests/integration/data/test_data_integrity.py b/tests/integration/data/test_data_integrity.py
index cb5587ebf5fd04b5c89790f7aa882e50d5cd1cc8..50e07d29a5beaaf106b4064fcced609c82da6ca5 100644
--- a/tests/integration/data/test_data_integrity.py
+++ b/tests/integration/data/test_data_integrity.py
@@ -1,6 +1,6 @@
################################################################################
-# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain #
-# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany #
+# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain #
+# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany #
# #
# This software was partially supported by the #
# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
@@ -32,23 +32,31 @@ import errno
import stat
import os
import ctypes
-import sh
import sys
import pytest
import string
import random
from harness.logger import logger
-nonexisting = "nonexisting"
-chunksize_start = 128192
-chunksize_end = 2097153
-step = 4096*9
def generate_random_data(size):
return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)])
+def check_write(gkfs_client, i, file):
+ buf = bytes(generate_random_data(i), sys.stdout.encoding)
+
+ ret = gkfs_client.write(file, buf, i)
+ assert ret.retval == i
+ ret = gkfs_client.stat(file)
+
+ assert ret.retval == 0
+ assert (ret.statbuf.st_size == i)
+
+ ret = gkfs_client.read(file, i)
+ assert ret.retval== i
+ assert ret.buf == buf
#@pytest.mark.xfail(reason="invalid errno returned on success")
def test_data_integrity(gkfs_daemon, gkfs_client):
@@ -89,29 +97,36 @@ def test_data_integrity(gkfs_daemon, gkfs_client):
# Read data
# Compare buffer
-
- for i in range (1, 512, 64):
- buf = bytes(generate_random_data(i), sys.stdout.encoding)
-
- ret = gkfs_client.write(file_a, buf, i)
+ ret = gkfs_client.write_validate(file_a, 1)
+ assert ret.retval == 1
- assert ret.retval == i
- ret = gkfs_client.stat(file_a)
+ ret = gkfs_client.write_validate(file_a, 256)
+ assert ret.retval == 1
- assert ret.retval == 0
- assert (ret.statbuf.st_size == i)
+ ret = gkfs_client.write_validate(file_a, 512)
+ assert ret.retval == 1
- ret = gkfs_client.read(file_a, i)
- assert ret.retval== i
- assert ret.buf == buf
+ # Step 2 - Compare bigger sizes exceeding typical chunksize and not aligned
+ ret = gkfs_client.write_validate(file_a, 128192)
+ assert ret.retval == 1
+ # < 1 chunk
+ ret = gkfs_client.write_validate(file_a, 400000)
+ assert ret.retval == 1
- # Step 2 - Compare bigger sizes exceeding typical chunksize
- for i in range (chunksize_start, chunksize_end, step):
- ret = gkfs_client.write_validate(file_a, i)
- assert ret.retval == 1
+ # > 1 chunk < 2 chunks
+ ret = gkfs_client.write_validate(file_a, 600000)
+ assert ret.retval == 1
+ # > 1 chunk < 2 chunks
+ ret = gkfs_client.write_validate(file_a, 900000)
+ assert ret.retval == 1
- return
+ # > 2 chunks
+ ret = gkfs_client.write_validate(file_a, 1100000)
+ assert ret.retval == 1
+ # > 4 chunks
+ ret = gkfs_client.write_validate(file_a, 2097153)
+ assert ret.retval == 1
diff --git a/tests/integration/directories/test_directories.py b/tests/integration/directories/test_directories.py
index 90cfcee482de4ea5bc2da082bd9c66e708c56f74..066b36962f9c77b5fdc59365dedce2b1a72e1711 100644
--- a/tests/integration/directories/test_directories.py
+++ b/tests/integration/directories/test_directories.py
@@ -32,7 +32,6 @@ import errno
import stat
import os
import ctypes
-import sh
import sys
import pytest
from harness.logger import logger
@@ -198,9 +197,8 @@ def test_finedir(gkfs_daemon, gkfs_client):
"""Tests several corner cases for directories scan"""
topdir = gkfs_daemon.mountdir / "finetop"
- longer = Path(topdir.parent, topdir.name + "_fine")
file_a = topdir / "file_"
- file_b = topdir / "dir_b"
+
# create topdir
ret = gkfs_client.mkdir(
@@ -216,34 +214,19 @@ def test_finedir(gkfs_daemon, gkfs_client):
# populate top directory
- for files in range (1,11):
- ret = gkfs_client.open(
- str(file_a) + str(files),
- os.O_CREAT,
- stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
-
- assert ret.retval != -1
-
- ret = gkfs_client.readdir(topdir)
+ for files in range (1,4):
+ ret = gkfs_client.directory_validate(
+ topdir, 1)
+ assert ret.retval == files
- assert len(ret.dirents) == files
- for files in range(11, 20):
- ret = gkfs_client.open(
- str(file_a) + str(files),
- os.O_CREAT,
- stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
-
- assert ret.retval != -1
-
- ret = gkfs_client.readdir(topdir)
-
- assert len(ret.dirents) == 19
+ ret = gkfs_client.directory_validate(
+ topdir, 1000)
+ assert ret.retval == 1000+3
def test_extended(gkfs_daemon, gkfs_shell, gkfs_client):
topdir = gkfs_daemon.mountdir / "test_extended"
- longer = Path(topdir.parent, topdir.name + "_plus")
dir_a = topdir / "dir_a"
dir_b = topdir / "dir_b"
file_a = topdir / "file_a"
@@ -307,11 +290,3 @@ def test_opendir(gkfs_daemon, gkfs_client, directory_path):
assert ret.dirp is None
assert ret.errno == errno.ENOENT
-# def test_stat(gkfs_daemon):
-# pass
-#
-# def test_rmdir(gkfs_daemon):
-# pass
-#
-# def test_closedir(gkfs_daemon):
-# pass
diff --git a/tests/integration/harness/CMakeLists.txt b/tests/integration/harness/CMakeLists.txt
index 762580c8684f1644e8a5b7e73aa62c082692eb27..a2aea938e94d55245b97656c8460d92fe82eadd2 100644
--- a/tests/integration/harness/CMakeLists.txt
+++ b/tests/integration/harness/CMakeLists.txt
@@ -61,6 +61,7 @@ add_executable(gkfs.io
gkfs.io/chdir.cpp
gkfs.io/getcwd_validate.cpp
gkfs.io/symlink.cpp
+ gkfs.io/directory_validate.cpp
)
include(FetchContent)
diff --git a/tests/integration/harness/gkfs.io/commands.hpp b/tests/integration/harness/gkfs.io/commands.hpp
index 39b74e07fa8b5bf69c89eeb1a6351378b561d413..1425e7d9fea22ac5efafc88e359620daf20fccbf 100644
--- a/tests/integration/harness/gkfs.io/commands.hpp
+++ b/tests/integration/harness/gkfs.io/commands.hpp
@@ -87,6 +87,9 @@ lseek_init(CLI::App& app);
void
write_validate_init(CLI::App& app);
+void
+directory_validate_init(CLI::App& app);
+
void
write_random_init(CLI::App& app);
diff --git a/tests/integration/harness/gkfs.io/directory_validate.cpp b/tests/integration/harness/gkfs.io/directory_validate.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2d03c11b4c2fffd8fc3cfc2db9fde39827d10ec2
--- /dev/null
+++ b/tests/integration/harness/gkfs.io/directory_validate.cpp
@@ -0,0 +1,218 @@
+/*
+ Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain
+ Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany
+
+ This software was partially supported by the
+ EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).
+
+ This software was partially supported by the
+ ADA-FS project under the SPPEXA project funded by the DFG.
+
+ This file is part of GekkoFS.
+
+ GekkoFS is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ GekkoFS is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with GekkoFS. If not, see .
+
+ SPDX-License-Identifier: GPL-3.0-or-later
+*/
+
+/* C++ includes */
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+/* C includes */
+#include
+#include
+#include
+#include
+#include
+
+using json = nlohmann::json;
+
+// Creates a file in the path, and does a readdir comparing the count size (may be accumulated with previous operations)
+struct directory_validate_options {
+ bool verbose{};
+ std::string pathname;
+ ::size_t count;
+
+ REFL_DECL_STRUCT(directory_validate_options, REFL_DECL_MEMBER(bool, verbose),
+ REFL_DECL_MEMBER(std::string, pathname),
+ REFL_DECL_MEMBER(::size_t, count));
+};
+
+struct directory_validate_output {
+ int retval;
+ int errnum;
+
+ REFL_DECL_STRUCT(directory_validate_output, REFL_DECL_MEMBER(int, retval),
+ REFL_DECL_MEMBER(int, errnum));
+};
+
+void
+to_json(json& record, const directory_validate_output& out) {
+ record = serialize(out);
+}
+
+
+/**
+ * returns the number of elements existing in the path
+ * @param opts
+ * @param dir Path checked
+ * @returns The number of elements in the directory
+ */
+int
+number_of_elements(const directory_validate_options& opts, const std::string dir) {
+ int num_elements = 0;
+
+ ::DIR* dirp = ::opendir(dir.c_str());
+
+ if(dirp == NULL) {
+ if(opts.verbose) {
+ fmt::print("readdir(pathname=\"{}\") = {}, errno: {} [{}]\n",
+ dir, "NULL", errno, ::strerror(errno));
+ return -3;
+ }
+
+ std::cout << "Error create directory" << std::endl;
+ json out = directory_validate_output{-3, errno};
+ fmt::print("{}\n", out.dump(2));
+
+ return -3;
+ }
+
+ struct ::dirent* entry;
+
+ while((entry = ::readdir(dirp)) != NULL) {
+ num_elements++;
+ }
+
+ return num_elements;
+}
+
+
+/**
+ * Creates `count` files, with a suffix starting at the number of elements existing in the path
+ * @param opts
+ * @param path Path where the file is created
+ * @param count Number of files to create
+ * @returns The number of elements created plus the number of elements already in the directory (calculated, not checked)
+ */
+int
+create_n_files (const directory_validate_options& opts, const std::string path, int count) {
+
+ // Read Directory and get number of entries
+ int num_elements = number_of_elements(opts, path);
+
+ for (int i = num_elements; i < count+num_elements; i++) {
+ std::string filename = path+"/file_auto_"+std::to_string(i);
+ int fd = ::creat(filename.c_str(), S_IRWXU);
+ if(fd == -1) {
+ if(opts.verbose) {
+ fmt::print(
+ "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n",
+ filename, i, fd, errno, ::strerror(errno));
+ return -2;
+ }
+ json out = directory_validate_output{-2, errno};
+ fmt::print("{}\n", out.dump(2));
+
+ return -2;
+ }
+
+ ::close(fd);
+ }
+ return num_elements+count;
+}
+
+/**
+ * Creates `count` files, and returns the number of elements in the path
+ * If count == 0, the path is the filename to be created. Parent directory is checked
+ * @param opts
+ */
+void
+directory_validate_exec(const directory_validate_options& opts) {
+
+ if (opts.count == 0) {
+ int fd = ::creat(opts.pathname.c_str(), S_IRWXU);
+
+ if(fd == -1) {
+ if(opts.verbose) {
+ fmt::print(
+ "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n",
+ opts.pathname, opts.count, fd, errno, ::strerror(errno));
+ return;
+ }
+ json out = directory_validate_output{-2, errno};
+ fmt::print("{}\n", out.dump(2));
+
+ return;
+ }
+
+ ::close(fd);
+ }
+ else {
+ int created = create_n_files(opts, opts.pathname, opts.count);
+ if (created <= 0) return;
+ }
+
+ // Do a readdir
+ std::string dir = opts.pathname;
+
+ if (opts.count == 0)
+ dir = ::dirname((char*)opts.pathname.c_str());
+
+ auto num_elements = number_of_elements(opts, dir);
+
+ if(opts.verbose) {
+ fmt::print("readdir(pathname=\"{}\") = [\n{}],\nerrno: {} [{}]\n",
+ opts.pathname, num_elements, errno,
+ ::strerror(errno));
+ return;
+ }
+
+ errno = 0;
+ json out = directory_validate_output{num_elements, errno};
+ fmt::print("{}\n", out.dump(2));
+ return;
+
+}
+
+void
+directory_validate_init(CLI::App& app) {
+
+ // Create the option and subcommand objects
+ auto opts = std::make_shared();
+ auto* cmd = app.add_subcommand(
+ "directory_validate",
+ "Create count files in the directory and execute a direntry system call and returns the number of elements");
+
+ // Add options to cmd, binding them to opts
+ cmd->add_flag("-v,--verbose", opts->verbose,
+ "Produce human writeable output");
+
+ cmd->add_option("pathname", opts->pathname, "directory to check or filename to create (if count is 0), elements will be checked in the parent dir")
+ ->required()
+ ->type_name("");
+
+ cmd->add_option("count", opts->count, "Number of files to create. If 0, it creates only the entry in the pathname.")
+ ->required()
+ ->type_name("");
+
+ cmd->callback([opts]() { directory_validate_exec(*opts); });
+}
diff --git a/tests/integration/harness/gkfs.io/main.cpp b/tests/integration/harness/gkfs.io/main.cpp
index ee79fc70bc43b59d7a97b9c6c8289e2d1c8bd794..2233c0d81ecf43e1bdaace9432957b354d3b96a4 100644
--- a/tests/integration/harness/gkfs.io/main.cpp
+++ b/tests/integration/harness/gkfs.io/main.cpp
@@ -54,6 +54,7 @@ init_commands(CLI::App& app) {
#endif
lseek_init(app);
write_validate_init(app);
+ directory_validate_init(app);
write_random_init(app);
truncate_init(app);
// utils
diff --git a/tests/integration/harness/io.py b/tests/integration/harness/io.py
index 37d2a8c13d2bbd85eee3f1c9303a3779d40c364e..6522cdad9394601fd5bd3ebbcc95dbbe45a14700 100644
--- a/tests/integration/harness/io.py
+++ b/tests/integration/harness/io.py
@@ -319,6 +319,15 @@ class WriteValidateOutputSchema(Schema):
def make_object(self, data, **kwargs):
return namedtuple('WriteValidateReturn', ['retval', 'errno'])(**data)
+class DirectoryValidateOutputSchema(Schema):
+ """Schema to deserialize the results of a write() execution"""
+
+ retval = fields.Integer(required=True)
+ errno = Errno(data_key='errnum', required=True)
+
+ @post_load
+ def make_object(self, data, **kwargs):
+ return namedtuple('DirectoryValidateReturn', ['retval', 'errno'])(**data)
class WriteRandomOutputSchema(Schema):
"""Schema to deserialize the results of a write() execution"""
@@ -409,6 +418,7 @@ class IOParser:
'write_random': WriteRandomOutputSchema(),
'write_validate' : WriteValidateOutputSchema(),
'truncate': TruncateOutputSchema(),
+ 'directory_validate' : DirectoryValidateOutputSchema(),
# UTIL
'file_compare': FileCompareOutputSchema(),
'chdir' : ChdirOutputSchema(),