Commit 667800a8 authored by Ramon Nou's avatar Ramon Nou
Browse files

Reduce test_integration and directory tests

parent ea34c661

tests/integration/data/test_data_integrity.py

+39 −24
Original line number Diff line number Diff line 
################################################################################

# Copyright 2018-2021, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2021, Johannes Gutenberg Universitaet Mainz, Germany          #

# Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain            #

# Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany          #

#                                                                              #

# This software was partially supported by the                                 #

# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).    #
@@ -32,23 +32,31 @@ import errno 
import stat

import os

import ctypes

import sh

import sys

import pytest

import string

import random

from harness.logger import logger



nonexisting = "nonexisting"

chunksize_start = 128192

chunksize_end = 2097153

step = 4096*9



def generate_random_data(size):

    return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)])





def check_write(gkfs_client, i, file):

        buf = bytes(generate_random_data(i), sys.stdout.encoding)

        

        ret = gkfs_client.write(file, buf, i)



        assert ret.retval == i

        ret = gkfs_client.stat(file)



        assert ret.retval == 0

        assert (ret.statbuf.st_size == i)



        ret = gkfs_client.read(file, i)

        assert ret.retval== i

        assert ret.buf == buf



#@pytest.mark.xfail(reason="invalid errno returned on success")

def test_data_integrity(gkfs_daemon, gkfs_client):
@@ -89,29 +97,36 @@ def test_data_integrity(gkfs_daemon, gkfs_client): 
    # Read data

    # Compare buffer



    ret = gkfs_client.write_validate(file_a, 1)

    assert ret.retval == 1    



    for i in range (1, 512, 64):

        buf = bytes(generate_random_data(i), sys.stdout.encoding)

        

        ret = gkfs_client.write(file_a, buf, i)



        assert ret.retval == i

        ret = gkfs_client.stat(file_a)

    ret = gkfs_client.write_validate(file_a, 256)

    assert ret.retval == 1  



        assert ret.retval == 0

        assert (ret.statbuf.st_size == i)

    ret = gkfs_client.write_validate(file_a, 512)

    assert ret.retval == 1  



        ret = gkfs_client.read(file_a, i)

        assert ret.retval== i

        assert ret.buf == buf

    # Step 2 - Compare bigger sizes exceeding typical chunksize and not aligned

    ret = gkfs_client.write_validate(file_a, 128192)

    assert ret.retval == 1



    # < 1 chunk   

    ret = gkfs_client.write_validate(file_a, 400000)

    assert ret.retval == 1



    # Step 2 - Compare bigger sizes exceeding typical chunksize

    for i in range (chunksize_start, chunksize_end, step):

        ret = gkfs_client.write_validate(file_a, i)

    # > 1 chunk < 2 chunks

    ret = gkfs_client.write_validate(file_a, 600000)

    assert ret.retval == 1



    # > 1 chunk < 2 chunks

    ret = gkfs_client.write_validate(file_a, 900000)

    assert ret.retval == 1



    return

    # > 2 chunks

    ret = gkfs_client.write_validate(file_a, 1100000) 

    assert ret.retval == 1



    # > 4 chunks

    ret = gkfs_client.write_validate(file_a, 2097153) 

    assert ret.retval == 1

tests/integration/directories/test_directories.py

+6 −25
Original line number Diff line number Diff line
@@ -32,7 +32,6 @@ import errno 
import stat

import os

import ctypes

import sh

import sys

import pytest

from harness.logger import logger
@@ -198,9 +197,8 @@ def test_finedir(gkfs_daemon, gkfs_client): 
    """Tests several corner cases for directories scan"""



    topdir = gkfs_daemon.mountdir / "finetop"

    longer = Path(topdir.parent, topdir.name + "_fine")

    file_a  = topdir / "file_"

    file_b  = topdir / "dir_b"

    

    

    # create topdir

    ret = gkfs_client.mkdir(
@@ -216,29 +214,12 @@ def test_finedir(gkfs_daemon, gkfs_client): 


    # populate top directory



    for files in range (1,11):

        ret = gkfs_client.open(

                str(file_a) + str(files),

                os.O_CREAT, 

                stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)

        

        assert ret.retval != -1



        ret = gkfs_client.readdir(topdir)



        assert len(ret.dirents) == files



    for files in range(11, 20):

        ret = gkfs_client.open(

                str(file_a) + str(files),

                os.O_CREAT,

                stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)

        

        assert ret.retval != -1

    

    ret = gkfs_client.readdir(topdir)

    for files in range (1,4):

        file_name = str(file_a) + str(files)

        ret = gkfs_client.directory_validate(

                Path(file_name), files) 

        assert ret.retval == files



    assert len(ret.dirents) == 19





def test_extended(gkfs_daemon, gkfs_shell, gkfs_client):

tests/integration/harness/CMakeLists.txt

+1 −0
Original line number Diff line number Diff line
@@ -61,6 +61,7 @@ add_executable(gkfs.io 
    gkfs.io/chdir.cpp

    gkfs.io/getcwd_validate.cpp

    gkfs.io/symlink.cpp

    gkfs.io/directory_validate.cpp

)



include(FetchContent)

tests/integration/harness/gkfs.io/commands.hpp

+3 −0
Original line number Diff line number Diff line
@@ -87,6 +87,9 @@ lseek_init(CLI::App& app); 
void

write_validate_init(CLI::App& app);



void

directory_validate_init(CLI::App& app);



void

write_random_init(CLI::App& app);

tests/integration/harness/gkfs.io/directory_validate.cpp

0 → 100644
+153 −0
Original line number Diff line number Diff line 
/*

  Copyright 2018-2022, Barcelona Supercomputing Center (BSC), Spain

  Copyright 2015-2022, Johannes Gutenberg Universitaet Mainz, Germany



  This software was partially supported by the

  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).



  This software was partially supported by the

  ADA-FS project under the SPPEXA project funded by the DFG.



  This file is part of GekkoFS.



  GekkoFS is free software: you can redistribute it and/or modify

  it under the terms of the GNU General Public License as published by

  the Free Software Foundation, either version 3 of the License, or

  (at your option) any later version.



  GekkoFS is distributed in the hope that it will be useful,

  but WITHOUT ANY WARRANTY; without even the implied warranty of

  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  GNU General Public License for more details.



  You should have received a copy of the GNU General Public License

  along with GekkoFS.  If not, see <https://www.gnu.org/licenses/>.



  SPDX-License-Identifier: GPL-3.0-or-later

*/



/* C++ includes */

#include <CLI11/CLI11.hpp>

#include <nlohmann/json.hpp>

#include <memory>

#include <fmt/format.h>

#include <commands.hpp>

#include <reflection.hpp>

#include <serialize.hpp>

#include <binary_buffer.hpp>



/* C includes */

#include <sys/types.h>

#include <sys/stat.h>

#include <fcntl.h>

#include <unistd.h>

#include <libgen.h>



using json = nlohmann::json;



// Creates a file in the path, and does a readdir comparing the count size (may be accumulated with previous operations)

struct directory_validate_options {

    bool verbose{};

    std::string pathname;

    ::size_t count;



    REFL_DECL_STRUCT(directory_validate_options, REFL_DECL_MEMBER(bool, verbose),

                     REFL_DECL_MEMBER(std::string, pathname),

                     REFL_DECL_MEMBER(::size_t, count));

};



struct directory_validate_output {

    int retval;

    int errnum;



    REFL_DECL_STRUCT(directory_validate_output, REFL_DECL_MEMBER(int, retval),

                     REFL_DECL_MEMBER(int, errnum));

};



void

to_json(json& record, const directory_validate_output& out) {

    record = serialize(out);

}



void

directory_validate_exec(const directory_validate_options& opts) {



    int fd = ::creat(opts.pathname.c_str(), S_IRWXU);

   

    if(fd == -1) {

        if(1) {

            fmt::print(

                    "directory_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n",

                    opts.pathname, opts.count, fd, errno, ::strerror(errno));

            return;

        }

        json out = directory_validate_output{-2, errno};

        fmt::print("{}\n", out.dump(2));



        return;

    }



    ::close(fd);



    // Do a readdir

    std::string dir = ::dirname((char*)opts.pathname.c_str());

    ::DIR* dirp = ::opendir(dir.c_str());



    if(dirp == NULL) {

        if(opts.verbose) {

            fmt::print("readdir(pathname=\"{}\") = {}, errno: {} [{}]\n",

                       opts.pathname, "NULL", errno, ::strerror(errno));

            return;

        }



    std::cout << "Error create directory" << std::endl;

        json out = directory_validate_output{-3, errno};

        fmt::print("{}\n", out.dump(2));



        return;

    }



    std::vector<struct ::dirent> entries;

    struct ::dirent* entry;



    while((entry = ::readdir(dirp)) != NULL) {

        entries.push_back(*entry);

    }



    if(opts.verbose) {

        fmt::print("readdir(pathname=\"{}\") = [\n{}],\nerrno: {} [{}]\n",

                   opts.pathname, fmt::join(entries, ",\n"), errno,

                   ::strerror(errno));

        return;

    }

    

    errno = 0;

    json out = directory_validate_output{(int) entries.size(), errno};

    fmt::print("{}\n", out.dump(2));

    return;

    

}



void

directory_validate_init(CLI::App& app) {



    // Create the option and subcommand objects

    auto opts = std::make_shared<directory_validate_options>();

    auto* cmd = app.add_subcommand(

            "directory_validate",

            "Create a file and execute a direntry system call and count the number of elements");



    // Add options to cmd, binding them to opts

    cmd->add_flag("-v,--verbose", opts->verbose,

                  "Produce human writeable output");



    cmd->add_option("pathname", opts->pathname, "file name, entries will be checked in last directory")

            ->required()

            ->type_name("");



    cmd->add_option("count", opts->count, "Number of files to check")

            ->required()

            ->type_name("");



    cmd->callback([opts]() { directory_validate_exec(*opts); });

}