Resolve "Add I/O tests"

Ramon Nourequested to merge
107-add-i-o-tests into master

Closes #107 (closed)

Edited by Marc Vef

Merge request reports

tests/integration/data/README.md

0 → 100644
+4 −0
Changes for tests/integration/data/README.md: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# README



This directory contains functional tests for any data-related

functionalities in GekkoFS.

tests/integration/data/test_data_integrity.py

0 → 100644
+105 −0
Changes for tests/integration/data/test_data_integrity.py: 105 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

#  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain           #

#  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany         #

#                                                                              #

#  This software was partially supported by the                                #

#  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).   #

#                                                                              #

#  This software was partially supported by the                                #

#  ADA-FS project under the SPPEXA project funded by the DFG.                  #

#                                                                              #

#  SPDX-License-Identifier: MIT                                                #

################################################################################



import harness

from pathlib import Path

import errno

import stat

import os

import ctypes

import sh

import sys

import pytest

import string

import random

from harness.logger import logger



nonexisting = "nonexisting"

chunksize_start = 128192

chunksize_end = 2097153

step = 4096*9



def generate_random_data(size):

    return ''.join([random.choice(string.ascii_letters + string.digits) for _ in range(size)])









#@pytest.mark.xfail(reason="invalid errno returned on success")

def test_data_integrity(gkfs_daemon, gkfs_client):

    """Test several data write-read commands and check that the data is correct"""

    topdir = gkfs_daemon.mountdir / "top"

    file_a = topdir / "file_a"



    # create topdir

    ret = gkfs_client.mkdir(

            topdir,

            stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!



    # test statx on existing dir

    ret = gkfs_client.statx(0, topdir, 0, 0)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!

    assert stat.S_ISDIR(ret.statbuf.stx_mode)



    ret = gkfs_client.open(file_a,

                   os.O_CREAT,

                   stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval != -1





    # test statx on existing file

    ret = gkfs_client.statx(0, file_a, 0, 0)



    assert ret.retval == 0

    assert (stat.S_ISDIR(ret.statbuf.stx_mode)==0)

    assert (ret.statbuf.stx_size == 0)





    # Step 1 - small sizes

    

    # Generate writes

    # Read data

    # Compare buffer



    

    for i in range (1, 512, 64):

        buf = bytes(generate_random_data(i), sys.stdout.encoding)

        

        ret = gkfs_client.write(file_a, buf, i)



        assert ret.retval == i

        ret = gkfs_client.statx(0, file_a, 0, 0)



        assert ret.retval == 0

        assert (ret.statbuf.stx_size == i)



        ret = gkfs_client.read(file_a, i)

        assert ret.retval== i

        assert ret.buf == buf





    # Step 2 - Compare bigger sizes exceeding typical chunksize

    for i in range (chunksize_start, chunksize_end, step):

        ret = gkfs_client.write_validate(file_a, i)

        assert ret.retval == 1





    return

tests/integration/harness/gkfs.io/commands.hpp

+3 −0
Changes for tests/integration/harness/gkfs.io/commands.hpp: 3 added lines, 0 removed lines.
Original line number Diff line number Diff line
@@ -67,4 +67,7 @@ statx_init(CLI::App& app); 
void

lseek_init(CLI::App& app);



void

write_validate_init(CLI::App& app);



#endif // IO_COMMANDS_HPP

tests/integration/harness/gkfs.io/main.cpp

+1 −0
Changes for tests/integration/harness/gkfs.io/main.cpp: 1 added line, 0 removed lines.
Original line number Diff line number Diff line
@@ -38,6 +38,7 @@ init_commands(CLI::App& app) { 
    statx_init(app);

    #endif

    lseek_init(app);

    write_validate_init(app);

}

tests/integration/harness/gkfs.io/write_validate.cpp

0 → 100644
+169 −0
Changes for tests/integration/harness/gkfs.io/write_validate.cpp: 169 added lines, 0 removed lines.
Original line number Diff line number Diff line 
/*

  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain

  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany



  This software was partially supported by the

  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).



  This software was partially supported by the

  ADA-FS project under the SPPEXA project funded by the DFG.



  SPDX-License-Identifier: MIT

*/



/* C++ includes */

#include <CLI/CLI.hpp>

#include <nlohmann/json.hpp>

#include <memory>

#include <fmt/format.h>

#include <commands.hpp>

#include <reflection.hpp>

#include <serialize.hpp>

#include <binary_buffer.hpp>



/* C includes */

#include <sys/types.h>

#include <sys/stat.h>

#include <fcntl.h>

#include <unistd.h>



using json = nlohmann::json;



struct write_validate_options {

    bool verbose;

    std::string pathname;

    ::size_t count;



    REFL_DECL_STRUCT(write_validate_options,

        REFL_DECL_MEMBER(bool, verbose),

        REFL_DECL_MEMBER(std::string, pathname),

        REFL_DECL_MEMBER(::size_t, count)

    );

};



struct write_validate_output {

    int retval;

    int errnum;



    REFL_DECL_STRUCT(write_validate_output,

        REFL_DECL_MEMBER(int, retval),

        REFL_DECL_MEMBER(int, errnum)

    );

};



void

to_json(json& record, 

        const write_validate_output& out) {

    record = serialize(out);

}



void 

write_validate_exec(const write_validate_options& opts) {



    int fd = ::open(opts.pathname.c_str(), O_WRONLY);



    if(fd == -1) {

        if(opts.verbose) {

            fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", 

                    opts.pathname, opts.count, fd, errno, ::strerror(errno));

            return;

        }



        json out = write_validate_output{fd, errno};

        fmt::print("{}\n", out.dump(2));



        return;

    }





    std::string data = "";

    for (::size_t i = 0 ; i < opts.count; i++)

    {

        data += char((i%10)+'0');

    }



    io::buffer buf(data);



    auto rv = ::write(fd, buf.data(), opts.count);



    if(opts.verbose) {

        fmt::print("write_validate(pathname=\"{}\", count={}) = {}, errno: {} [{}]\n", 

                   opts.pathname, opts.count, rv, errno, ::strerror(errno));

        return;

    }



    if (rv < 0 or ::size_t(rv) != opts.count) {

        json out = write_validate_output{(int)rv, errno};

        fmt::print("{}\n", out.dump(2));

        return;

    }





    io::buffer bufread(opts.count);



    size_t total = 0;

    do{

        rv = ::read(fd, bufread.data(), opts.count-total);

        total += rv;

    } while (rv > 0 and total < opts.count);



    if (rv < 0 and total != opts.count) {

        json out = write_validate_output{(int)rv, errno};

        fmt::print("{}\n", out.dump(2));

        return;

    }



    if ( memcmp(buf.data(),bufread.data(),opts.count) ) {

        rv = 1;

        errno = 0;

        json out = write_validate_output{(int)rv, errno};

        fmt::print("{}\n", out.dump(2));

        return;

    }

    else  {

        rv = 2;

        errno = EINVAL;

        json out = write_validate_output{(int)-1, errno};

        fmt::print("{}\n", out.dump(2));

    }



}



void

write_validate_init(CLI::App& app) {



    // Create the option and subcommand objects

    auto opts = std::make_shared<write_validate_options>();

    auto* cmd = app.add_subcommand(

            "write_validate", 

            "Execute the write()-read() system call and compare the content of the buffer");



    // Add options to cmd, binding them to opts

    cmd->add_flag(

            "-v,--verbose",

            opts->verbose,

            "Produce human writeable output"

        );



    cmd->add_option(

            "pathname", 

            opts->pathname,

            "Directory name"

        )

        ->required()

        ->type_name("");



    cmd->add_option(

            "count", 

            opts->count,

            "Number of bytes to test"

        )

        ->required()

        ->type_name("");



    cmd->callback([opts]() { 

        write_validate_exec(*opts); 

    });

}