Resolve "Implement a testing harness for functional tests"

Alberto Mirandarequested to merge
76-implement-a-testing-harness-for-functional-tests into master

This MR adds a Python-based testing harness that helps writing functional/integration tests. The harness provides common functionalities such as creating a workspace for tests, starting a GekkoFS server, and running client processes with an appropriate environment (e.g. LD_PRELOAD, LD_LIBRARY_PATH etc.), among others. Tests are automatically found by the harness and return easy to interpret error codes (e.g. success/failure). More details on how the harness work can be found in the wiki.

  • Make tests self-contained
  • Generate detailed logs per-test
  • Allow using installed binaries to reduced CI artifact size
  • Log asserts
  • Add support for filtering out syscalls (e.g. epoll_wait()) in client logs (can be solved using lnav)
  • Add support for multiple client log files for complex tests (will solve in future MR)
  • Add support defining the network interface for tests via CMake option
  • Clean up .gitlab-ci.yml

Closes #76 (closed)

Edited by Alberto Miranda

Merge request reports

CMake/GkfsPythonTesting.cmake

0 → 100644
+218 −0
Changes for CMake/GkfsPythonTesting.cmake: 218 added lines, 0 removed lines.
Original line number Diff line number Diff line 
include(CMakeParseArguments)



function(gkfs_enable_python_testing)

    # Parse arguments

    set(MULTI BINARY_DIRECTORIES LIBRARY_PREFIX_DIRECTORIES)



    cmake_parse_arguments(PYTEST "${OPTION}" "${SINGLE}" "${MULTI}" ${ARGN})



    if(PYTEST_UNPARSED_ARGUMENTS)

        message(WARNING "Unparsed arguments in gkfs_enable_python_testing: This often indicates typos!")

    endif()



    if(PYTEST_BINARY_DIRECTORIES)

        set(GKFS_PYTEST_BINARY_DIRECTORIES ${PYTEST_BINARY_DIRECTORIES} PARENT_SCOPE)

    endif()



    if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)

        set(GKFS_PYTEST_LIBRARY_PREFIX_DIRECTORIES ${PYTEST_LIBRARY_PREFIX_DIRECTORIES} PARENT_SCOPE)

    endif()



    set(PYTEST_BINDIR_ARGS, "")

    if(PYTEST_BINARY_DIRECTORIES)

        foreach(dir IN LISTS PYTEST_BINARY_DIRECTORIES)

            list(APPEND PYTEST_BINDIR_ARGS "--bin-dir=${dir}")

        endforeach()

    endif()



    set(PYTEST_LIBDIR_ARGS, "")

    if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)

        foreach(dir IN LISTS PYTEST_LIBRARY_PREFIX_DIRECTORIES)



            if(NOT IS_ABSOLUTE ${dir})

                set(dir ${CMAKE_BINARY_DIR}/${dir})

            endif()



            file(TO_CMAKE_PATH "${dir}/lib" libdir)

            file(TO_CMAKE_PATH "${dir}/lib64" lib64dir)



            if(EXISTS ${libdir})

                list(APPEND PYTEST_LIBDIR_ARGS "--lib-dir=${libdir}")

            endif()



            if(EXISTS ${lib64dir})

                list(APPEND PYTEST_LIBDIR_ARGS "--lib-dir=${lib64dir}")

            endif()

        endforeach()

    endif()



    # convert path lists to space separated arguments

    string(REPLACE ";" " " PYTEST_BINDIR_ARGS "${PYTEST_BINDIR_ARGS}")

    string(REPLACE ";" " " PYTEST_BINDIR_ARGS "${PYTEST_BINDIR_ARGS}")



    configure_file(pytest.ini.in pytest.ini @ONLY)

    configure_file(conftest.py.in conftest.py @ONLY)

    configure_file(harness/cli.py harness/cli.py COPYONLY)



    if(GKFS_INSTALL_TESTS)

        configure_file(pytest.install.ini.in pytest.install.ini @ONLY)

        install(FILES ${CMAKE_CURRENT_BINARY_DIR}/pytest.install.ini

            DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration

            RENAME pytest.ini

        )



        install(FILES conftest.py

            DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration

        )



        if(NOT PYTEST_VIRTUALENV)

            set(PYTEST_VIRTUALENV ${CMAKE_INSTALL_FULL_DATAROOTDIR}/gkfs/tests/integration/pytest-venv)

        endif()



        # Python's virtual environments are not relocatable, we need to

        # recreate the virtualenv at the appropriate install location

        # find an appropriate python interpreter

        find_package(Python3

            3.6

            REQUIRED

            COMPONENTS Interpreter)



        if(NOT Python3_FOUND)

            message(FATAL_ERROR "Unable to find Python 3")

        endif()



        install(

            CODE "message(\"Install pytest virtual environment...\")"

            CODE "message(\"-- Create virtual environment: ${PYTEST_VIRTUALENV}\")"

            CODE "execute_process(COMMAND ${Python3_EXECUTABLE} -m venv ${PYTEST_VIRTUALENV})"

            CODE "message(\"-- Installing packages...\")"

            CODE "execute_process(COMMAND ${PYTEST_VIRTUALENV}/bin/pip install --upgrade pip -v)"

            CODE "execute_process(COMMAND ${PYTEST_VIRTUALENV}/bin/pip install -r ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt --upgrade -v)"

        )

    endif()



    # enable testing

    set(GKFS_PYTHON_TESTING_ENABLED ON PARENT_SCOPE)



endfunction()



function(gkfs_add_python_test)

    # ignore call if testing is not enabled

    if(NOT CMAKE_TESTING_ENABLED OR NOT GKFS_PYTHON_TESTING_ENABLED)

        return()

    endif()



    # Parse arguments

    set(OPTION)

    set(SINGLE NAME PYTHON_VERSION WORKING_DIRECTORY VIRTUALENV)

    set(MULTI SOURCE BINARY_DIRECTORIES LIBRARY_PREFIX_DIRECTORIES)



    cmake_parse_arguments(PYTEST "${OPTION}" "${SINGLE}" "${MULTI}" ${ARGN})



    if(PYTEST_UNPARSED_ARGUMENTS)

        message(WARNING "Unparsed arguments in gkfs_add_python_test: This often indicates typos!")

    endif()



    if(NOT PYTEST_NAME)

        message(FATAL_ERROR "gkfs_add_python_test requires a NAME argument")

    endif()



    # set default values for arguments not provided

    if(NOT PYTEST_PYTHON_VERSION)

        set(PYTEST_PYTHON_VERSION 3.0)

    endif()



    if(NOT PYTEST_WORKING_DIRECTORY)

        set(PYTEST_WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})

    endif()



    if(NOT PYTEST_VIRTUALENV)

        set(PYTEST_VIRTUALENV ${CMAKE_CURRENT_BINARY_DIR}/pytest-venv)

    endif()



    # if the test doesn't provide a list of binary or library prefix

    # directories, use the one set on gkfs_enable_python_testing()

    if(NOT PYTEST_BINARY_DIRECTORIES)

        set(PYTEST_BINARY_DIRECTORIES ${GKFS_PYTEST_BINARY_DIRECTORIES})

    endif()



    if(NOT PYTEST_LIBRARY_PREFIX_DIRECTORIES)

        set(PYTEST_LIBRARY_PREFIX_DIRECTORIES ${GKFS_PYTEST_LIBRARY_PREFIX_DIRECTORIES})

    endif()



    set(PYTEST_COMMAND_ARGS, "")

    if(PYTEST_BINARY_DIRECTORIES)

        foreach(dir IN LISTS PYTEST_BINARY_DIRECTORIES)

            list(APPEND PYTEST_COMMAND_ARGS "--bin-dir=${dir}")

        endforeach()

    endif()



    if(PYTEST_LIBRARY_PREFIX_DIRECTORIES)

        foreach(dir IN LISTS PYTEST_LIBRARY_PREFIX_DIRECTORIES)



            if(NOT IS_ABSOLUTE ${dir})

                set(dir ${CMAKE_BINARY_DIR}/${dir})

            endif()



            file(TO_CMAKE_PATH "${dir}/lib" libdir)

            file(TO_CMAKE_PATH "${dir}/lib64" lib64dir)



            if(EXISTS "${dir}/lib")

                list(APPEND PYTEST_COMMAND_ARGS "--lib-dir=${libdir}")

            endif()



            if(EXISTS "${dir}/lib64")

                list(APPEND PYTEST_COMMAND_ARGS "--lib-dir=${lib64dir}")

            endif()

        endforeach()

    endif()



    # Extend the given virtualenv to be a full path.

    if(NOT IS_ABSOLUTE ${PYTEST_VIRTUALENV})

        set(PYTEST_VIRTUALENV ${CMAKE_BINARY_DIR}/${PYTEST_VIRTUALENV})

    endif()



    # find an appropriate python interpreter

    find_package(Python3

        ${PYTEST_PYTHON_VERSION}

        REQUIRED

        COMPONENTS Interpreter)



    set(PYTEST_VIRTUALENV_PIP ${PYTEST_VIRTUALENV}/bin/pip)

    set(PYTEST_VIRTUALENV_INTERPRETER ${PYTEST_VIRTUALENV}/bin/python)



    # create a virtual environment to run the test

    configure_file(requirements.txt.in requirements.txt @ONLY)



    add_custom_command(

        OUTPUT ${PYTEST_VIRTUALENV}

        COMMENT "Creating virtual environment ${PYTEST_VIRTUALENV}"

        COMMAND Python3::Interpreter -m venv "${PYTEST_VIRTUALENV}"

        COMMAND ${PYTEST_VIRTUALENV_PIP} install --upgrade pip -q

        COMMAND ${PYTEST_VIRTUALENV_PIP} install -r requirements.txt --upgrade -q

    )



    if(NOT TARGET venv)

        # ensure that the virtual environment is created by the build process

        # (this is required because we can't add dependencies between

        # "test targets" and "normal targets"

        add_custom_target(venv

            ALL

            DEPENDS ${PYTEST_VIRTUALENV}

            DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/requirements.txt)

    endif()



    add_test(NAME ${PYTEST_NAME}

             COMMAND ${PYTEST_VIRTUALENV_INTERPRETER}

                    -m pytest -v -s

                    ${PYTEST_COMMAND_ARGS}

                    ${PYTEST_SOURCE}

             WORKING_DIRECTORY ${PYTEST_WORKING_DIRECTORY})



    # instruct Python to not create __pycache__ directories,

    # otherwise they will pollute ${PYTEST_WORKING_DIRECTORY} which

    # is typically ${PROJECT_SOURCE_DIR}

    set_tests_properties(${PYTEST_NAME} PROPERTIES

        ENVIRONMENT PYTHONDONTWRITEBYTECODE=1)



endfunction()

docker/debian_build_env.docker

+13 −8
Changes for docker/debian_build_env.docker: 13 added lines, 8 removed lines.
Original line number Diff line number Diff line
@@ -35,14 +35,19 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ 
		libboost-program-options-dev \

		valgrind \

		uuid-dev \

		python3 \

		python3-dev \

		python3-venv \

		expect \

# Clean apt cache to reduce image layer size

&& rm -rf /var/lib/apt/lists/*



# Download dependencies source

COPY scripts/dl_dep.sh		$SCRIPTS_PATH/

RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all



# Compile dependencies

COPY scripts/compile_dep.sh $SCRIPTS_PATH/

COPY scripts/patches        $SCRIPTS_PATH/patches

RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH
 
## COPY scripts/dl_dep.sh		$SCRIPTS_PATH/

## COPY scripts/compile_dep.sh $SCRIPTS_PATH/

## COPY scripts/patches        $SCRIPTS_PATH/patches

## 

## # Download dependencies source

## RUN /bin/bash $SCRIPTS_PATH/dl_dep.sh $DEPS_SRC_PATH all

## 

## # Compile dependencies

## RUN /bin/bash $SCRIPTS_PATH/compile_dep.sh $DEPS_SRC_PATH $INSTALL_PATH

test/README.md

0 → 100644
+11 −0
Changes for test/README.md: 11 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# README



This directory contains old/deprecated GekkoFS tests. It is kept here until all

tests have been migrated to the new testing framework. 





***

**IMPORTANT:** 



Some of these tests are still active in the CI scripts.

***

tests/integration/directories/README.md

0 → 100644
+4 −0
Changes for tests/integration/directories/README.md: 4 added lines, 0 removed lines.
Original line number Diff line number Diff line 
# README



This directory contains functional tests for any directory-related

functionalities in GekkoFS.

tests/integration/directories/test_directories.py

0 → 100644
+213 −0
Changes for tests/integration/directories/test_directories.py: 213 added lines, 0 removed lines.
Original line number Diff line number Diff line 
################################################################################

#  Copyright 2018-2020, Barcelona Supercomputing Center (BSC), Spain           #

#  Copyright 2015-2020, Johannes Gutenberg Universitaet Mainz, Germany         #

#                                                                              #

#  This software was partially supported by the                                #

#  EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu).   #

#                                                                              #

#  This software was partially supported by the                                #

#  ADA-FS project under the SPPEXA project funded by the DFG.                  #

#                                                                              #

#  SPDX-License-Identifier: MIT                                                #

################################################################################



import harness

from pathlib import Path

import errno

import stat

import os

import ctypes

import sh

import sys

import pytest

from harness.logger import logger



nonexisting = "nonexisting"





#@pytest.mark.xfail(reason="invalid errno returned on success")

def test_mkdir(gkfs_daemon, gkfs_client):

    """Create a new directory in the FS's root"""



    topdir = gkfs_daemon.mountdir / "top"

    longer = Path(topdir.parent, topdir.name + "_plus")

    dir_a  = topdir / "dir_a"

    dir_b  = topdir / "dir_b"

    file_a = topdir / "file_a"

    subdir_a  = dir_a / "subdir_a"



    # create topdir

    ret = gkfs_client.mkdir(

            topdir,

            stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!



    # test stat on existing dir

    ret = gkfs_client.stat(topdir)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!

    assert stat.S_ISDIR(ret.statbuf.st_mode)



    # open topdir

    ret = gkfs_client.open(topdir, os.O_DIRECTORY)

    assert ret.retval != -1

    assert ret.errno == 115 #FIXME: Should be 0!





    # read and write should be impossible on directories

    ret = gkfs_client.read(topdir, 1)



    assert ret.buf is None

    assert ret.retval == -1

    assert ret.errno == errno.EISDIR



    # buf = bytes('42', sys.stdout.encoding)

    # print(buf.hex())

    buf = b'42'

    ret = gkfs_client.write(topdir, buf, 1)



    assert ret.retval == -1

    assert ret.errno == errno.EISDIR





    # read top directory that is empty

    ret = gkfs_client.opendir(topdir)



    assert ret.dirp is not None

    assert ret.errno == 115 #FIXME: Should be 0!



    ret = gkfs_client.readdir(topdir)



    # XXX: This might change in the future if we add '.' and '..'

    assert len(ret.dirents) == 0

    assert ret.errno == 115 #FIXME: Should be 0!



    # close directory

    # TODO: disabled for now because we have no way to keep DIR* alive

    # between gkfs.io executions

    # ret = gkfs_client.opendir(XXX)

    # assert ret.errno == 115 #FIXME: Should be 0!





    # populate top directory

    for d in [dir_a, dir_b]:

        ret = gkfs_client.mkdir(

                d,

                stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



        assert ret.retval == 0

        assert ret.errno == 115 #FIXME: Should be 0!



    ret = gkfs_client.open(file_a,

                           os.O_CREAT,

                           stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval != -1

    assert ret.errno == 115 #FIXME: Should be 0!



    ret = gkfs_client.readdir(gkfs_daemon.mountdir)



    # XXX: This might change in the future if we add '.' and '..'

    assert len(ret.dirents) == 1

    assert ret.dirents[0].d_name == 'top'

    assert ret.dirents[0].d_type == 4 # DT_DIR

    assert ret.errno == 115 #FIXME: Should be 0!



    expected = [

        ( dir_a.name,  4 ), # DT_DIR

        ( dir_b.name,  4 ),

        ( file_a.name, 8 ) # DT_REG

    ]



    ret = gkfs_client.readdir(topdir)

    assert len(ret.dirents) == len(expected)

    assert ret.errno == 115 #FIXME: Should be 0!



    for d,e in zip(ret.dirents, expected):

        assert d.d_name == e[0]

        assert d.d_type == e[1]



    # remove file using rmdir should produce an error

    ret = gkfs_client.rmdir(file_a)

    assert ret.retval == -1

    assert ret.errno == errno.ENOTDIR



    # create a directory with the same prefix as topdir but longer name

    ret = gkfs_client.mkdir(

            longer,

            stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!



    expected = [

        ( topdir.name,  4 ), # DT_DIR

        ( longer.name,  4 ), # DT_DIR

    ]



    ret = gkfs_client.readdir(gkfs_daemon.mountdir)

    assert len(ret.dirents) == len(expected)

    assert ret.errno == 115 #FIXME: Should be 0!



    for d,e in zip(ret.dirents, expected):

        assert d.d_name == e[0]

        assert d.d_type == e[1]



    # create 2nd level subdir and check it's not included in readdir()

    ret = gkfs_client.mkdir(

            subdir_a,

            stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)



    assert ret.retval == 0

    assert ret.errno == 115 #FIXME: Should be 0!



    expected = [

        ( topdir.name,  4 ), # DT_DIR

        ( longer.name,  4 ), # DT_DIR

    ]



    ret = gkfs_client.readdir(gkfs_daemon.mountdir)

    assert len(ret.dirents) == len(expected)

    assert ret.errno == 115 #FIXME: Should be 0!



    for d,e in zip(ret.dirents, expected):

        assert d.d_name == e[0]

        assert d.d_type == e[1]



    expected = [

        ( subdir_a.name,  4 ), # DT_DIR

    ]



    ret = gkfs_client.readdir(dir_a)



    assert len(ret.dirents) == len(expected)

    assert ret.errno == 115 #FIXME: Should be 0!



    for d,e in zip(ret.dirents, expected):

        assert d.d_name == e[0]

        assert d.d_type == e[1]





    return



@pytest.mark.skip(reason="invalid errno returned on success")

@pytest.mark.parametrize("directory_path",

    [ nonexisting ])

def test_opendir(gkfs_daemon, gkfs_client, directory_path):



    ret = gkfs_client.opendir(gkfs_daemon.mountdir / directory_path)



    assert ret.dirp is None

    assert ret.errno == errno.ENOENT



# def test_stat(gkfs_daemon):

#     pass

#

# def test_rmdir(gkfs_daemon):

#     pass

#

# def test_closedir(gkfs_daemon):

#     pass