simpler naming for the python bindings, start test infrastructure (1779c028) · Commits · hpc / gekkofs

src/client/CMakeLists.txt

+2 −0

Original line number	Diff line number	Diff line
		@@ -85,6 +85,8 @@ if (GKFS_BUILD_PYTHON_BINDINGS)
		gkfs_user_lib
		)

		set_target_properties(gkfs_python_bindings PROPERTIES OUTPUT_NAME gkfs_python)

		install(
		TARGETS gkfs_python_bindings
		LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}

src/client/gkfs_python.cpp

+40 −40

Original line number	Diff line number	Diff line
		@@ -7,44 +7,44 @@ namespace py = pybind11;
		PYBIND11_MODULE(gkfs_python, m) {
		m.doc() = "Python bindings for GekkoFS POSIX interface";

		m.def("gkfs_open", &gkfs::syscall::gkfs_open);
		m.def("open", &gkfs::syscall::gkfs_open);

		m.def("gkfs_create", &gkfs::syscall::gkfs_create);
		m.def("create", &gkfs::syscall::gkfs_create);

		m.def("gkfs_libcremove", &gkfs::syscall::gkfs_libcremove);
		m.def("libcremove", &gkfs::syscall::gkfs_libcremove);

		m.def("gkfs_remove", &gkfs::syscall::gkfs_remove);
		m.def("remove", &gkfs::syscall::gkfs_remove);

		m.def("gkfs_rmdir", &gkfs::syscall::gkfs_rmdir);
		m.def("rmdir", &gkfs::syscall::gkfs_rmdir);

		m.def("gkfs_write", [](int fd, py::bytes data) -> ssize_t {
		m.def("write", [](int fd, py::bytes data) -> ssize_t {
		py::buffer_info info(py::buffer(data).request());
		ssize_t ret = gkfs::syscall::gkfs_write(fd, info.ptr, info.size);
		if(ret < 0)
		throw std::runtime_error("gkfs_write failed");
		throw std::runtime_error("write failed");
		return ret;
		});

		m.def("gkfs_read", [](int fd, size_t count) -> py::bytes {
		m.def("read", [](int fd, size_t count) -> py::bytes {
		std::vector<char> buf(count);
		ssize_t ret = gkfs::syscall::gkfs_read(fd, buf.data(), count);
		if(ret < 0)
		throw std::runtime_error("gkfs_read failed");
		throw std::runtime_error("read failed");
		return py::bytes(buf.data(), ret);
		});

		m.def("gkfs_close", &gkfs::syscall::gkfs_close);
		m.def("close", &gkfs::syscall::gkfs_close);


		m.def(
		"gkfs_lseek",
		"lseek",
		[](unsigned int fd, int64_t offset, unsigned int whence) {
		return gkfs::syscall::gkfs_lseek(fd, offset, whence);
		},
		py::arg("fd"), py::arg("offset"), py::arg("whence"));

		m.def(
		"gkfs_pwrite",
		"pwrite",
		[](int fd, py::bytes data, int64_t offset) {
		std::string buf = data;
		return gkfs::syscall::gkfs_pwrite(fd, buf.data(), buf.size(),
		@@ -53,20 +53,20 @@ PYBIND11_MODULE(gkfs_python, m) {
		py::arg("fd"), py::arg("data"), py::arg("offset"));

		m.def(
		"gkfs_pread",
		"pread",
		[](int fd, size_t count, int64_t offset) {
		std::string buf(count, '\0');
		ssize_t nread = gkfs::syscall::gkfs_pread(fd, buf.data(), count,
		offset);
		if(nread < 0) {
		throw std::runtime_error("gkfs_pread failed");
		throw std::runtime_error("pread failed");
		}
		return py::bytes(buf.data(), nread);
		},
		py::arg("fd"), py::arg("count"), py::arg("offset"));

		m.def(
		"gkfs_readv",
		"readv",
		[](int fd, size_t iovcnt) {
		// TODO better allocation or provide size to calc
		std::vector<char> buffer(4096 * iovcnt);
		@@ -81,13 +81,13 @@ PYBIND11_MODULE(gkfs_python, m) {
		ssize_t nread =
		gkfs::syscall::gkfs_readv(fd, iov.data(), iovcnt);
		if(nread < 0)
		throw std::runtime_error("gkfs_readv failed");
		throw std::runtime_error("readv failed");
		return py::bytes(buffer.data(), nread);
		},
		py::arg("fd"), py::arg("iovcnt"));

		m.def(
		"gkfs_writev",
		"writev",
		[](int fd, const std::vector<py::bytes>& bufs) {
		std::vector<std::string> buf_data;
		std::vector<struct iovec> iov;
		@@ -105,7 +105,7 @@ PYBIND11_MODULE(gkfs_python, m) {
		py::arg("fd"), py::arg("buffers"));

		m.def(
		"gkfs_preadv",
		"preadv",
		[](int fd, size_t iovcnt, int64_t offset) {
		std::vector<char> buffer(4096 * iovcnt); // adjust as needed
		std::vector<struct iovec> iov(iovcnt);
		@@ -119,13 +119,13 @@ PYBIND11_MODULE(gkfs_python, m) {
		ssize_t nread = gkfs::syscall::gkfs_preadv(fd, iov.data(),
		iovcnt, offset);
		if(nread < 0)
		throw std::runtime_error("gkfs_preadv failed");
		throw std::runtime_error("preadv failed");
		return py::bytes(buffer.data(), nread);
		},
		py::arg("fd"), py::arg("iovcnt"), py::arg("offset"));

		m.def(
		"gkfs_pwritev",
		"pwritev",
		[](int fd, const std::vector<py::bytes>& bufs, int64_t offset) {
		std::vector<std::string> buf_data;
		std::vector<struct iovec> iov;
		@@ -143,7 +143,7 @@ PYBIND11_MODULE(gkfs_python, m) {
		},
		py::arg("fd"), py::arg("buffers"), py::arg("offset"));

		m.def("gkfs_stat",
		m.def("stat",
		[](const std::string& path, bool follow_links, bool bypass_rename) {
		struct stat buf;
		int res = gkfs::syscall::gkfs_stat(path, &buf, follow_links,
		@@ -167,7 +167,7 @@ PYBIND11_MODULE(gkfs_python, m) {
		return std::make_tuple(res, stat_result);
		});

		m.def("gkfs_statx", [](int dirfd, const std::string& path, int flags,
		m.def("statx", [](int dirfd, const std::string& path, int flags,
		unsigned int mask, bool follow_links) {
		struct statx stx;
		memset(&stx, 0, sizeof(stx));
		@@ -192,57 +192,57 @@ PYBIND11_MODULE(gkfs_python, m) {
		return py::make_tuple(ret, meta);
		});

		m.def("gkfs_dup", &gkfs::syscall::gkfs_dup);
		m.def("dup", &gkfs::syscall::gkfs_dup);

		m.def("gkfs_dup2", &gkfs::syscall::gkfs_dup2);
		m.def("dup2", &gkfs::syscall::gkfs_dup2);

		m.def("gkfs_access", &gkfs::syscall::gkfs_access, py::arg("path"),
		m.def("access", &gkfs::syscall::gkfs_access, py::arg("path"),
		py::arg("mask"), py::arg("follow_links") = true);

		m.def("gkfs_get_file_list", &gkfs::syscall::gkfs_get_file_list);
		m.def("get_file_list", &gkfs::syscall::gkfs_get_file_list);

		m.def("gkfs_opendir", &gkfs::syscall::gkfs_opendir);
		m.def("opendir", &gkfs::syscall::gkfs_opendir);

		m.def("gkfs_getdents", [](unsigned int fd, unsigned int count) {
		m.def("getdents", [](unsigned int fd, unsigned int count) {
		std::vector<char> buf(count);
		int ret = gkfs::syscall::gkfs_getdents(
		fd, reinterpret_cast<struct linux_dirent*>(buf.data()), count);
		if(ret < 0)
		throw std::runtime_error("gkfs_getdents failed");
		throw std::runtime_error("getdents failed");
		return py::bytes(buf.data(), ret);
		});

		m.def("gkfs_getdents64", [](unsigned int fd, unsigned int count) {
		m.def("getdents64", [](unsigned int fd, unsigned int count) {
		std::vector<char> buf(count);
		int ret = gkfs::syscall::gkfs_getdents64(
		fd, reinterpret_cast<struct linux_dirent64*>(buf.data()),
		count);
		if(ret < 0)
		throw std::runtime_error("gkfs_getdents64 failed");
		throw std::runtime_error("getdents64 failed");
		return py::bytes(buf.data(), ret);
		});

		m.def("gkfs_truncate", &gkfs::syscall::gkfs_truncate);
		m.def("truncate", &gkfs::syscall::gkfs_truncate);

		#ifdef HAS_SYMLINKS
		m.def("gkfs_mk_symlink", &gkfs::syscall::gkfs_mk_symlink);
		m.def("mk_symlink", &gkfs::syscall::gkfs_mk_symlink);

		m.def("gkfs_readlink", [](const std::string& path, int bufsize) {
		m.def("readlink", [](const std::string& path, int bufsize) {
		std::vector<char> buf(bufsize);
		int ret = gkfs::syscall::gkfs_readlink(path, buf.data(), bufsize);
		if(ret < 0)
		throw std::runtime_error("gkfs_readlink failed");
		throw std::runtime_error("readlink failed");
		return std::string(buf.data(), ret);
		});
		#endif

		#ifdef HAS_RENAME
		m.def("gkfs_rename", &gkfs::syscall::gkfs_rename);
		m.def("rename", &gkfs::syscall::gkfs_rename);
		#endif

		m.def("gkfs_fsync", &gkfs::syscall::gkfs_fsync);
		m.def("fsync", &gkfs::syscall::gkfs_fsync);

		m.def("gkfs_mmap", [](uintptr_t addr, size_t length, int prot, int flags,
		m.def("mmap", [](uintptr_t addr, size_t length, int prot, int flags,
		int fd, off_t offset) {
		void* result = gkfs::syscall::gkfs_mmap(
		reinterpret_cast<void*>(addr), length, prot, flags, fd, offset);
		@@ -251,12 +251,12 @@ PYBIND11_MODULE(gkfs_python, m) {
		return reinterpret_cast<uintptr_t>(result);
		});

		m.def("gkfs_munmap", [](uintptr_t addr, size_t length) {
		m.def("munmap", [](uintptr_t addr, size_t length) {
		return gkfs::syscall::gkfs_munmap(reinterpret_cast<void*>(addr),
		length);
		});

		m.def("gkfs_msync", [](uintptr_t addr, size_t length, int flags) {
		m.def("msync", [](uintptr_t addr, size_t length, int flags) {
		return gkfs::syscall::gkfs_msync(reinterpret_cast<void*>(addr), length,
		flags);
		});

tests/integration/CMakeLists.txt

+9 −0

Original line number	Diff line number	Diff line
		@@ -143,6 +143,15 @@ if (GKFS_RENAME_SUPPORT)
		)
		endif ()

		if (GKFS_BUILD_PYTHON_BINDINGS)
		gkfs_add_python_test(
		NAME test_python_bindings
		PYTHON_VERSION 3.6
		WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/tests/integration
		SOURCE python_bindings/
		)
		endif ()

		if (GKFS_INSTALL_TESTS)
		install(DIRECTORY harness
		DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/gkfs/tests/integration

tests/integration/python_bindings/test_python_bindings.py

0 → 100644

+107 −0

Original line number	Diff line number	Diff line
		################################################################################
		# Copyright 2018-2025, Barcelona Supercomputing Center (BSC), Spain #
		# Copyright 2015-2025, Johannes Gutenberg Universitaet Mainz, Germany #
		# #
		# This software was partially supported by the #
		# EC H2020 funded project NEXTGenIO (Project ID: 671951, www.nextgenio.eu). #
		# #
		# This software was partially supported by the #
		# ADA-FS project under the SPPEXA project funded by the DFG. #
		# #
		# This file is part of GekkoFS. #
		# #
		# GekkoFS is free software: you can redistribute it and/or modify #
		# it under the terms of the GNU General Public License as published by #
		# the Free Software Foundation, either version 3 of the License, or #
		# (at your option) any later version. #
		# #
		# GekkoFS is distributed in the hope that it will be useful, #
		# but WITHOUT ANY WARRANTY; without even the implied warranty of #
		# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
		# GNU General Public License for more details. #
		# #
		# You should have received a copy of the GNU General Public License #
		# along with GekkoFS. If not, see <https://www.gnu.org/licenses/>. #
		# #
		# SPDX-License-Identifier: GPL-3.0-or-later #
		################################################################################

		import harness
		from pathlib import Path
		import errno
		import stat
		import os
		import ctypes
		import sys
		import pytest
		import gkfs_python
		from harness.logger import logger



		nonexisting = "nonexisting"
		gkfs_hosts_file = 'gkfs_hosts.txt'


		def test_pybindings(gkfs_daemon, gkfs_client):

		file = gkfs_daemon.mountdir / "file"
		file2 = gkfs_daemon.mountdir / "file2"

		os.environ["GKFS_HOSTS_FILE"] = str(gkfs_daemon.cwd / gkfs_hosts_file)

		# create a file in gekkofs
		ret = gkfs_python.init()
		assert ret == 0
		ret = gkfs_python.get_file_list("/")
		assert ret == ['.', '..']
		# ret = gkfs_python.open(str(file),
		# os.O_CREAT \| os.O_WRONLY,
		# stat.S_IRWXU \| stat.S_IRWXG \| stat.S_IRWXO)
		#
		# assert ret != -1
		#
		# ret = gkfs_python.access(file, os.R_OK)
		# assert ret == 0
		#
		# # write a buffer we know
		# buf = b'42'
		# ret = gkfs_python.write(file, buf, len(buf))
		#
		# assert ret == len(buf) # Return the number of written bytes
		#
		# ret, st = gkfs_python.stat(file)
		# assert ret != 1
		#
		# ret, st = gkfs_python.stat(file2)
		# assert ret == -1
		#
		# ret = gkfs_python.rename(file, file2)
		# assert ret == 0
		#
		# ret = gkfs_python.access(file, os.R_OK)
		# assert ret == -1
		#
		# ret = gkfs_python.access(file2, os.R_OK)
		# assert ret == 0
		#
		# ret, st = gkfs_python.stat(file)
		# assert ret == -1
		#
		# ret, st = gkfs_python.stat(file2)
		# assert ret != 1

		# File is renamed, and innacesible

		# Read contents of file2
		# ret = gkfs_python.open(file2, os.O_RDONLY)
		# assert ret != -1
		#
		# ret = gkfs_python.read(file2, len(buf))
		# assert ret.retval == len(buf)
		# assert ret.buf == buf

		ret = gkfs_python.end()
		assert ret == 0