Merge branch '117-refactor-readdir' into 'master' (bcc08481) · Commits · David Auer / gekkofs

include/client/rpc/forward_data.hpp

+6 −4

Original line number	Diff line number	Diff line
		@@ -24,14 +24,16 @@ struct ChunkStat {
		unsigned long chunk_free;
		};

		ssize_t forward_write(const std::string& path, const void* buf, bool append_flag, off64_t in_offset,
		// TODO once we have LEAF, remove all the error code returns and throw them as an exception.

		std::pair<int, ssize_t> forward_write(const std::string& path, const void* buf, bool append_flag, off64_t in_offset,
		size_t write_size, int64_t updated_metadentry_size);

		ssize_t forward_read(const std::string& path, void* buf, off64_t offset, size_t read_size);
		std::pair<int, ssize_t> forward_read(const std::string& path, void* buf, off64_t offset, size_t read_size);

		int forward_truncate(const std::string& path, size_t current_size, size_t new_size);

		ChunkStat forward_get_chunk_stat();
		std::pair<int, ChunkStat> forward_get_chunk_stat();

		} // namespace rpc
		} // namespace gkfs

include/client/rpc/forward_metadata.hpp

+7 −4

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@
		#define GEKKOFS_CLIENT_FORWARD_METADATA_HPP

		#include <string>
		#include <memory>

		/* Forward declaration */
		namespace gkfs {
		@@ -27,6 +28,8 @@ struct MetadentryUpdateFlags;
		class Metadata;
		}

		// TODO once we have LEAF, remove all the error code returns and throw them as an exception.

		namespace rpc {

		int forward_create(const std::string& path, mode_t mode);
		@@ -40,12 +43,12 @@ int forward_decr_size(const std::string& path, size_t length);
		int forward_update_metadentry(const std::string& path, const gkfs::metadata::Metadata& md,
		const gkfs::metadata::MetadentryUpdateFlags& md_flags);

		int forward_update_metadentry_size(const std::string& path, size_t size, off64_t offset, bool append_flag,
		off64_t& ret_size);
		std::pair<int, off64_t>
		forward_update_metadentry_size(const std::string& path, size_t size, off64_t offset, bool append_flag);

		int forward_get_metadentry_size(const std::string& path, off64_t& ret_size);
		std::pair<int, off64_t> forward_get_metadentry_size(const std::string& path);

		void forward_get_dirents(gkfs::filemap::OpenDir& open_dir);
		std::pair<int, std::shared_ptr<gkfs::filemap::OpenDir>> forward_get_dirents(const std::string& path);

		#ifdef HAS_SYMLINKS

src/client/gkfs_functions.cpp

+334 −53

Original line number	Diff line number	Diff line
		@@ -61,6 +61,12 @@ struct linux_dirent64 {

		namespace {

		/**
		* Checks if metadata for parent directory exists (can be disabled with CREATE_CHECK_PARENTS).
		* errno may be set
		* @param path
		* @return 0 on success, -1 on failure
		*/
		int check_parent_dir(const std::string& path) {
		#if CREATE_CHECK_PARENTS
		auto p_comp = gkfs::path::dirname(path);
		@@ -86,6 +92,14 @@ int check_parent_dir(const std::string& path) {
		namespace gkfs {
		namespace syscall {

		/**
		* gkfs wrapper for open() system calls
		* errno may be set
		* @param path
		* @param mode
		* @param flags
		* @return 0 on success, -1 on failure
		*/
		int gkfs_open(const std::string& path, mode_t mode, int flags) {

		if (flags & O_PATH) {
		@@ -171,6 +185,13 @@ int gkfs_open(const std::string& path, mode_t mode, int flags) {
		return CTX->file_map()->add(std::make_shared<gkfs::filemap::OpenFile>(path, flags));
		}

		/**
		* Wrapper function for file/directory creation
		* errno may be set
		* @param path
		* @param mode
		* @return 0 on success, -1 on failure
		*/
		int gkfs_create(const std::string& path, mode_t mode) {

		//file type must be set
		@@ -197,13 +218,19 @@ int gkfs_create(const std::string& path, mode_t mode) {
		if (check_parent_dir(path)) {
		return -1;
		}
		return gkfs::rpc::forward_create(path, mode);
		auto err = gkfs::rpc::forward_create(path, mode);
		if (err) {
		errno = err;
		return -1;
		}
		return 0;
		}

		/**
		* This sends internally a broadcast (i.e. n RPCs) to clean their chunk folders for that path
		* gkfs wrapper for unlink() system calls
		* errno may be set
		* @param path
		* @return
		* @return 0 on success, -1 on failure
		*/
		int gkfs_remove(const std::string& path) {
		auto md = gkfs::util::get_metadata(path);
		@@ -211,9 +238,22 @@ int gkfs_remove(const std::string& path) {
		return -1;
		}
		bool has_data = S_ISREG(md->mode()) && (md->size() != 0);
		return gkfs::rpc::forward_remove(path, !has_data, md->size());
		auto err = gkfs::rpc::forward_remove(path, !has_data, md->size());
		if (err) {
		errno = err;
		return -1;
		}
		return 0;
		}

		/**
		* gkfs wrapper for access() system calls
		* errno may be set
		* @param path
		* @param mask
		* @param follow_links
		* @return 0 on success, -1 on failure
		*/
		int gkfs_access(const std::string& path, const int mask, bool follow_links) {
		auto md = gkfs::util::get_metadata(path, follow_links);
		if (!md) {
		@@ -223,6 +263,14 @@ int gkfs_access(const std::string& path, const int mask, bool follow_links) {
		return 0;
		}

		/**
		* gkfs wrapper for stat() system calls
		* errno may be set
		* @param path
		* @param buf
		* @param follow_links
		* @return 0 on success, -1 on failure
		*/
		int gkfs_stat(const string& path, struct stat* buf, bool follow_links) {
		auto md = gkfs::util::get_metadata(path, follow_links);
		if (!md) {
		@@ -233,6 +281,18 @@ int gkfs_stat(const string& path, struct stat* buf, bool follow_links) {
		}

		#ifdef STATX_TYPE

		/**
		* gkfs wrapper for statx() system calls
		* errno may be set
		* @param dirfs
		* @param path
		* @param flags
		* @param mask
		* @param buf
		* @param follow_links
		* @return 0 on success, -1 on failure
		*/
		int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask, struct statx* buf, bool follow_links) {
		auto md = gkfs::util::get_metadata(path, follow_links);
		if (!md) {
		@@ -270,14 +330,22 @@ int gkfs_statx(int dirfs, const std::string& path, int flags, unsigned int mask,
		}
		#endif

		/**
		* gkfs wrapper for statfs() system calls
		* errno may be set
		* @param buf
		* @return 0 on success, -1 on failure
		*/
		int gkfs_statfs(struct statfs* buf) {
		gkfs::rpc::ChunkStat blk_stat{};
		try {
		blk_stat = gkfs::rpc::forward_get_chunk_stat();
		} catch (const std::exception& e) {
		LOG(ERROR, "{}() Failure with error: '{}'", e.what());

		auto ret = gkfs::rpc::forward_get_chunk_stat();
		auto err = ret.first;
		if (err) {
		LOG(ERROR, "{}() Failure with error: '{}'", err);
		errno = err;
		return -1;
		}
		auto blk_stat = ret.second;
		buf->f_type = 0;
		buf->f_bsize = blk_stat.chunk_size;
		buf->f_blocks = blk_stat.chunk_total;
		@@ -293,14 +361,24 @@ int gkfs_statfs(struct statfs* buf) {
		return 0;
		}

		/**
		* gkfs wrapper for statvfs() system calls
		* errno may be set
		*
		* NOTE: Currently unused.
		*
		* @param buf
		* @return 0 on success, -1 on failure
		*/
		int gkfs_statvfs(struct statvfs* buf) {
		gkfs::rpc::ChunkStat blk_stat{};
		try {
		blk_stat = gkfs::rpc::forward_get_chunk_stat();
		} catch (const std::exception& e) {
		LOG(ERROR, "{}() Failure with error: '{}'", e.what());
		auto ret = gkfs::rpc::forward_get_chunk_stat();
		auto err = ret.first;
		if (err) {
		LOG(ERROR, "{}() Failure with error: '{}'", err);
		errno = err;
		return -1;
		}
		auto blk_stat = ret.second;
		buf->f_bsize = blk_stat.chunk_size;
		buf->f_blocks = blk_stat.chunk_total;
		buf->f_bfree = blk_stat.chunk_free;
		@@ -316,10 +394,26 @@ int gkfs_statvfs(struct statvfs* buf) {
		return 0;
		}

		/**
		* gkfs wrapper for lseek() system calls with available file descriptor
		* errno may be set
		* @param fd
		* @param offset
		* @param whence
		* @return 0 on success, -1 on failure
		*/
		off_t gkfs_lseek(unsigned int fd, off_t offset, unsigned int whence) {
		return gkfs_lseek(CTX->file_map()->get(fd), offset, whence);
		}

		/**
		* gkfs wrapper for lseek() system calls with available shared ptr to gkfs FileMap
		* errno may be set
		* @param gkfs_fd
		* @param offset
		* @param whence
		* @return 0 on success, -1 on failure
		*/
		off_t gkfs_lseek(shared_ptr<gkfs::filemap::OpenFile> gkfs_fd, off_t offset, unsigned int whence) {
		switch (whence) {
		case SEEK_SET:
		@@ -329,15 +423,15 @@ off_t gkfs_lseek(shared_ptr<gkfs::filemap::OpenFile> gkfs_fd, off_t offset, unsi
		gkfs_fd->pos(gkfs_fd->pos() + offset);
		break;
		case SEEK_END: {
		off64_t file_size;
		auto err = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path(), file_size);

		if (err < 0) {
		errno = err; // Negative numbers are explicitly for error codes
		auto ret = gkfs::rpc::forward_get_metadentry_size(gkfs_fd->path());
		auto err = ret.first;
		if (err) {
		errno = err;
		return -1;
		}

		if (offset < 0 and file_size < -offset) {
		auto file_size = ret.second;
		if (offset < 0 && file_size < -offset) {
		errno = EINVAL;
		return -1;
		}
		@@ -362,6 +456,14 @@ off_t gkfs_lseek(shared_ptr<gkfs::filemap::OpenFile> gkfs_fd, off_t offset, unsi
		return gkfs_fd->pos();
		}

		/**
		* wrapper function for gkfs_truncate
		* errno may be set
		* @param path
		* @param old_size
		* @param new_size
		* @return 0 on success, -1 on failure
		*/
		int gkfs_truncate(const std::string& path, off_t old_size, off_t new_size) {
		assert(new_size >= 0);
		assert(new_size <= old_size);
		@@ -369,19 +471,29 @@ int gkfs_truncate(const std::string& path, off_t old_size, off_t new_size) {
		if (new_size == old_size) {
		return 0;
		}

		if (gkfs::rpc::forward_decr_size(path, new_size)) {
		auto err = gkfs::rpc::forward_decr_size(path, new_size);
		if (err) {
		LOG(DEBUG, "Failed to decrease size");
		errno = err;
		return -1;
		}

		if (gkfs::rpc::forward_truncate(path, old_size, new_size)) {
		err = gkfs::rpc::forward_truncate(path, old_size, new_size);
		if (err) {
		LOG(DEBUG, "Failed to truncate data");
		errno = err;
		return -1;
		}
		return 0;
		}

		/**
		* gkfs wrapper for truncate() system calls
		* errno may be set
		* @param path
		* @param length
		* @return 0 on success, -1 on failure
		*/
		int gkfs_truncate(const std::string& path, off_t length) {
		/* TODO CONCURRENCY:
		* At the moment we first ask the length to the metadata-server in order to
		@@ -410,14 +522,36 @@ int gkfs_truncate(const std::string& path, off_t length) {
		return gkfs_truncate(path, size, length);
		}

		/**
		* gkfs wrapper for dup() system calls
		* errno may be set
		* @param oldfd
		* @return file descriptor int or -1 on error
		*/
		int gkfs_dup(const int oldfd) {
		return CTX->file_map()->dup(oldfd);
		}

		/**
		* gkfs wrapper for dup2() system calls
		* errno may be set
		* @param oldfd
		* @param newfd
		* @return file descriptor int or -1 on error
		*/
		int gkfs_dup2(const int oldfd, const int newfd) {
		return CTX->file_map()->dup2(oldfd, newfd);
		}

		/**
		* Wrapper function for all gkfs write operations
		* errno may be set
		* @param file
		* @param buf
		* @param count
		* @param offset
		* @return written size or -1 on error
		*/
		ssize_t gkfs_pwrite(std::shared_ptr<gkfs::filemap::OpenFile> file, const char* buf, size_t count, off64_t offset) {
		if (file->type() != gkfs::filemap::FileType::regular) {
		assert(file->type() == gkfs::filemap::FileType::directory);
		@@ -427,30 +561,47 @@ ssize_t gkfs_pwrite(std::shared_ptr<gkfs::filemap::OpenFile> file, const char* b
		}
		auto path = make_shared<string>(file->path());
		auto append_flag = file->get_flag(gkfs::filemap::OpenFile_flags::append);
		ssize_t ret = 0;
		long updated_size = 0;

		ret = gkfs::rpc::forward_update_metadentry_size(*path, count, offset, append_flag, updated_size);
		if (ret != 0) {
		LOG(ERROR, "update_metadentry_size() failed with ret {}", ret);
		return ret; // ERR
		auto ret_update_size = gkfs::rpc::forward_update_metadentry_size(*path, count, offset, append_flag);
		auto err = ret_update_size.first;
		if (err) {
		LOG(ERROR, "update_metadentry_size() failed with err '{}'", err);
		errno = err;
		return -1;
		}
		ret = gkfs::rpc::forward_write(*path, buf, append_flag, offset, count, updated_size);
		if (ret < 0) {
		LOG(WARNING, "gkfs::rpc::forward_write() failed with ret {}", ret);
		auto updated_size = ret_update_size.second;

		auto ret_write = gkfs::rpc::forward_write(*path, buf, append_flag, offset, count, updated_size);
		err = ret_write.first;
		if (err) {
		LOG(WARNING, "gkfs::rpc::forward_write() failed with err '{}'", err);
		errno = err;
		return -1;
		}
		return ret; // return written size or -1 as error
		return ret_write.second; // return written size
		}

		/**
		* gkfs wrapper for pwrite() system calls
		* errno may be set
		* @param fd
		* @param buf
		* @param count
		* @param offset
		* @return written size or -1 on error
		*/
		ssize_t gkfs_pwrite_ws(int fd, const void* buf, size_t count, off64_t offset) {
		auto file = CTX->file_map()->get(fd);
		return gkfs_pwrite(file, reinterpret_cast<const char*>(buf), count, offset);
		}

		/* Write counts bytes starting from current file position
		* It also update the file position accordingly
		*
		* Same as write syscall.
		/**
		* gkfs wrapper for write() system calls
		* errno may be set
		* @param fd
		* @param buf
		* @param count
		* @return written size or -1 on error
		*/
		ssize_t gkfs_write(int fd, const void* buf, size_t count) {
		auto gkfs_fd = CTX->file_map()->get(fd);
		@@ -465,6 +616,15 @@ ssize_t gkfs_write(int fd, const void* buf, size_t count) {
		return ret;
		}

		/**
		* gkfs wrapper for pwritev() system calls
		* errno may be set
		* @param fd
		* @param iov
		* @param iovcnt
		* @param offset
		* @return written size or -1 on error
		*/
		ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset) {

		auto file = CTX->file_map()->get(fd);
		@@ -495,6 +655,14 @@ ssize_t gkfs_pwritev(int fd, const struct iovec* iov, int iovcnt, off_t offset)
		return written;
		}

		/**
		* gkfs wrapper for writev() system calls
		* errno may be set
		* @param fd
		* @param iov
		* @param iovcnt
		* @return written size or -1 on error
		*/
		ssize_t gkfs_writev(int fd, const struct iovec* iov, int iovcnt) {

		auto gkfs_fd = CTX->file_map()->get(fd);
		@@ -508,6 +676,14 @@ ssize_t gkfs_writev(int fd, const struct iovec* iov, int iovcnt) {
		return ret;
		}

		/**
		* Wrapper function for all gkfs read operations
		* @param file
		* @param buf
		* @param count
		* @param offset
		* @return read size or -1 on error
		*/
		ssize_t gkfs_pread(std::shared_ptr<gkfs::filemap::OpenFile> file, char* buf, size_t count, off64_t offset) {
		if (file->type() != gkfs::filemap::FileType::regular) {
		assert(file->type() == gkfs::filemap::FileType::directory);
		@@ -521,13 +697,24 @@ ssize_t gkfs_pread(std::shared_ptr<gkfs::filemap::OpenFile> file, char* buf, siz
		memset(buf, 0, sizeof(char) * count);
		}
		auto ret = gkfs::rpc::forward_read(file->path(), buf, offset, count);
		if (ret < 0) {
		LOG(WARNING, "gkfs::rpc::forward_read() failed with ret {}", ret);
		auto err = ret.first;
		if (err) {
		LOG(WARNING, "gkfs::rpc::forward_read() failed with ret '{}'", err);
		errno = err;
		return -1;
		}
		// XXX check that we don't try to read past end of the file
		return ret; // return read size or -1 as error
		return ret.second; // return read size
		}

		/**
		* gkfs wrapper for read() system calls
		* errno may be set
		* @param fd
		* @param buf
		* @param count
		* @return read size or -1 on error
		*/
		ssize_t gkfs_read(int fd, void* buf, size_t count) {
		auto gkfs_fd = CTX->file_map()->get(fd);
		auto pos = gkfs_fd->pos(); //retrieve the current offset
		@@ -539,6 +726,15 @@ ssize_t gkfs_read(int fd, void* buf, size_t count) {
		return ret;
		}

		/**
		* gkfs wrapper for preadv() system calls
		* errno may be set
		* @param fd
		* @param iov
		* @param iovcnt
		* @param offset
		* @return read size or -1 on error
		*/
		ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) {

		auto file = CTX->file_map()->get(fd);
		@@ -569,6 +765,14 @@ ssize_t gkfs_preadv(int fd, const struct iovec* iov, int iovcnt, off_t offset) {
		return read;
		}

		/**
		* gkfs wrapper for readv() system calls
		* errno may be set
		* @param fd
		* @param iov
		* @param iovcnt
		* @return read size or -1 on error
		*/
		ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) {

		auto gkfs_fd = CTX->file_map()->get(fd);
		@@ -582,11 +786,26 @@ ssize_t gkfs_readv(int fd, const struct iovec* iov, int iovcnt) {
		return ret;
		}

		/**
		* gkfs wrapper for pread() system calls
		* errno may be set
		* @param fd
		* @param buf
		* @param count
		* @param offset
		* @return read size or -1 on error
		*/
		ssize_t gkfs_pread_ws(int fd, void* buf, size_t count, off64_t offset) {
		auto gkfs_fd = CTX->file_map()->get(fd);
		return gkfs_pread(gkfs_fd, reinterpret_cast<char*>(buf), count, offset);
		}

		/**
		* wrapper function for opening directories
		* errno may be set
		* @param path
		* @return 0 on success or -1 on error
		*/
		int gkfs_opendir(const std::string& path) {

		auto md = gkfs::util::get_metadata(path);
		@@ -599,11 +818,22 @@ int gkfs_opendir(const std::string& path) {
		return -1;
		}

		auto open_dir = std::make_shared<gkfs::filemap::OpenDir>(path);
		gkfs::rpc::forward_get_dirents(*open_dir);
		return CTX->file_map()->add(open_dir);
		auto ret = gkfs::rpc::forward_get_dirents(path);
		auto err = ret.first;
		if (err) {
		errno = err;
		return -1;
		}
		assert(ret.second);
		return CTX->file_map()->add(ret.second);
		}

		/**
		* gkfs wrapper for rmdir() system calls
		* errno may be set
		* @param path
		* @return 0 on success or -1 on error
		*/
		int gkfs_rmdir(const std::string& path) {
		auto md = gkfs::util::get_metadata(path);
		if (!md) {
		@@ -617,15 +847,34 @@ int gkfs_rmdir(const std::string& path) {
		return -1;
		}

		auto open_dir = std::make_shared<gkfs::filemap::OpenDir>(path);
		gkfs::rpc::forward_get_dirents(*open_dir);
		auto ret = gkfs::rpc::forward_get_dirents(path);
		auto err = ret.first;
		if (err) {
		errno = err;
		return -1;
		}
		assert(ret.second);
		auto open_dir = ret.second;
		if (open_dir->size() != 0) {
		errno = ENOTEMPTY;
		return -1;
		}
		return gkfs::rpc::forward_remove(path, true, 0);
		err = gkfs::rpc::forward_remove(path, true, 0);
		if (err) {
		errno = err;
		return -1;
		}
		return 0;
		}

		/**
		* gkfs wrapper for getdents() system calls
		* errno may be set
		* @param fd
		* @param dirp
		* @param count
		* @return 0 on success or -1 on error
		*/
		int gkfs_getdents(unsigned int fd,
		struct linux_dirent* dirp,
		unsigned int count) {
		@@ -688,7 +937,14 @@ int gkfs_getdents(unsigned int fd,
		return written;
		}


		/**
		* gkfs wrapper for getdents64() system calls
		* errno may be set
		* @param fd
		* @param dirp
		* @param count
		* @return 0 on success or -1 on error
		*/
		int gkfs_getdents64(unsigned int fd,
		struct linux_dirent64* dirp,
		unsigned int count) {
		@@ -749,6 +1005,16 @@ int gkfs_getdents64(unsigned int fd,

		#ifdef HAS_SYMLINKS

		/**
		* gkfs wrapper for make symlink() system calls
		* errno may be set
		*
		* * NOTE: Currently unused
		*
		* @param path
		* @param target_path
		* @return 0 on success or -1 on error
		*/
		int gkfs_mk_symlink(const std::string& path, const std::string& target_path) {
		gkfs::preload::init_ld_env_if_needed();
		/* The following check is not POSIX compliant.
		@@ -777,10 +1043,25 @@ int gkfs_mk_symlink(const std::string& path, const std::string& target_path) {
		errno = EEXIST;
		return -1;
		}

		return gkfs::rpc::forward_mk_symlink(path, target_path);
		auto err = gkfs::rpc::forward_mk_symlink(path, target_path);
		if (err) {
		errno = err;
		return -1;
		}
		return 0;
		}

		/**
		* gkfs wrapper for reading symlinks
		* errno may be set
		*
		* NOTE: Currently unused
		*
		* @param path
		* @param buf
		* @param bufsize
		* @return 0 on success or -1 on error
		*/
		int gkfs_readlink(const std::string& path, char* buf, int bufsize) {
		gkfs::preload::init_ld_env_if_needed();
		auto md = gkfs::util::get_metadata(path, false);

src/client/preload_util.cpp

+9 −1

Original line number	Diff line number	Diff line
		@@ -72,11 +72,18 @@ hermes::endpoint lookup_endpoint(const std::string& uri,
		namespace gkfs {
		namespace util {


		/**
		* Retrieve metadata from daemon
		* errno may be set
		* @param path
		* @param follow_links
		* @return shared_ptr for metadata, nullptr else
		*/
		std::shared_ptr<gkfs::metadata::Metadata> get_metadata(const string& path, bool follow_links) {
		std::string attr;
		auto err = gkfs::rpc::forward_stat(path, attr);
		if (err) {
		errno = err;
		return nullptr;
		}
		#ifdef HAS_SYMLINKS
		@@ -85,6 +92,7 @@ std::shared_ptr<gkfs::metadata::Metadata> get_metadata(const string& path, bool
		while (md.is_link()) {
		err = gkfs::rpc::forward_stat(md.target_path(), attr);
		if (err) {
		errno = err;
		return nullptr;
		}
		md = gkfs::metadata::Metadata{attr};

src/client/rpc/forward_data.cpp

+80 −58

File changed.

Preview size limit exceeded, changes collapsed.