Adding config for async opendir (4258d135) · Commits · hpc / gekkofs

include/config.hpp

+1 −0

Original line number	Diff line number	Diff line
		@@ -137,6 +137,7 @@ namespace rpc {
		constexpr auto chunksize = 524288; // in bytes (e.g., 524288 == 512KB)
		// size of preallocated buffer to hold directory entries in rpc call
		constexpr auto dirents_buff_size = (8 * 1024 * 1024); // 8 mega
		constexpr auto async_opendir = true;
		/*
		* Indicates the number of concurrent progress to drive I/O operations of chunk
		* files to and from local file systems The value is directly mapped to created

src/client/gkfs_functions.cpp

+61 −62

Original line number	Diff line number	Diff line
		@@ -1311,7 +1311,7 @@ gkfs_opendir(const std::string& path) {
		// this is used in get_metadata() later to avoid stat RPCs
		if(CTX->use_cache()) {
		ret.second = make_shared<gkfs::filemap::OpenDir>(path);

		if constexpr(gkfs::config::rpc::async_opendir) {
		std::vector<std::future<
		pair<int, unique_ptr<vector<tuple<const basic_string<char>,
		bool, size_t, time_t>>>>>>
		@@ -1321,7 +1321,8 @@ gkfs_opendir(const std::string& path) {
		futures.push_back(std::async(std::launch::async, [&, i]() {
		if(gkfs::config::proxy::fwd_get_dirents_single &&
		CTX->use_proxy()) {
		return gkfs::rpc::forward_get_dirents_single_proxy(path, i);
		return gkfs::rpc::forward_get_dirents_single_proxy(path,
		i);
		} else {
		return gkfs::rpc::forward_get_dirents_single(path, i);
		}
		@@ -1336,50 +1337,48 @@ gkfs_opendir(const std::string& path) {
		LOG(DEBUG, "name: {} type: {} size: {} ctime: {}",
		get<0>(dentry), get<1>(dentry), get<2>(dentry),
		get<3>(dentry));
		auto ftype = get<1>(dentry) ? gkfs::filemap::FileType::directory
		auto ftype = get<1>(dentry)
		? gkfs::filemap::FileType::directory
		: gkfs::filemap::FileType::regular;
		// filename, is_dir, size, ctime
		ret.second->add(get<0>(dentry), ftype);
		CTX->cache()->insert(path, get<0>(dentry),
		gkfs::cache::cache_entry{ftype,
		get<2>(dentry),
		CTX->cache()->insert(
		path, get<0>(dentry),
		gkfs::cache::cache_entry{ftype, get<2>(dentry),
		get<3>(dentry)});
		}
		ret.first = res.first;
		}
		// TODO parallelize
		// for(uint64_t i = 0; i < CTX->hosts().size(); i++) {
		// pair<int, unique_ptr<vector<tuple<const
		// basic_string<char>, bool,
		// size_t, time_t>>>>
		// res{};
		// if(gkfs::config::proxy::fwd_get_dirents_single &&
		// CTX->use_proxy()) {
		// res =
		// gkfs::rpc::forward_get_dirents_single_proxy(path, i);
		// } else {
		// res = gkfs::rpc::forward_get_dirents_single(path, i);
		// }
		// auto& open_dir = *res.second;
		// for(auto& dentry : open_dir) {
		// // type returns as a boolean. true if it is a
		// directory LOG(DEBUG, "name: {} type: {} size: {}
		// ctime: {}",
		// get<0>(dentry), get<1>(dentry), get<2>(dentry),
		// get<3>(dentry));
		// auto ftype = get<1>(dentry) ?
		// gkfs::filemap::FileType::directory
		// :
		// gkfs::filemap::FileType::regular;
		// // filename, is_dir, size, ctime
		// ret.second->add(get<0>(dentry), ftype);
		// CTX->cache()->insert(path, get<0>(dentry),
		// gkfs::cache::cache_entry{ftype,
		// get<2>(dentry),
		// get<3>(dentry)});
		// }
		// ret.first = res.first;
		// }
		} else {
		for(uint64_t i = 0; i < CTX->hosts().size(); i++) {
		pair<int, unique_ptr<vector<tuple<const basic_string<char>,
		bool, size_t, time_t>>>>
		res{};
		if(gkfs::config::proxy::fwd_get_dirents_single &&
		CTX->use_proxy()) {
		res = gkfs::rpc::forward_get_dirents_single_proxy(path, i);
		} else {
		res = gkfs::rpc::forward_get_dirents_single(path, i);
		}
		auto& open_dir = *res.second;
		for(auto& dentry : open_dir) {
		// LOG(DEBUG, "name: {} type: {} size: {}
		// ctime {} ",
		// get<0>(dentry), get<1>(dentry),
		// get<2>(dentry), get<3>(dentry));
		auto ftype = get<1>(dentry)
		? gkfs::filemap::FileType::directory
		: gkfs::filemap::FileType::regular;
		// filename, is_dir, size, ctime
		ret.second->add(get<0>(dentry), ftype);
		CTX->cache()->insert(
		path, get<0>(dentry),
		gkfs::cache::cache_entry{ftype, get<2>(dentry),
		get<3>(dentry)});
		}
		ret.first = res.first;
		}
		}
		// CTX->cache()->dump_cache_to_log(path);
		} else {
		ret = gkfs::rpc::forward_get_dirents(path);