Adding implicit_data_removal config for optimized remove operations (0d52833e) · Commits · David Auer / gekkofs

include/config.hpp

+7 −0

Original line number	Original line	Diff line number	Diff line
	@@ -48,6 +48,13 @@ constexpr auto use_ctime = false;
	constexpr auto use_mtime = false;		constexpr auto use_mtime = false;
	constexpr auto use_link_cnt = false;		constexpr auto use_link_cnt = false;
	constexpr auto use_blocks = false;		constexpr auto use_blocks = false;
			/*
			* If true, all chunks on the same host are removed during a metadata remove
			* rpc. This is a technical optimization that reduces the number of RPCs for
			* remove operations. This setting could be useful for future asynchronous
			* remove implementations where the data should not be removed immediately.
			*/
			constexpr auto implicit_data_removal = true;

	// metadata logic		// metadata logic
	// Check for existence of file metadata before create. This done on RocksDB		// Check for existence of file metadata before create. This done on RocksDB

+9 −6

Original line number	Original line	Diff line number	Diff line
	@@ -169,12 +169,15 @@ forward_remove(const std::string& path) {
	for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) {		for(uint64_t chnk_id = chnk_start; chnk_id <= chnk_end; chnk_id++) {
	const auto chnk_host_id =		const auto chnk_host_id =
	CTX->distributor()->locate_data(path, chnk_id);		CTX->distributor()->locate_data(path, chnk_id);
			if constexpr(gkfs::config::metadata::implicit_data_removal) {
	/*		/*
	* If the chnk host matches the metadata host the remove request		* If the chnk host matches the metadata host the remove
	* as already been sent as part of the metadata remove request.		* request as already been sent as part of the metadata
			* remove request.
	*/		*/
	if(chnk_host_id == metadata_host_id)		if(chnk_host_id == metadata_host_id)
	continue;		continue;
			}
	const auto endp_chnk = CTX->hosts().at(chnk_host_id);		const auto endp_chnk = CTX->hosts().at(chnk_host_id);

	LOG(DEBUG, "Sending RPC to host: {}", endp_chnk.to_string());		LOG(DEBUG, "Sending RPC to host: {}", endp_chnk.to_string());

+2 −2

Original line number	Original line	Diff line number	Diff line
	@@ -108,8 +108,8 @@ ChunkStorage::destroy_chunk_space(const string& file_path) const {
	try {		try {
	// Note: remove_all does not throw an error when path doesn't exist.		// Note: remove_all does not throw an error when path doesn't exist.
	auto n = fs::remove_all(chunk_dir);		auto n = fs::remove_all(chunk_dir);
	log_->debug("{}() Removed '{}' files from '{}'", __func__, n,		log_->debug("{}() Removed '{}' files and directories from '{}'",
	chunk_dir);		__func__, n, chunk_dir);
	} catch(const fs::filesystem_error& e) {		} catch(const fs::filesystem_error& e) {
	auto err_str = fmt::format(		auto err_str = fmt::format(
	"{}() Failed to remove chunk directory. Path: '{}', Error: '{}'",		"{}() Failed to remove chunk directory. Path: '{}', Error: '{}'",

+9 −0

Original line number	Original line	Diff line number	Diff line
	@@ -167,10 +167,19 @@ rpc_srv_remove_metadata(hg_handle_t handle) {
	out.err = 0;		out.err = 0;
	out.mode = md.mode();		out.mode = md.mode();
	out.size = md.size();		out.size = md.size();
			if constexpr(gkfs::config::metadata::implicit_data_removal) {
			if(S_ISREG(md.mode()) && (md.size() != 0))
			GKFS_DATA->storage()->destroy_chunk_space(in.path);
			}
	} catch(const gkfs::metadata::DBException& e) {		} catch(const gkfs::metadata::DBException& e) {
	GKFS_DATA->spdlogger()->error("{}(): path '{}' message '{}'", __func__,		GKFS_DATA->spdlogger()->error("{}(): path '{}' message '{}'", __func__,
	in.path, e.what());		in.path, e.what());
	out.err = EIO;		out.err = EIO;
			} catch(const gkfs::data::ChunkStorageException& e) {
			GKFS_DATA->spdlogger()->error(
			"{}(): path '{}' errcode '{}' message '{}'", __func__, in.path,
			e.code().value(), e.what());
			out.err = e.code().value();
	} catch(const std::exception& e) {		} catch(const std::exception& e) {
	GKFS_DATA->spdlogger()->error("{}() path '{}' message '{}'", __func__,		GKFS_DATA->spdlogger()->error("{}() path '{}' message '{}'", __func__,
	in.path, e.what());		in.path, e.what());