fix compressed order (!281) · Merge requests · hpc / gekkofs

include/client/fuse/fuse_client.hpp

+1 −1

Changes for include/client/fuse/fuse_client.hpp: 1 added line, 1 removed line.
Original line number	Diff line number	Diff line
		@@ -36,7 +36,7 @@

		SPDX-License-Identifier: LGPL-3.0-or-later
		*/

		// clang-format off
		#ifndef GKFS_CLIENT_FUSE_CONTEXT_HPP
		#define GKFS_CLIENT_FUSE_CONTEXT_HPP

include/client/rpc/utils.hpp

+30 −4

Changes for include/client/rpc/utils.hpp: 30 added lines, 4 removed lines.
Original line number	Diff line number	Diff line
		@@ -34,6 +34,8 @@
		#include <tuple>
		#include <string>
		#include <stdexcept>
		#include <cstdint>
		#include <cstring>
		#include <type_traits>

		namespace gkfs::rpc {
		@@ -104,16 +106,40 @@ decompress_and_parse_entries(const OutputOrErr& out,
		std::vector<std::tuple<const std::string, bool, size_t, time_t>> entries;

		while(p < end) {
		bool is_dir = reinterpret_cast<const bool>(p);
		p += sizeof(bool);
		if(p + sizeof(uint8_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing is_dir",
		__func__);
		break;
		}
		bool is_dir = (reinterpret_cast<const uint8_t>(p) != 0);
		p += sizeof(uint8_t);

		if(p + sizeof(size_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing file_size",
		__func__);
		break;
		}
		size_t file_size = reinterpret_cast<const size_t>(p);
		p += sizeof(size_t);

		if(p + sizeof(time_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing ctime",
		__func__);
		break;
		}
		time_t ctime = reinterpret_cast<const time_t>(p);
		p += sizeof(time_t);

		std::string name(p);
		p += name.length() + 1;
		// Name is null-terminated. We must check we don't go past 'end'.
		size_t name_len = strnlen(p, end - p);
		if(p + name_len >= end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing name",
		__func__);
		break;
		}

		std::string name(p, name_len);
		p += name_len + 1;

		if(!name.empty()) {
		entries.emplace_back(name, is_dir, file_size, ctime);

src/client/rpc/forward_metadata.cpp

+47 −10

Changes for src/client/rpc/forward_metadata.cpp: 47 added lines, 10 removed lines.
Original line number	Diff line number	Diff line
		@@ -36,6 +36,8 @@
		#include <iterator>
		#include <algorithm>
		#include <zstd.h>
		#include <cstdint>
		#include <cstring>


		using namespace std;
		@@ -246,11 +248,24 @@ decompress_and_parse_entries_standard(
		entries.reserve(out.dirents_size); // Approx

		while(p < end) {
		bool is_dir = (*p != 0);
		p += 1;
		if(p + sizeof(uint8_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing is_dir",
		__func__);
		break;
		}
		bool is_dir = (reinterpret_cast<const uint8_t>(p) != 0);
		p += sizeof(uint8_t);

		// Name is null-terminated. We must check we don't go past 'end'.
		size_t name_len = strnlen(p, end - p);
		if(p + name_len >= end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing name",
		__func__);
		break;
		}

		std::string name(p);
		p += name.length() + 1;
		std::string name(p, name_len);
		p += name_len + 1;

		if(!name.empty()) {
		entries.emplace_back(name, is_dir, 0, 0);
		@@ -311,24 +326,46 @@ decompress_and_parse_entries_filtered(
		p = static_cast<const char*>(compressed_buffer);
		end = p + out.dirents_size;
		}

		std::vector<std::tuple<const std::string, bool, size_t, time_t>> entries;
		// We don't know exact count, but can optimize reserve?
		// entries.reserve(out.dirents_size / 32);

		while(p < end) {
		bool is_dir = (*p != 0);
		p += 1;

		std::string name(p);
		p += name.length() + 1;
		if(p + sizeof(uint8_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing is_dir",
		__func__);
		break;
		}
		bool is_dir = (reinterpret_cast<const uint8_t>(p) != 0);
		p += sizeof(uint8_t);

		if(p + sizeof(size_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing size",
		__func__);
		break;
		}
		size_t size = reinterpret_cast<const size_t>(p);
		p += sizeof(size_t);

		if(p + sizeof(time_t) > end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing ctime",
		__func__);
		break;
		}
		time_t ctime = reinterpret_cast<const time_t>(p);
		p += sizeof(time_t);

		// Name is null-terminated. We must check we don't go past 'end'.
		size_t name_len = strnlen(p, end - p);
		if(p + name_len >= end) {
		LOG(ERROR, "{}() Unexpected end of buffer while parsing name",
		__func__);
		break;
		}

		std::string name(p, name_len);
		p += name_len + 1;

		if(!name.empty()) {
		entries.emplace_back(name, is_dir, size, ctime);
		}

src/daemon/backend/metadata/rocksdb_backend.cpp

+2 −2

Changes for src/daemon/backend/metadata/rocksdb_backend.cpp: 2 added lines, 2 removed lines.
Original line number	Diff line number	Diff line
		@@ -580,8 +580,6 @@ RocksDBBackend::get_dirents_filtered_impl(const std::string& dir,
		eof = false;
		break;
		}
		scanned_count++;

		// Get File name
		auto name = it->key().ToString();
		// save as potential last key (full path for now, stripped later if
		@@ -609,6 +607,8 @@ RocksDBBackend::get_dirents_filtered_impl(const std::string& dir,
		continue;
		}

		scanned_count++;

		Metadata md(it->value().ToString());
		#ifdef HAS_RENAME
		// Remove entries with negative blocks (rename)

tests/integration/directories/test_packing_order.py

0 → 100644

+119 −0

Changes for tests/integration/directories/test_packing_order.py: 119 added lines, 0 removed lines.
Original line number	Diff line number	Diff line

		import pytest
		import logging
		import time
		from harness.gkfs import Daemon, ShellClient, Client, find_command

		log = logging.getLogger(__name__)

		def test_packing_order_all_fields(test_workspace, request):
		"""
		Comprehensive test to verify the packing order of compressed directory entries.
		It checks filtering by name, size, and ctime (newer).
		"""
		# Start Daemon with compression ON
		daemon_env = {
		"GKFS_DAEMON_LOG_LEVEL": "info",
		"GKFS_DAEMON_USE_DIRENTS_COMPRESSION": "ON"
		}
		daemon = Daemon(request.config.getoption('--interface'), "rocksdb", test_workspace, env=daemon_env)
		daemon.run()

		try:
		client = ShellClient(test_workspace)
		mount_dir = test_workspace.mountdir
		client_env = {"LIBGKFS_USE_DIRENTS_COMPRESSION": "ON"}

		# 1. Create files with specific properties
		# file_a: size 100
		# file_b: size 200
		# (wait)
		# file_c: size 100

		file_a = mount_dir / "file_a"
		file_b = mount_dir / "file_b"
		file_c = mount_dir / "file_c"

		# Use dd to create files with specific sizes
		client.run("dd", "if=/dev/zero", f"of={file_a}", "bs=100", "count=1", env=client_env)
		client.run("dd", "if=/dev/zero", f"of={file_b}", "bs=200", "count=1", env=client_env)

		log.info("Waiting 2 seconds to distinguish ctimes...")
		time.sleep(2)
		log.info("Resuming test after sleep.")

		# Create a timestamp file for 'newer' check
		timestamp_file = test_workspace.twd / "timestamp"
		timestamp_file.touch()

		time.sleep(1)
		client.run("dd", "if=/dev/zero", f"of={file_c}", "bs=100", "count=1", env=client_env)

		sfind_bin = find_command("sfind", test_workspace.bindirs)
		assert sfind_bin, "sfind binary not found"

		common_args = ["-M", str(mount_dir), "-S", "1", "--server-side"]

		import re
		def check_match(ret, expected_matches, expected_total=3):
		output = ret.stdout.decode()
		log.info(f"SFIND OUTPUT:\n{output}")
		match = re.search(r"MATCHED (\d+)/(\d+)", output)
		if not match:
		assert False, f"Could not find MATCHED line in output:\n{output}"

		actual_matches = int(match.group(1))
		total_checked = int(match.group(2))

		log.info(f"Actual matches: {actual_matches}, Total checked: {total_checked}")
		assert actual_matches == expected_matches, f"Expected {expected_matches} matches, but found {actual_matches}. Full output:\n{output}"
		assert total_checked == expected_total, f"Expected {expected_total} total checked, but found {total_checked}. Full output:\n{output}"

		# Scenario 1: Filter by Name
		log.info("Testing -name filter...")
		ret = client.run(str(sfind_bin), str(mount_dir), "-name", "file_a", *common_args, env=client_env)
		assert ret.exit_code == 0
		check_match(ret, 1)

		# Scenario 2: Filter by Size (100c)
		log.info("Testing -size filter...")
		# Should match file_a and file_c (both 100 bytes)
		ret = client.run(str(sfind_bin), str(mount_dir), "-size", "100c", *common_args, env=client_env)
		assert ret.exit_code == 0
		check_match(ret, 2)

		# Scenario 3: Filter by Size (200c)
		log.info("Testing -size filter (200c)...")
		# Should match file_b
		ret = client.run(str(sfind_bin), str(mount_dir), "-size", "200c", *common_args, env=client_env)
		assert ret.exit_code == 0
		check_match(ret, 1)

		# Scenario 4: Filter by Newer (ctime)
		log.info("Testing -newer filter...")
		# Should match file_c (created after timestamp_file)
		ret = client.run(str(sfind_bin), str(mount_dir), "-newer", str(timestamp_file), *common_args, env=client_env)
		assert ret.exit_code == 0
		check_match(ret, 1)

		# Scenario 5: Combined Filter (Name file AND Size 100c AND Newer)
		log.info("Testing combined filter...")
		# Should match only file_c
		ret = client.run(str(sfind_bin), str(mount_dir), "-name", "file", "-size", "100c", "-newer", str(timestamp_file), *common_args, env=client_env)
		assert ret.exit_code == 0
		check_match(ret, 1)

		log.info("Packing order verification successful!")

		finally:
		# Dump daemon log to inspect scanned keys and ctime
		# daemon_log = test_workspace.logdir / "daemon.log"
		# if daemon_log.exists():
		## log.info("DUMPING DAEMON LOG DEBUG ENTRIES:")
		# with open(daemon_log, 'r') as f:
		# for line in f:
		# if "DEBUG" in line:
		# print(line.strip())
		#else:
		# log.warning(f"Daemon log not found at {daemon_log}")
		daemon.shutdown()

fix compressed order

Merge request reports