Restore RPC consistency checker (920cb49a) · Commits · hpc / gekkofs

rpc-consistency-checker/src/checker.py

0 → 100755

+585 −0

Changes for rpc-consistency-checker/src/checker.py: 585 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		"""RPC Consistency Checker for GekkoFS

		Cross-references RPC chain across 7+ files to detect:
		[ORPHAN] Tag without handler registration (or vice versa)
		[MISSING] Handler references undefined input/output struct
		[WARN] Missing forward declaration or unused struct
		[LEGACY] Deprecated RPC alias (expand_* -> mutate_*)
		[MISMATCH] Handler registered but implementation not found
		"""
		from __future__ import annotations

		import argparse
		import os
		import re
		import sys
		from pathlib import Path
		from dataclasses import dataclass, field
		from typing import List, Dict, Set

		@dataclass
		class RPCTag:
		name: str
		value: str
		file: str
		namespace: str

		@dataclass
		class StructDef:
		name: str
		kind: str # 'in' or 'out'
		file: str

		@dataclass
		class HandlerDecl:
		name: str
		input_struct: str # e.g. 'rpc_mk_node_in_t' (no gkfs::rpc:: prefix)
		has_engine: bool
		file: str

		@dataclass
		class HandlerReg:
		tag_name: str
		handler_name: str
		input_struct: str # e.g. 'rpc_mk_node_in_t'
		file: str
		category: str # 'normal', 'malleable', 'proxy'

		@dataclass
		class ForwardDecl:
		func_name: str
		input_struct: str
		file: str

		@dataclass
		class Report:
		severity: str
		message: str
		files: List[str] = field(default_factory=list)


		def scan_project(project_root: str) -> RPCScanner:
		"""Scan one project and return the populated consistency scanner."""
		scanner = RPCScanner(os.path.abspath(project_root))
		scanner.scan_tags()
		scanner.scan_structs()
		scanner.scan_handler_decls()
		scanner.scan_handler_regs()
		scanner.scan_handler_impls()
		scanner.scan_forwards()
		scanner.check_consistency()
		return scanner


		def has_critical_issues(scanner: RPCScanner) -> bool:
		"""Return whether the scan contains an issue that must fail CI."""
		return any(
		issue.severity in ("ORPHAN", "MISSING", "MISMATCH")
		for issue in scanner.errors
		)


		class RPCScanner:
		def __init__(self, project_root: str):
		self.project_root = Path(project_root)
		self.include_dir = self.project_root / "include"
		self.src_dir = self.project_root / "src"
		self.tags: List[RPCTag] = []
		self.structs: List[StructDef] = []
		self.handler_decls: List[HandlerDecl] = []
		self.handler_regs: List[HandlerReg] = []
		self.forward_decls: List[ForwardDecl] = []
		self.handler_impls: Dict[str, str] = {} # name -> file
		self.errors: List[Report] = []

		# ── helpers ──

		def _read(self, path: Path) -> str:
		return path.read_text() if path.exists() else ""

		@staticmethod
		def _strip_rpc_ns(name: str) -> str:
		"""Strip 'gkfs::rpc::' prefix from struct names."""
		for prefix in ["gkfs::rpc::", "gkfs::rpc:"]:
		if name.startswith(prefix):
		return name[len(prefix):]
		return name

		# ── scans ──

		def scan_tags(self):
		"""Parse common_defs.hpp for constexpr auto <name> = \"<value>\""""
		fpath = self.include_dir / "common" / "common_defs.hpp"
		if not fpath.exists():
		self.errors.append(Report("WARN", f"common_defs.hpp not found", [str(fpath)]))
		return
		content = self._read(fpath)
		pattern = r'constexpr\s+auto\s+(\w+)\s=\s"([^"]+)";'
		in_tag_ns = False
		in_mutate_ns = False
		for line in content.splitlines():
		stripped = line.strip()
		if 'namespace malleable::rpc::tag' in stripped:
		in_mutate_ns = True
		in_tag_ns = False
		continue
		if in_mutate_ns and stripped == '}':
		in_mutate_ns = False
		continue
		if 'namespace rpc::tag' in stripped:
		in_tag_ns = True
		in_mutate_ns = False
		continue
		if in_tag_ns and stripped == '}':
		in_tag_ns = False
		continue
		m = re.search(pattern, stripped)
		if m and (in_tag_ns or in_mutate_ns):
		ns = "gkfs::malleable::rpc::tag" if in_mutate_ns else "gkfs::rpc::tag"
		self.tags.append(RPCTag(m.group(1), m.group(2), str(fpath), ns))

		def scan_structs(self):
		"""Parse rpc_types_thallium.hpp for struct rpc__in_t / rpc__out_t.
		Also scan common_defs.hpp (inside gkfs::rpc namespace) for RPC structs.
		"""
		for fpath in [
		self.include_dir / "common" / "rpc" / "rpc_types_thallium.hpp",
		self.include_dir / "common" / "common_defs.hpp",
		]:
		if not fpath.exists():
		self.errors.append(Report("WARN", f"{fpath.name} not found", [str(fpath)]))
		continue
		content = self._read(fpath)
		is_common_defs = fpath.name == "common_defs.hpp"

		if is_common_defs:
		# Only extract structs inside namespace gkfs::rpc (before "} // namespace rpc")
		in_rpc_ns = False
		for line in content.splitlines():
		stripped = line.strip()
		if 'namespace rpc {' in stripped:
		in_rpc_ns = True
		continue
		if in_rpc_ns and stripped == '} // namespace rpc':
		in_rpc_ns = False
		continue
		if not in_rpc_ns:
		continue

		pattern = r'struct\s+(rpc_\w+_(?:in\|out)_t)\s*\{'
		for line in content.splitlines():
		m = re.search(pattern, line)
		if m:
		if is_common_defs:
		# Skip if outside namespace rpc
		continue # already filtered above by in_rpc_ns
		name = m.group(1)
		kind = "in" if "_in_t" in name else "out"
		if not any(s.name == name for s in self.structs):
		self.structs.append(StructDef(name, kind, str(fpath)))

		# Second pass: common_defs.hpp structs need to be collected properly
		fpath = self.include_dir / "common" / "common_defs.hpp"
		if fpath.exists():
		content = self._read(fpath)
		in_rpc_ns = False
		for line in content.splitlines():
		stripped = line.strip()
		if 'namespace rpc {' in stripped:
		in_rpc_ns = True
		continue
		if in_rpc_ns and stripped == '} // namespace rpc':
		in_rpc_ns = False
		continue
		if not in_rpc_ns:
		continue
		m = re.search(r'struct\s+(rpc_\w+_(?:in\|out)_t)\s*\{', stripped)
		if m:
		name = m.group(1)
		kind = "in" if "_in_t" in name else "out"
		if not any(s.name == name for s in self.structs):
		self.structs.append(StructDef(name, kind, str(fpath)))

		def scan_handler_decls(self):
		"""Parse daemon/proxy rpc_defs.hpp for handler declarations.

		Handles multi-line declarations like:
		void\\n rpc_srv_create(const tl::request& req, const gkfs::rpc::rpc_mk_node_in_t& in);

		IMPORTANT: str input_struct stores ONLY 'rpc_mk_node_in_t' (no gkfs::rpc:: prefix)
		"""
		for subdir in ["daemon/handler", "proxy/rpc"]:
		fpath = self.include_dir / subdir / "rpc_defs.hpp"
		if not fpath.exists():
		continue
		content = self._read(fpath)
		if not content:
		continue
		lines = content.splitlines()
		combined = ' '.join(lines)

		# Pattern 1: declarations with a struct argument
		pattern = r'(?:void\s+)(rpc_srv_\w+\|proxy_rpc_srv_\w+)\(([^)](?:gkfs::rpc::rpc_\w+_(?:in\|out)_t)[^)])\);'
		for m in re.finditer(pattern, combined):
		name = m.group(1)
		args = m.group(2)
		struct_m = re.search(r'gkfs::rpc::rpc_(\w+_(?:in\|out)_t)', args)
		# Store WITHOUT gkfs::rpc:: prefix
		input_struct = struct_m.group(0)[len('gkfs::rpc::'):] if struct_m else ""

		# Skip commented-out declarations
		is_commented = False
		for line in lines:
		stripped = line.strip()
		if name in stripped and not stripped.startswith('//'):
		break
		if name in stripped and stripped.startswith('//'):
		is_commented = True
		if is_commented:
		continue

		has_engine = 'engine' in args
		self.handler_decls.append(HandlerDecl(name, input_struct, has_engine, str(fpath)))

		# Pattern 2: zero-arg handlers like mutate_status(const tl::request& req);
		pattern2 = r'(?:void\s+)(rpc_srv_\w+)\(\s(?:const\s+tl::request&\sreq)\s\)\s;'
		for m in re.finditer(pattern2, combined):
		name = m.group(1)
		if not any(d.name == name for d in self.handler_decls):
		# Check if commented
		is_commented = False
		for line in lines:
		stripped = line.strip()
		if name in stripped and not stripped.startswith('//'):
		break
		if name in stripped and stripped.startswith('//'):
		is_commented = True
		if not is_commented:
		self.handler_decls.append(HandlerDecl(name, "", False, str(fpath)))

		def _find_handler_input(self, handler_name: str) -> str:
		for d in self.handler_decls:
		if d.name == handler_name:
		return d.input_struct
		return ""

		def scan_handler_regs(self):
		"""Parse daemon.cpp and proxy.cpp for engine->define(tag, handler) lines.

		Handles both simple and lambda-style defines.
		input_struct stored WITHOUT gkfs::rpc:: prefix.
		"""
		for reg_file in ["daemon/daemon.cpp", "proxy/proxy.cpp"]:
		fpath = self.src_dir / reg_file
		if not fpath.exists():
		continue
		content = self._read(fpath)
		if not content:
		continue
		combined = ' '.join(content.splitlines())

		# Pattern 1: Simple define -> engine.define/->define(tag::name, handler_name);
		for m in re.finditer(
		r'engine\s(?:->\|\.)\sdefine\s\(\s(?:gkfs::malleable::rpc::tag::\|gkfs::rpc::tag::)(\w+),\s(\w+)\s\)',
		combined
		):
		tag_name = m.group(1)
		handler_name = m.group(2)
		category = "malleable" if 'malleable' in combined[m.start():m.end()+100] else (
		"proxy" if 'proxy' in reg_file else "normal"
		)
		input_struct = self._find_handler_input(handler_name)
		self.handler_regs.append(HandlerReg(tag_name, handler_name, input_struct, str(fpath), category))

		# Pattern 2: Lambda define -> engine.define(tag::name, [lambda](...) { rpc_srv_HANDLER(engine, req, in); });
		for m in re.finditer(
		r'engine\s(?:->\|\.)\sdefine\s\(\s(?:gkfs::malleable::rpc::tag::\|gkfs::rpc::tag::)(\w+),\s\[?\w+\]?\s\(.?\)\s\{[^}]*?(rpc_srv_\w+)\(',
		combined
		):
		tag_name = m.group(1)
		handler_name = m.group(2)
		input_struct = self._find_handler_input(handler_name)
		# Check if already found
		if not any(r.tag_name == tag_name and r.handler_name == handler_name for r in self.handler_regs):
		self.handler_regs.append(HandlerReg(tag_name, handler_name, input_struct, str(fpath), "proxy" if 'proxy' in reg_file else "normal"))

		def scan_handler_impls(self):
		"""Parse srv_.cpp and proxy_.cpp files for handler implementations."""
		# Daemon handlers: src/daemon/handler/.cpp, src/daemon/backend/.cpp
		for d in ["daemon/handler", "daemon/backend"]:
		base = self.src_dir / d
		if not base.exists():
		continue
		for cpp_file in base.glob("*.cpp"):
		content = self._read(cpp_file)
		if not content:
		continue
		combined = ' '.join(content.splitlines())
		for m in re.finditer(r'void\s+(rpc_srv_\w+)\s*\(', combined):
		name = m.group(1)
		if name not in self.handler_impls:
		self.handler_impls[name] = str(cpp_file)

		# Proxy handlers: src/proxy/rpc/*.cpp (recursive)
		proxy_base = self.src_dir / "proxy"
		if proxy_base.exists():
		for cpp_file in proxy_base.rglob("*.cpp"):
		content = self._read(cpp_file)
		if not content:
		continue
		combined = ' '.join(content.splitlines())
		for m in re.finditer(r'void\s+(proxy_rpc_srv_\w+)\s*\(', combined):
		name = m.group(1)
		if name not in self.handler_impls:
		self.handler_impls[name] = str(cpp_file)
		for m in re.finditer(r'void\s+(rpc_srv_\w+)\s*\(', combined):
		name = m.group(1)
		if name not in self.handler_impls:
		self.handler_impls[name] = str(cpp_file)

		def scan_forwards(self):
		"""Parse forward_.hpp files for forward_ function declarations."""
		for fpath in self.include_dir.rglob("forward_*.hpp"):
		content = self._read(fpath)
		if not content:
		continue
		combined = ' '.join(content.splitlines())
		# Simple pattern: just find forward_* followed by ( to avoid nested template issues
		for m in re.finditer(r'\b(forward_\w+)\s*\(', combined):
		func_name = m.group(1)
		self.forward_decls.append(ForwardDecl(func_name, "", str(fpath)))

		# ── checks ──

		def check_consistency(self):
		"""Run all consistency checks."""
		self._check_orphan_tags()
		self._check_orphan_impls()
		self._check_orphan_handlers()
		self._check_missing_structs()
		self._check_unregistered_handlers()
		self._check_forward_orphans()
		self._check_legacy_rpc()

		def _check_orphan_tags(self):
		"""Tags defined but not registered in engine->define()."""
		reg_tags = {r.tag_name for r in self.handler_regs}
		for tag in self.tags:
		if tag.name not in reg_tags:
		self.errors.append(Report(
		"ORPHAN",
		f"Tag '{tag.value}' ({tag.name}) defined in {os.path.basename(tag.file)}"
		f" but NOT registered in engine->define()",
		[tag.file]
		))

		def _check_orphan_impls(self):
		"""Declared handlers without implementations."""
		impl_names = set(self.handler_impls.keys())
		for decl in self.handler_decls:
		if decl.name not in impl_names:
		self.errors.append(Report(
		"ORPHAN",
		f"Handler '{decl.name}' declared in {os.path.basename(decl.file)}"
		f" but no implementation found in src/",
		[decl.file]
		))

		def _check_orphan_handlers(self):
		"""Implemented handlers without declarations."""
		decl_names = {d.name for d in self.handler_decls}
		for impl_name, impl_file in self.handler_impls.items():
		if impl_name not in decl_names:
		self.errors.append(Report(
		"ORPHAN",
		f"Handler '{impl_name}' implemented in {os.path.basename(impl_file)}"
		f" but not declared in include/",
		[impl_file]
		))

		def _check_missing_structs(self):
		"""Handler references struct not defined in rpc_types_thallium.hpp."""
		struct_names = {s.name for s in self.structs}

		# Check handler decls
		for decl in self.handler_decls:
		if decl.input_struct:
		check_name = self._strip_rpc_ns(decl.input_struct)
		if check_name not in struct_names:
		self.errors.append(Report(
		"MISSING",
		f"Handler '{decl.name}' references struct '{check_name}'"
		f" NOT defined in rpc_types_thallium.hpp",
		[decl.file]
		))

		# Check registrations
		for reg in self.handler_regs:
		if reg.input_struct:
		check_name = self._strip_rpc_ns(reg.input_struct)
		if check_name not in struct_names:
		self.errors.append(Report(
		"MISSING",
		f"Registration of '{reg.tag_name}' uses struct '{check_name}' not found",
		[reg.file]
		))

		# Track used structs for unused warning
		used_structs = set()
		for d in self.handler_decls:
		if d.input_struct:
		used_structs.add(self._strip_rpc_ns(d.input_struct))
		for r in self.handler_regs:
		if r.input_struct:
		used_structs.add(self._strip_rpc_ns(r.input_struct))

		# Also scan all source/include files for struct references (catches commented-out declarations)
		for search_root in [self.src_dir, self.include_dir]:
		if not search_root.exists():
		continue
		for fpath in search_root.rglob("."):
		if fpath.suffix not in ('.cpp', '.hpp', '.h', '.c', '.cc'):
		continue
		try:
		content = self._read(fpath)
		if not content:
		continue
		# Scan ALL structs in this file — removing previous break so we catch
		# multiple structs that may coexist in the same file
		for struct in self.structs:
		if struct.name in content and struct.name not in used_structs:
		used_structs.add(struct.name)
		except Exception:
		pass

		# Flag unused input structs
		for s in self.structs:
		if s.kind == "in" and s.name not in used_structs:
		self.errors.append(Report(
		"WARN",
		f"Struct '{s.name}' in {os.path.basename(s.file)} appears unused",
		[s.file]
		))

		def _check_unregistered_handlers(self):
		"""Declared handlers not registered in engine->define()."""
		reg_handlers = {r.handler_name for r in self.handler_regs}
		for decl in self.handler_decls:
		if decl.name not in reg_handlers:
		self.errors.append(Report(
		"WARN",
		f"Handler '{decl.name}' declared but NOT registered in engine->define()",
		[decl.file]
		))

		def _check_forward_orphans(self):
		"""Registered RPCs without matching client-side forward declaration."""
		forward_names = {f.func_name for f in self.forward_decls}
		for reg in self.handler_regs:
		base = reg.handler_name
		if base.startswith("proxy_rpc_srv_"):
		base = base.replace("proxy_rpc_srv_", "forward_", 1) + "_proxy"
		elif base.startswith("rpc_srv_"):
		base = "forward_" + base.replace("rpc_srv_", "", 1)
		else:
		continue # skip non-handler registrations
		if base not in forward_names and reg.category == "normal":
		# Check if there's an alternative forward with a similar name
		handler_base = base.replace("forward_", "")
		# e.g. forward_remove_metadata -> look for forward_remove
		alt_base = None
		for fn in forward_names:
		if fn.startswith("forward_") and handler_base.startswith(fn.replace("forward_", "")):
		alt_base = fn
		break
		if alt_base:
		continue # Alternative forward exists
		self.errors.append(Report(
		"WARN",
		f"RPC '{reg.tag_name}' -> '{reg.handler_name}' has no client-side forward '{base}'",
		[reg.file]
		))

		def _check_legacy_rpc(self):
		"""Flag deprecated expand_* RPCs."""
		legacy_names = {"expand_start", "expand_status", "expand_finalize"}
		reg_tags = {r.tag_name for r in self.handler_regs}
		for tag in self.tags:
		if tag.name in legacy_names and tag.name in reg_tags:
		self.errors.append(Report(
		"LEGACY",
		f"Tag '{tag.value}' ({tag.name}) is legacy/alias (forwarded to mutate_*)",
		[tag.file]
		))


		def print_report(scan: RPCScanner):
		print("=" * 78)
		print(" GekkoFS RPC Consistency Checker")
		print("=" * 78)
		print(f"\n{'='*78}")
		print(" SUMMARY")
		print(f"{'='*78}")
		counts = {}
		for e in scan.errors:
		counts[e.severity] = counts.get(e.severity, 0) + 1

		print(f" Tags defined: {len(scan.tags)}")
		print(f" Structs defined: {len(scan.structs)}")
		print(f" Handler declarations: {len(scan.handler_decls)}")
		print(f" Handler registrations: {len(scan.handler_regs)}")
		print(f" Forward declarations: {len(scan.forward_decls)}")
		print(f" Handler implementations:{len(scan.handler_impls)}")

		print(f"\n Issues by category:")
		for sev in ["ORPHAN", "MISSING", "MISMATCH", "WARN", "LEGACY"]:
		count = counts.get(sev, 0)
		if count > 0:
		print(f" {sev:<8}: {count}")
		if not counts:
		print(" None! All RPCs are consistent.")

		print(f"\n{'='*78}")
		print(" DETAILED ISSUES")
		print(f"{'='*78}")

		if not scan.errors:
		print(" No issues found.")
		else:
		for e in scan.errors:
		files_str = ", ".join(os.path.basename(f) for f in e.files) if e.files else "N/A"
		marker = f"[{e.severity:<7}]"
		print(f"\n {marker} {e.message}")
		if files_str:
		print(f" Files: {files_str}")

		print(f"\n{'='*78}")


		def main():
		parser = argparse.ArgumentParser(
		description="Check GekkoFS RPC tags, structs, handlers, and forwards."
		)
		parser.add_argument(
		"project_root",
		nargs="?",
		default=os.getcwd(),
		help="GekkoFS project root (default: current directory)",
		)
		args = parser.parse_args()
		project_root = args.project_root
		project_root = os.path.abspath(project_root)
		if not os.path.isdir(project_root):
		print(f"Error: {project_root} is not a directory", file=sys.stderr)
		sys.exit(1)

		print(f"Scanning project: {project_root}\n")

		scanner = scan_project(project_root)
		print_report(scanner)

		sys.exit(1 if has_critical_issues(scanner) else 0)

		if __name__ == "__main__":
		main()

rpc-consistency-checker/src/tools/server.py

0 → 100644

+228 −0

File added.

Preview size limit exceeded, changes collapsed.

rpc-consistency-checker/test_checker.py

0 → 100644

+53 −0

File added.

Preview size limit exceeded, changes collapsed.

scripts/check_rpc_consistency.py

0 → 100644

+40 −0

Changes for scripts/check_rpc_consistency.py: 40 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		#!/usr/bin/env python3
		"""Run the repository RPC consistency checker from the project root."""

		import os
		import sys
		import argparse


		PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
		CHECKER_SRC = os.path.join(PROJECT_ROOT, "rpc-consistency-checker", "src")
		sys.path.insert(0, CHECKER_SRC)

		from checker import has_critical_issues, print_report, scan_project # noqa: E402


		def main() -> int:
		parser = argparse.ArgumentParser(
		description="Check GekkoFS RPC tags, structs, handlers, and forwards."
		)
		parser.add_argument(
		"project_root",
		nargs="?",
		default=PROJECT_ROOT,
		help="GekkoFS project root (default: repository root)",
		)
		args = parser.parse_args()
		project_root = args.project_root
		project_root = os.path.abspath(project_root)
		if not os.path.isdir(project_root):
		print(f"Error: {project_root} is not a directory", file=sys.stderr)
		return 1

		print(f"Scanning project: {project_root}\n")
		scanner = scan_project(project_root)
		print_report(scanner)
		return 1 if has_critical_issues(scanner) else 0


		if __name__ == "__main__":
		raise SystemExit(main())
		No newline at end of file