feat(convert): integrate process metrics into JSON and SQLite converters

- add process_metrics function to record peak RSS and CPU time - update JSON converter to include metrics in completion marker - modify SQLite converter to utilize new metrics for performance tracking - enhance storage size updates with locking mechanism for concurrent access
2026-07-11 21:39:08 -04:00
parent 96ed7bc918
commit 48c102d2a4
17 changed files with 1928 additions and 187 deletions
--- a/common/monitoring.py
+++ b/common/monitoring.py
@@ -0,0 +1,66 @@
+"""Remote host metrics via Prometheus / node_exporter (stdlib urllib only).
+
+The PostgreSQL instance is remote, so psutil cannot attribute its CPU/RAM
+(spec 09 assumed a local instance). The database host runs Prometheus;
+its base URL comes from the TRIBO_PROM_URL environment variable
+(e.g. http://192.168.10.73:9090). Unset = monitoring disabled.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import urllib.parse
+import urllib.request
+
+logger = logging.getLogger(__name__)
+
+QUERY_STEP_S = 5
+HTTP_TIMEOUT_S = 15
+# rate() window must span at least two scrapes (15 s default interval)
+CPU_QUERY = '100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[30s])))'
+RAM_QUERY = "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes"
+
+
+def get_prom_url() -> str:
+	return os.environ.get("TRIBO_PROM_URL", "")
+
+
+def query_range(base_url: str, promql: str, start: float, end: float) -> list[float]:
+	params = urllib.parse.urlencode({"query": promql, "start": start, "end": end, "step": QUERY_STEP_S})
+	url = base_url.rstrip("/") + "/api/v1/query_range?" + params
+	with urllib.request.urlopen(url, timeout=HTTP_TIMEOUT_S) as resp:
+		data = json.load(resp)
+	if data.get("status") != "success":
+		raise RuntimeError(f"prometheus query failed: {data.get('errorType')}: {data.get('error')}")
+	result = data["data"]["result"]
+	if not result:
+		return []
+	return [float(v) for _, v in result[0]["values"]]
+
+
+def host_window_metrics(base_url: str, start: float, end: float) -> dict | None:
+	"""CPU/RAM of the monitored host over [start, end] wall-clock seconds.
+
+	Returns None (with a warning) on any failure - monitoring is best-effort
+	and must never fail the owning task.
+	"""
+	try:
+		if end - start < 2 * QUERY_STEP_S:
+			end = start + 2 * QUERY_STEP_S
+		cpu = query_range(base_url, CPU_QUERY, start, end)
+		ram = query_range(base_url, RAM_QUERY, start, end)
+		if not cpu or not ram:
+			logger.warning("prometheus returned no samples for the window (expected condition: short window or scrape lag)")
+			return None
+		return {
+			"host_cpu_avg_pct": round(sum(cpu) / len(cpu), 1),
+			"host_cpu_max_pct": round(max(cpu), 1),
+			"host_ram_used_baseline_mb": round(ram[0] / 1048576),
+			"host_ram_used_peak_mb": round(max(ram) / 1048576),
+			"host_ram_used_delta_mb": round((max(ram) - ram[0]) / 1048576),
+		}
+	except Exception:
+		logger.warning("host metrics collection skipped (prometheus unreachable or query failed)", exc_info=True)
+		return None
--- a/common/pg.py
+++ b/common/pg.py
@@ -0,0 +1,16 @@
+"""PostgreSQL DSN access (docs/rules/code-config-yaml.md section 3).
+
+The DSN comes from the TRIBO_PG_DSN environment variable and is read ONLY
+here. It may embed a password: never log it, never hardcode it, never
+commit it - log host/database names at most.
+"""
+
+from __future__ import annotations
+
+import os
+
+DEFAULT_DSN = "postgresql://postgres@localhost:5432/tribo"
+
+
+def get_dsn() -> str:
+	return os.environ.get("TRIBO_PG_DSN", DEFAULT_DSN)
--- a/common/pipeline.py
+++ b/common/pipeline.py
@@ -10,6 +10,7 @@ import logging
 import sys
 from pathlib import Path

+import psutil
 import yaml

 logger = logging.getLogger(__name__)
@@ -57,3 +58,22 @@ def write_marker(out_root: Path, task_id: str, entries: dict) -> Path:
 def load_lab_config(out_root: Path) -> dict:
 	with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
 		return yaml.safe_load(fh)
+
+
+def process_metrics() -> dict:
+	"""Peak RSS and CPU time of the current process.
+
+	Every converter records its own build cost in its marker so the task 10
+	hardware-sizing model can extrapolate per-format RAM/CPU needs, not just
+	disk. peak_wset exists on Windows; the rss fallback covers Linux, where
+	the value is the current (not peak) RSS - good enough for streaming
+	converters whose RSS is flat by design.
+	"""
+	proc = psutil.Process()
+	mem = proc.memory_info()
+	peak = getattr(mem, "peak_wset", 0) or mem.rss
+	cpu = proc.cpu_times()
+	return {
+		"proc_peak_rss_mb": round(peak / 1048576, 1),
+		"proc_cpu_seconds": round(cpu.user + cpu.system, 1),
+	}
--- a/common/queries/init.py
+++ b/common/queries/init.py
@@ -0,0 +1,15 @@
+"""Benchmark query implementations for the file-based formats (spec 08).
+
+One module per format (csv_queries, json_queries, rdf_queries), each with
+functions q1..q7 returning canonical result rows (see common/results.py).
+The runnable wrappers under out/bench/queries/ are generated by
+make_queries.py and delegate here. SQLite / PostgreSQL implementations are
+plain SQL files written by make_queries.py.
+
+The relational formats answer Q2/Q3/Q4/Q7 from the precomputed
+track_summary (their idiomatic strength); the file formats derive the same
+values from raw cycles via common/track_summary.py - identical algorithm,
+identical results (bench-methodology section 5).
+"""
+
+Q1_TRACK_CODE = "B722-W2-C13-T2"  # fixed benchmark track, spec 08
--- a/common/queries/csv_queries.py
+++ b/common/queries/csv_queries.py
@@ -0,0 +1,139 @@
+"""Q1-Q7 over the raw CSV tree: pure python + csv module, streaming.
+
+Q1 is a direct path read (the honest CSV strength, spec 08); Q2-Q7 are
+deterministic directory walks with programmatic joins. No manifest
+validation here - queries measure retrieval, not integrity checking.
+"""
+
+from __future__ import annotations
+
+import csv
+from pathlib import Path
+
+from common.queries import Q1_TRACK_CODE
+from common.track_summary import summarize_track
+
+
+def _rows(path: Path) -> tuple[list[str], list[list[str]]]:
+	with open(path, newline="", encoding="ascii") as fh:
+		reader = csv.reader(fh)
+		header = [h.lower() for h in next(reader)]
+		return header, list(reader)
+
+
+def _dict_row(path: Path) -> dict[str, str]:
+	header, rows = _rows(path)
+	return dict(zip(header, rows[0]))
+
+
+def _track_path(csv_root: Path, track_code: str) -> Path:
+	batch, wafer, coupon, track = track_code.split("-")
+	return csv_root / f"batch_{batch}" / f"wafer_{wafer}" / f"coupon_{coupon}" / f"track_{track}"
+
+
+def _coupon_dirs(csv_root: Path):
+	for batch_dir in sorted(csv_root.glob("batch_*")):
+		batch_code = batch_dir.name[6:]
+		for wafer_dir in sorted(batch_dir.glob("wafer_*")):
+			for coupon_dir in sorted(wafer_dir.glob("coupon_*")):
+				yield batch_code, coupon_dir
+
+
+def _track_dirs(coupon_dir: Path) -> list[Path]:
+	return sorted(coupon_dir.glob("track_T*"))
+
+
+def _cofs(track_dir: Path) -> list[float]:
+	_, rows = _rows(track_dir / "cof_vs_cycle.csv")
+	return [float(r[1]) for r in rows]
+
+
+def _mean(values: list[float]) -> float:
+	total = 0.0
+	for v in values:
+		total += v
+	return total / len(values)
+
+
+def q1(csv_root: Path) -> list[tuple]:
+	_, rows = _rows(_track_path(csv_root, Q1_TRACK_CODE) / "cof_vs_cycle.csv")
+	return [(int(r[0]), float(r[1])) for r in rows]
+
+
+def q2(csv_root: Path) -> list[tuple]:
+	out = []
+	for _, coupon_dir in _coupon_dirs(csv_root):
+		info = _dict_row(coupon_dir / "coupon_info.csv")
+		if info["run_code"] == "RESERVE" or not 9.5 <= float(info["au_wtpct_mean"]) <= 10.5:
+			continue
+		tracks = _track_dirs(coupon_dir)
+		env = _dict_row(tracks[0] / "track_info.csv")["environment"]
+		cof_ss = _mean([summarize_track(_cofs(t))[0] for t in tracks])
+		out.append((info["coupon_code"], env, cof_ss))
+	return out
+
+
+def q3(csv_root: Path) -> list[tuple]:
+	out = []
+	for batch_code, coupon_dir in _coupon_dirs(csv_root):
+		info = _dict_row(coupon_dir / "coupon_info.csv")
+		if info["run_code"] == "RESERVE":
+			continue
+		header, rows = _rows(coupon_dir / "nanoindentation.csv")
+		h_col = header.index("hardness_gpa")
+		hardness = _mean([float(r[h_col]) for r in rows])
+		cof_ss = _mean([summarize_track(_cofs(t))[0] for t in _track_dirs(coupon_dir)])
+		out.append((info["coupon_code"], batch_code, hardness, cof_ss))
+	return out
+
+
+def q4(csv_root: Path) -> list[tuple]:
+	out = []
+	for _, coupon_dir in _coupon_dirs(csv_root):
+		for track_dir in _track_dirs(coupon_dir):
+			info = _dict_row(track_dir / "track_info.csv")
+			if info["environment"] != "dry_n2" or float(info["load_mn"]) != 100.0:
+				continue
+			cof_ss = summarize_track(_cofs(track_dir))[0]
+			if cof_ss > 0.20:
+				out.append((info["track_code"], cof_ss))
+	return out
+
+
+def q5(csv_root: Path) -> list[tuple]:
+	sums: dict[str, list[float]] = {}
+	for batch_code, coupon_dir in _coupon_dirs(csv_root):
+		for track_dir in _track_dirs(coupon_dir):
+			run_in = summarize_track(_cofs(track_dir))[2]
+			acc = sums.setdefault(batch_code, [0.0, 0.0])
+			acc[0] += run_in
+			acc[1] += 1
+	return [(batch, acc[0] / acc[1]) for batch, acc in sorted(sums.items())]
+
+
+def q6(csv_root: Path) -> list[tuple]:
+	out = []
+	for _, coupon_dir in _coupon_dirs(csv_root):
+		info = _dict_row(coupon_dir / "coupon_info.csv")
+		if info["run_code"] == "RESERVE":
+			continue
+		tracks = _track_dirs(coupon_dir)
+		load = float(_dict_row(tracks[0] / "track_info.csv")["load_mn"])
+		wear = _mean([float(_dict_row(t / "wear.csv")["wear_volume_um3"]) for t in tracks])
+		out.append((info["coupon_code"], load, wear))
+	return out
+
+
+def q7(csv_root: Path) -> list[tuple]:
+	groups: dict[tuple[str, int], list[float]] = {}
+	for _, coupon_dir in _coupon_dirs(csv_root):
+		for track_dir in _track_dirs(coupon_dir):
+			info = _dict_row(track_dir / "track_info.csv")
+			bucket = int(round(float(info["speed_mm_s"]) * float(info["load_mn"])))
+			cofs = _cofs(track_dir)
+			run_in = summarize_track(cofs)[2]
+			acc = groups.setdefault((info["environment"], bucket), [0.0, 0.0])
+			for cof in cofs[run_in:]:  # cycles strictly past run-in
+				acc[0] += cof
+				acc[1] += 1
+	return [(env, bucket, acc[0] / acc[1]) for (env, bucket), acc in sorted(groups.items())]
--- a/common/queries/json_queries.py
+++ b/common/queries/json_queries.py
@@ -0,0 +1,169 @@
+"""Q1-Q7 over the JSON-LD FULL variant: ijson streaming parser (spec 08).
+
+Each coupon file is streamed event-by-event; scalar coupon fields appear
+before the bulk arrays in the serialization, so filtering queries (Q2) can
+abandon a non-matching file before parsing its megabytes of points.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import ijson
+
+from common.queries import Q1_TRACK_CODE
+from common.track_summary import summarize_track
+
+
+class CouponDoc:
+	__slots__ = ("coupon_code", "batch_code", "run_code", "au", "hardness", "tracks")
+
+	def __init__(self) -> None:
+		self.coupon_code = ""
+		self.batch_code = ""
+		self.run_code = ""
+		self.au = 0.0
+		self.hardness: list[float] = []
+		self.tracks: list[dict] = []
+
+
+def _scan(path: Path, au_range: tuple[float, float] | None = None) -> CouponDoc | None:
+	"""Stream one coupon file; with au_range set, bail out early on mismatch."""
+	doc = CouponDoc()
+	track: dict | None = None
+	with open(path, "rb") as fh:
+		for prefix, event, value in ijson.parse(fh, use_float=True):
+			if prefix == "coupon_code":
+				doc.coupon_code = value
+			elif prefix == "batch_code":
+				doc.batch_code = value
+			elif prefix == "run_code":
+				doc.run_code = value
+				if doc.run_code == "RESERVE":
+					return None
+			elif prefix == "au_wtpct_mean":
+				doc.au = value
+				if au_range and not au_range[0] <= value <= au_range[1]:
+					return None
+			elif prefix == "nanoindentation.indents.item.hardness_gpa":
+				doc.hardness.append(value)
+			elif prefix == "tracks.item" and event == "start_map":
+				track = {"cycles": [], "cofs": []}
+			elif track is not None:
+				if prefix == "tracks.item.track_code":
+					track["track_code"] = value
+				elif prefix == "tracks.item.environment":
+					track["environment"] = value
+				elif prefix == "tracks.item.load_mn":
+					track["load_mn"] = value
+				elif prefix == "tracks.item.speed_mm_s":
+					track["speed_mm_s"] = value
+				elif prefix == "tracks.item.cycles.item.cycle":
+					track["cycles"].append(value)
+				elif prefix == "tracks.item.cycles.item.cof":
+					track["cofs"].append(value)
+				elif prefix == "tracks.item.wear.wear_volume_um3":
+					track["wear"] = value
+				elif prefix == "tracks.item" and event == "end_map":
+					doc.tracks.append(track)
+					track = None
+	return doc
+
+
+def _files(json_root: Path) -> list[Path]:
+	return sorted((json_root / "full").glob("*.jsonld"))
+
+
+def _mean(values: list[float]) -> float:
+	total = 0.0
+	for v in values:
+		total += v
+	return total / len(values)
+
+
+def _cof_ss(doc: CouponDoc) -> float:
+	return _mean([summarize_track(t["cofs"])[0] for t in doc.tracks])
+
+
+def q1(json_root: Path) -> list[tuple]:
+	coupon_code = Q1_TRACK_CODE.rsplit("-", 1)[0]
+	doc = _scan(json_root / "full" / f"{coupon_code}.jsonld")
+	for track in doc.tracks:
+		if track["track_code"] == Q1_TRACK_CODE:
+			return [(int(c), cof) for c, cof in zip(track["cycles"], track["cofs"])]
+	raise LookupError(f"track {Q1_TRACK_CODE} not found")
+
+
+def q2(json_root: Path) -> list[tuple]:
+	out = []
+	for path in _files(json_root):
+		doc = _scan(path, au_range=(9.5, 10.5))
+		if doc is None:
+			continue
+		out.append((doc.coupon_code, doc.tracks[0]["environment"], _cof_ss(doc)))
+	return out
+
+
+def q3(json_root: Path) -> list[tuple]:
+	out = []
+	for path in _files(json_root):
+		doc = _scan(path)
+		if doc is None:
+			continue
+		out.append((doc.coupon_code, doc.batch_code, _mean(doc.hardness), _cof_ss(doc)))
+	return out
+
+
+def q4(json_root: Path) -> list[tuple]:
+	out = []
+	for path in _files(json_root):
+		doc = _scan(path)
+		if doc is None:
+			continue
+		for track in doc.tracks:
+			if track["environment"] != "dry_n2" or track["load_mn"] != 100.0:
+				continue
+			cof_ss = summarize_track(track["cofs"])[0]
+			if cof_ss > 0.20:
+				out.append((track["track_code"], cof_ss))
+	return out
+
+
+def q5(json_root: Path) -> list[tuple]:
+	sums: dict[str, list[float]] = {}
+	for path in _files(json_root):
+		doc = _scan(path)
+		if doc is None:
+			continue
+		acc = sums.setdefault(doc.batch_code, [0.0, 0.0])
+		for track in doc.tracks:
+			acc[0] += summarize_track(track["cofs"])[2]
+			acc[1] += 1
+	return [(batch, acc[0] / acc[1]) for batch, acc in sorted(sums.items())]
+
+
+def q6(json_root: Path) -> list[tuple]:
+	out = []
+	for path in _files(json_root):
+		doc = _scan(path)
+		if doc is None:
+			continue
+		load = doc.tracks[0]["load_mn"]
+		out.append((doc.coupon_code, load, _mean([t["wear"] for t in doc.tracks])))
+	return out
+
+
+def q7(json_root: Path) -> list[tuple]:
+	groups: dict[tuple[str, int], list[float]] = {}
+	for path in _files(json_root):
+		doc = _scan(path)
+		if doc is None:
+			continue
+		for track in doc.tracks:
+			bucket = int(round(track["speed_mm_s"] * track["load_mn"]))
+			run_in = summarize_track(track["cofs"])[2]
+			acc = groups.setdefault((track["environment"], bucket), [0.0, 0.0])
+			for cof in track["cofs"][run_in:]:
+				acc[0] += cof
+				acc[1] += 1
+	return [(env, bucket, acc[0] / acc[1]) for (env, bucket), acc in sorted(groups.items())]
--- a/common/queries/rdf_queries.py
+++ b/common/queries/rdf_queries.py
@@ -0,0 +1,173 @@
+"""Q1-Q7 over the oxigraph triplestore: SPARQL retrieval (spec 08).
+
+The store holds raw data only (no precomputed summaries), so queries that
+need steady-state COF or run-in retrieve the raw cycles via SPARQL and
+derive the values with the shared algorithm (common/track_summary.py) -
+identical semantics to every other format. The SPARQL text of each query
+is exported to out/bench/queries/rdf/q<N>.sparql by make_queries.py.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pyoxigraph import Store
+
+from common.queries import Q1_TRACK_CODE
+from common.track_summary import summarize_track
+
+PREFIX = "PREFIX tribo: <https://sandia.gov/ontology/tribology#>\n"
+TRACK_IRI = f"<https://sandia.gov/ontology/tribology/id/track/{Q1_TRACK_CODE}>"
+
+SPARQL = {
+	"q1": PREFIX + f"""SELECT ?cycle ?cof WHERE {{
+  {TRACK_IRI} tribo:hasCycle ?c .
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}}""",
+	"q2": PREFIX + """SELECT ?ccode ?env ?track ?cycle ?cof WHERE {
+  ?coupon tribo:au_wtpct_mean ?au ; tribo:coupon_code ?ccode .
+  FILTER(?au >= 9.5 && ?au <= 10.5)
+  ?track tribo:partOf ?coupon ; tribo:environment ?env ; tribo:hasCycle ?c .
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}""",
+	"q3_hardness": PREFIX + """SELECT ?ccode ?bcode (AVG(?h) AS ?hardness) WHERE {
+  ?coupon tribo:coupon_code ?ccode ; tribo:cutFrom ?wafer ; tribo:nanoindentation ?m .
+  ?wafer tribo:partOf ?batch .
+  ?batch tribo:batch_code ?bcode .
+  ?m tribo:hasIndent ?i .
+  ?i tribo:hardness_gpa ?h .
+} GROUP BY ?ccode ?bcode""",
+	"q3_cycles": PREFIX + """SELECT ?ccode ?track ?cycle ?cof WHERE {
+  ?track a tribo:FrictionTest ; tribo:partOf ?coupon ; tribo:hasCycle ?c .
+  ?coupon tribo:coupon_code ?ccode .
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}""",
+	"q4": PREFIX + """SELECT ?tcode ?track ?cycle ?cof WHERE {
+  ?track tribo:environment "dry_n2" ; tribo:load_mn ?load ; tribo:track_code ?tcode ; tribo:hasCycle ?c .
+  FILTER(?load = 100)
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}""",
+	"q5": PREFIX + """SELECT ?bcode ?track ?cycle ?cof WHERE {
+  ?track a tribo:FrictionTest ; tribo:partOf ?coupon ; tribo:hasCycle ?c .
+  ?coupon tribo:cutFrom ?wafer .
+  ?wafer tribo:partOf ?batch .
+  ?batch tribo:batch_code ?bcode .
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}""",
+	"q6": PREFIX + """SELECT ?ccode ?load (AVG(?wv) AS ?wear) WHERE {
+  ?track tribo:partOf ?coupon ; tribo:load_mn ?load ; tribo:wear ?w .
+  ?w tribo:wear_volume_um3 ?wv .
+  ?coupon tribo:coupon_code ?ccode .
+} GROUP BY ?ccode ?load""",
+	"q7": PREFIX + """SELECT ?env ?speed ?load ?track ?cycle ?cof WHERE {
+  ?track tribo:environment ?env ; tribo:speed_mm_s ?speed ; tribo:load_mn ?load ; tribo:hasCycle ?c .
+  ?c tribo:cycle ?cycle ; tribo:cof ?cof .
+}""",
+}
+
+
+def _store(store_path: Path) -> Store:
+	try:
+		return Store.read_only(str(store_path))
+	except AttributeError:
+		return Store(str(store_path))
+
+
+def _track_cofs(solutions, track_var: str, keep: tuple[str, ...]) -> dict[str, dict]:
+	"""Group cycle solutions by track: ordered cof list + kept literals."""
+	tracks: dict[str, dict] = {}
+	for sol in solutions:
+		key = str(sol[track_var])
+		entry = tracks.get(key)
+		if entry is None:
+			entry = {"pairs": []}
+			for name in keep:
+				entry[name] = sol[name].value
+			tracks[key] = entry
+		entry["pairs"].append((int(sol["cycle"].value), float(sol["cof"].value)))
+	for entry in tracks.values():
+		entry["pairs"].sort()
+		entry["cofs"] = [cof for _, cof in entry["pairs"]]
+	return tracks
+
+
+def _mean(values: list[float]) -> float:
+	total = 0.0
+	for v in values:
+		total += v
+	return total / len(values)
+
+
+def q1(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	return [(int(s["cycle"].value), float(s["cof"].value)) for s in store.query(SPARQL["q1"])]
+
+
+def q2(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	tracks = _track_cofs(store.query(SPARQL["q2"]), "track", ("ccode", "env"))
+	coupons: dict[tuple[str, str], list[float]] = {}
+	for entry in sorted(tracks.values(), key=lambda e: e["ccode"]):
+		coupons.setdefault((entry["ccode"], entry["env"]), []).append(summarize_track(entry["cofs"])[0])
+	return [(code, env, _mean(values)) for (code, env), values in sorted(coupons.items())]
+
+
+def q3(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	hardness = {
+		s["ccode"].value: (s["bcode"].value, float(s["hardness"].value))
+		for s in store.query(SPARQL["q3_hardness"])
+	}
+	tracks = _track_cofs(store.query(SPARQL["q3_cycles"]), "track", ("ccode",))
+	coupons: dict[str, list[float]] = {}
+	for entry in sorted(tracks.values(), key=lambda e: e["ccode"]):
+		coupons.setdefault(entry["ccode"], []).append(summarize_track(entry["cofs"])[0])
+	return [
+		(code, hardness[code][0], hardness[code][1], _mean(values))
+		for code, values in sorted(coupons.items())
+	]
+
+
+def q4(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	tracks = _track_cofs(store.query(SPARQL["q4"]), "track", ("tcode",))
+	out = []
+	for entry in sorted(tracks.values(), key=lambda e: e["tcode"]):
+		cof_ss = summarize_track(entry["cofs"])[0]
+		if cof_ss > 0.20:
+			out.append((entry["tcode"], cof_ss))
+	return out
+
+
+def q5(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	tracks = _track_cofs(store.query(SPARQL["q5"]), "track", ("bcode",))
+	sums: dict[str, list[float]] = {}
+	for entry in sorted(tracks.values(), key=lambda e: e["bcode"]):
+		acc = sums.setdefault(entry["bcode"], [0.0, 0.0])
+		acc[0] += summarize_track(entry["cofs"])[2]
+		acc[1] += 1
+	return [(batch, acc[0] / acc[1]) for batch, acc in sorted(sums.items())]
+
+
+def q6(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	return [
+		(s["ccode"].value, float(s["load"].value), float(s["wear"].value))
+		for s in store.query(SPARQL["q6"])
+	]
+
+
+def q7(store_path: Path) -> list[tuple]:
+	store = _store(store_path)
+	tracks = _track_cofs(store.query(SPARQL["q7"]), "track", ("env", "speed", "load"))
+	groups: dict[tuple[str, int], list[float]] = {}
+	for key in sorted(tracks):
+		entry = tracks[key]
+		bucket = int(round(float(entry["speed"]) * float(entry["load"])))
+		run_in = summarize_track(entry["cofs"])[2]
+		acc = groups.setdefault((entry["env"], bucket), [0.0, 0.0])
+		for cof in entry["cofs"][run_in:]:
+			acc[0] += cof
+			acc[1] += 1
+	return [(env, bucket, acc[0] / acc[1]) for (env, bucket), acc in sorted(groups.items())]
--- a/common/relational.py
+++ b/common/relational.py
@@ -0,0 +1,161 @@
+"""Shared relational row stream for the SQLite and PostgreSQL converters.
+
+Walks the CSV corpus once (manifest-validated via CorpusReader) and yields
+(table, row_tuple) pairs in FK-dependency-safe order: a parent row is always
+yielded before any row that references it. Both engines load the SAME
+logical schema from this stream (docs/rules/db-sql-schema.md section 6).
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Iterator
+
+from common.corpus import CorpusReader
+from common.track_summary import summarize_track
+
+logger = logging.getLogger(__name__)
+
+# Dependency-safe order; consumers that buffer rows must flush in this order.
+TABLES = [
+	"instruments",
+	"batches",
+	"runs",
+	"wafers",
+	"coupons",
+	"tracks",
+	"simtra_profiles",
+	"xrf_points",
+	"profilometry_points",
+	"nanoindentation",
+	"afm",
+	"friction_cycles",
+	"friction_loop_points",
+	"wear",
+	"track_summary",
+]
+
+COLUMNS = {
+	"instruments": ("instrument_id", "instrument_code", "name", "role"),
+	"batches": ("batch_id", "batch_code", "pt_gun_tilt_deg", "au_gun_tilt_deg", "pt_power_w", "au_power_w", "pt_discharge_v", "au_discharge_v", "deposition_date"),
+	"runs": ("run_id", "run_code", "environment", "date", "plates", "operator"),
+	"wafers": ("wafer_id", "wafer_code", "batch_id", "wafer_index", "deposition_date", "coupons", "friction_coupons", "reserve_coupons"),
+	"coupons": ("coupon_id", "coupon_code", "wafer_id", "batch_id", "grid_row", "grid_col", "thickness_um", "ra_nm", "au_wtpct_mean", "run_id", "plate", "probe", "square"),
+	"tracks": ("track_id", "track_code", "coupon_id", "run_id", "environment", "load_mn", "stroke_mm", "speed_mm_s", "counterface_id", "started_at"),
+	"simtra_profiles": ("batch_id", "row_no", "angle_deg", "energy_ev", "pt_flux", "au_flux"),
+	"xrf_points": ("coupon_id", "grid_x", "grid_y", "pt_wtpct", "au_wtpct"),
+	"profilometry_points": ("coupon_id", "grid_x", "grid_y", "thickness_um"),
+	"nanoindentation": ("coupon_id", "indent_id", "x_um", "y_um", "hardness_gpa", "reduced_modulus_gpa", "max_load_mn"),
+	"afm": ("coupon_id", "ra_nm", "rq_nm", "image_file"),
+	"friction_cycles": ("track_id", "cycle", "cof"),
+	"friction_loop_points": ("track_id", "cycle", "pt", "position_um", "friction_force_mn"),
+	"wear": ("track_id", "wear_volume_um3", "k_archard", "sliding_distance_m"),
+	"track_summary": ("track_id", "cof_ss_mean", "cof_ss_std", "run_in_cycles"),
+}
+
+EXPECTED_COUNTS_KEYS = TABLES  # every table is count-validated after load
+
+
+def expected_counts(cfg: dict) -> dict[str, int]:
+	v = cfg["volumes"]
+	h = cfg["hierarchy"]
+	return {
+		"instruments": len(cfg["instruments"]),
+		"batches": h["batches"],
+		"runs": cfg["friction_assignment"]["runs"],
+		"wafers": h["batches"] * h["wafers_per_batch"],
+		"coupons": v["coupons_total"],
+		"tracks": v["tracks_total"],
+		"simtra_profiles": h["batches"] * v["simtra_rows_per_batch"],
+		"xrf_points": v["xrf_points_total"],
+		"profilometry_points": v["coupons_total"] * v["profilometry_points_per_coupon"],
+		"nanoindentation": v["coupons_total"] * v["nanoindentation_indents_per_coupon"],
+		"afm": v["coupons_total"],
+		"friction_cycles": v["cycle_rows_total"],
+		"friction_loop_points": v["loop_points_total"],
+		"wear": v["tracks_total"],
+		"track_summary": v["tracks_total"],
+	}
+
+
+def stream_rows(cfg: dict, reader: CorpusReader) -> Iterator[tuple[str, tuple]]:
+	batch_ids: dict[str, int] = {}
+	run_ids: dict[str, int] = {}
+
+	for i, inst in enumerate(cfg["instruments"], 1):
+		yield "instruments", (i, inst["instrument_id"], inst["name"], inst["role"])
+	for i, row in enumerate(reader.dicts("batches.csv"), 1):
+		batch_ids[row["batch_code"]] = i
+		yield "batches", (
+			i, row["batch_code"], row["pt_gun_tilt_deg"], row["au_gun_tilt_deg"],
+			row["pt_power_w"], row["au_power_w"], row["pt_discharge_v"],
+			row["au_discharge_v"], row["deposition_date"],
+		)
+	for i, row in enumerate(reader.dicts("runs.csv"), 1):
+		run_ids[row["run_code"]] = i
+		yield "runs", (i, row["run_code"], row["environment"], row["date"], row["plates"], row["operator"])
+	for code, batch_id in batch_ids.items():
+		for row_no, row in enumerate(reader.dicts(f"simtra/simtra_profile_{code}.csv"), 1):
+			yield "simtra_profiles", (batch_id, row_no, row["angle_deg"], row["energy_ev"], row["pt_flux"], row["au_flux"])
+
+	h = cfg["hierarchy"]
+	fa = cfg["friction_assignment"]
+	wafer_id = coupon_id = track_id = 0
+	for batch in cfg["deposition_matrix"]:
+		b_code = batch["batch_code"]
+		for w in range(1, h["wafers_per_batch"] + 1):
+			wafer_id += 1
+			wafer_dir = f"batch_{b_code}/wafer_W{w}"
+			info = reader.dicts(f"{wafer_dir}/wafer_info.csv")[0]
+			yield "wafers", (
+				wafer_id, info["wafer_code"], batch_ids[b_code], info["wafer_index"],
+				info["deposition_date"], info["coupons"], info["friction_coupons"], info["reserve_coupons"],
+			)
+			for c in range(1, h["coupons_per_wafer"] + 1):
+				coupon_id += 1
+				rel_dir = f"{wafer_dir}/coupon_C{c:02d}"
+				info = reader.dicts(f"{rel_dir}/coupon_info.csv")[0]
+				is_friction = info["run_code"] != "RESERVE"
+				yield "coupons", (
+					coupon_id, info["coupon_code"], wafer_id, batch_ids[b_code],
+					info["grid_row"], info["grid_col"], info["thickness_um"], info["ra_nm"],
+					info["au_wtpct_mean"], run_ids[info["run_code"]] if is_friction else None,
+					info.get("plate"), info.get("probe"), info.get("square"),
+				)
+				for row in reader.dicts(f"{rel_dir}/xrf_map.csv"):
+					yield "xrf_points", (coupon_id, row["grid_x"], row["grid_y"], row["pt_wtpct"], row["au_wtpct"])
+				for row in reader.dicts(f"{rel_dir}/profilometry.csv"):
+					yield "profilometry_points", (coupon_id, row["grid_x"], row["grid_y"], row["thickness_um"])
+				for row in reader.dicts(f"{rel_dir}/nanoindentation.csv"):
+					yield "nanoindentation", (
+						coupon_id, row["indent_id"], row["x_um"], row["y_um"],
+						row["hardness_gpa"], row["reduced_modulus_gpa"], row["max_load_mn"],
+					)
+				afm = reader.dicts(f"{rel_dir}/afm.csv")[0]
+				yield "afm", (coupon_id, afm["ra_nm"], afm["rq_nm"], afm["image_file"])
+				if not is_friction:
+					continue
+				for t in range(1, fa["tracks_per_friction_coupon"] + 1):
+					track_id += 1
+					track_dir = f"{rel_dir}/track_T{t}"
+					tinfo = reader.dicts(f"{track_dir}/track_info.csv")[0]
+					yield "tracks", (
+						track_id, tinfo["track_code"], coupon_id, run_ids[tinfo["run_code"]],
+						tinfo["environment"], tinfo["load_mn"], tinfo["stroke_mm"], tinfo["speed_mm_s"],
+						tinfo["counterface_id"], tinfo["started_at"],
+					)
+					cofs: list[float] = []
+					for row in reader.dicts(f"{track_dir}/cof_vs_cycle.csv"):
+						cofs.append(row["cof"])
+						yield "friction_cycles", (track_id, row["cycle"], row["cof"])
+					ss_mean, ss_std, run_in = summarize_track(cofs)
+					yield "track_summary", (track_id, ss_mean, ss_std, run_in)
+					pt = 0
+					last_cycle = None
+					for row in reader.dicts(f"{track_dir}/friction_loops.csv"):
+						pt = pt + 1 if row["cycle"] == last_cycle else 1
+						last_cycle = row["cycle"]
+						yield "friction_loop_points", (track_id, row["cycle"], pt, row["position_um"], row["friction_force_mn"])
+					wear = reader.dicts(f"{track_dir}/wear.csv")[0]
+					yield "wear", (track_id, wear["wear_volume_um3"], wear["k_archard"], wear["sliding_distance_m"])
+		logger.info("batch %s streamed (through coupon %d, track %d)", b_code, coupon_id, track_id)
--- a/common/results.py
+++ b/common/results.py
@@ -0,0 +1,73 @@
+"""Canonical benchmark result handling (spec 08, test-pipeline-validation).
+
+Every implementation of a benchmark query returns the same canonical rows:
+tuples of str / int / float columns. Rows are compared SORTED; floats agree
+within 1e-9. The canonical expected artifacts are produced from PostgreSQL
+and cross-validated against SQLite (task 08); every measured run (task 09)
+re-validates its rows against them.
+
+Two serializations:
+- full: floats via repr() - lossless, used for stdout and expected/*.rows.csv;
+- rounded (6 decimals): used ONLY for the sha256 fingerprint, so engines
+  whose aggregate arithmetic differs in the last bits hash identically.
+The tolerant row comparison is the authoritative check; the sha256 is a
+compact fingerprint for reports and markers.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import sys
+
+from common.corpus import coerce
+
+FLOAT_TOL = 1e-9
+
+Row = tuple
+
+
+def sort_rows(rows: list[Row]) -> list[Row]:
+	return sorted(rows)
+
+
+def _fmt(value, rounded: bool) -> str:
+	if isinstance(value, float):
+		return f"{round(value, 6):.6f}" if rounded else repr(value)
+	return str(value)
+
+
+def serialize(rows: list[Row], rounded: bool = False) -> str:
+	return "\n".join(",".join(_fmt(v, rounded) for v in row) for row in rows) + "\n"
+
+
+def sha256_of(rows: list[Row]) -> str:
+	return hashlib.sha256(serialize(sort_rows(rows), rounded=True).encode("ascii")).hexdigest()
+
+
+def print_rows(rows: list[Row]) -> None:
+	sys.stdout.write(serialize(sort_rows(rows)))
+
+
+def parse_rows(text: str) -> list[Row]:
+	rows = []
+	for line in text.splitlines():
+		if line:
+			rows.append(tuple(coerce(cell) for cell in line.split(",")))
+	return rows
+
+
+def compare(expected: list[Row], actual: list[Row], tol: float = FLOAT_TOL) -> str | None:
+	"""Return None when equal within tolerance, else a first-difference message."""
+	exp, act = sort_rows(expected), sort_rows(actual)
+	if len(exp) != len(act):
+		return f"row count {len(act)} != expected {len(exp)}"
+	for i, (er, ar) in enumerate(zip(exp, act)):
+		if len(er) != len(ar):
+			return f"row {i}: arity {len(ar)} != {len(er)}"
+		for j, (ev, av) in enumerate(zip(er, ar)):
+			if isinstance(ev, float) or isinstance(av, float):
+				if abs(float(ev) - float(av)) > tol:
+					return f"row {i} col {j}: {av!r} != {ev!r} (tol {tol})"
+			elif ev != av:
+				return f"row {i} col {j}: {av!r} != {ev!r}"
+	return None
--- a/common/storage_sizes.py
+++ b/common/storage_sizes.py
@@ -8,24 +8,45 @@ touches other formats' rows.
 from __future__ import annotations

 import logging
+import os
+import time
 from pathlib import Path

 logger = logging.getLogger(__name__)

 HEADER = "format,variant,bytes"
+LOCK_TIMEOUT_S = 30.0  # converters 04-07 may run in parallel; the update is a
+LOCK_POLL_S = 0.2      # read-modify-write, so it is serialized via a lock file


 def update_storage_sizes(bench_dir: Path, fmt: str, variants: list[tuple[str, int]]) -> Path:
-	path = bench_dir / "storage_sizes.csv"
-	kept: list[str] = []
-	if path.exists():
-		lines = path.read_text(encoding="ascii").splitlines()
-		if lines and lines[0] != HEADER:
-			raise ValueError(f"{path}: unexpected header {lines[0]!r}")
-		kept = [ln for ln in lines[1:] if ln and ln.split(",", 1)[0] != fmt]
-	rows = kept + [f"{fmt},{variant},{nbytes}" for variant, nbytes in variants]
 	bench_dir.mkdir(parents=True, exist_ok=True)
-	with open(path, "w", encoding="ascii", newline="\n") as fh:
-		fh.write(HEADER + "\n" + "\n".join(rows) + "\n")
+	path = bench_dir / "storage_sizes.csv"
+	lock = bench_dir / "storage_sizes.lock"
+	fd = _acquire(lock)
+	try:
+		kept: list[str] = []
+		if path.exists():
+			lines = path.read_text(encoding="ascii").splitlines()
+			if lines and lines[0] != HEADER:
+				raise ValueError(f"{path}: unexpected header {lines[0]!r}")
+			kept = [ln for ln in lines[1:] if ln and ln.split(",", 1)[0] != fmt]
+		rows = kept + [f"{fmt},{variant},{nbytes}" for variant, nbytes in variants]
+		with open(path, "w", encoding="ascii", newline="\n") as fh:
+			fh.write(HEADER + "\n" + "\n".join(rows) + "\n")
+	finally:
+		os.close(fd)
+		lock.unlink()
 	logger.info("storage_sizes.csv: %s -> %s", fmt, ", ".join(f"{v}={b}" for v, b in variants))
 	return path
+
+
+def _acquire(lock: Path) -> int:
+	deadline = time.monotonic() + LOCK_TIMEOUT_S
+	while True:
+		try:
+			return os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+		except FileExistsError:
+			if time.monotonic() > deadline:
+				raise TimeoutError(f"storage_sizes lock held too long: {lock}")
+			time.sleep(LOCK_POLL_S)