Merge pull request 'feat(convert): add task 05 sqlite converter with variant B keys" -m "- convert: convert_sqlite.py builds tribo.db (composite PKs, enforced FKs, Q1-Q7 indexes, track_summary)' (#4) from dev_masha into master

Reviewed-on: #4

README.md

@@ -61,7 +61,7 @@ machine (no parallel work). Every task writes a completion marker
 | 02 Process flow diagrams | `make_process_flow.py` | `out/report/process_flow.md`, `out/report/diagrams/` | implemented |
 | 03 CSV data generation | `generate_data.py` | `out/csv/` tree + `MANIFEST.csv` | implemented |
 | 04 Convert to JSON-LD | `convert_json.py` | `out/json/full/`, `out/json/hybrid/dataset.jsonld` | implemented |
-| 05 Convert to SQLite | `convert_sqlite.py` | `out/sqlite/tribo.db` | planned |
+| 05 Convert to SQLite | `convert_sqlite.py` | `out/sqlite/tribo.db` | implemented |
 | 06 Convert to PostgreSQL | `convert_postgresql.py` | live `tribo` DB + `out/pg/tribo.dump` | planned |
 | 07 Convert to RDF | `convert_rdf.py` | `out/rdf/dataset.nt.gz`, oxigraph store | planned |
 | 08 Benchmark queries | `make_queries.py` | `out/bench/queries/`, `out/bench/expected/` | planned |
@@ -142,6 +142,7 @@ measurement counts. See
 docs/
   specs/        task specifications 00-11 (WHAT to build)
   rules/        binding conventions (HOW work is done)
+  research/     measured design studies (e.g. FK key-schema study)
   examples/     imported reference materials (not binding)
 out/            ALL generated artifacts (git-ignored, reproducible):
   config/ csv/ json/ sqlite/ pg/ rdf/ bench/ report/ .done/
@@ -150,6 +151,7 @@ make_lab_config.py    task 01 entry script (one script per task, repo root)
 make_process_flow.py  task 02 entry script
 generate_data.py      task 03 entry script
 convert_json.py       task 04 entry script
+convert_sqlite.py     task 05 entry script
 requirements.txt      closed dependency list (docs/rules/code-python-style.md)
 ```
 

common/corpus.py

@@ -0,0 +1,73 @@
+"""Reading the CSV corpus with MANIFEST validation (converters, tasks 04-07).
+
+Every file consumed from ./out/csv/ is verified against MANIFEST.csv (byte
+size, sha256, row count) before its content is trusted - a mismatch aborts
+the converter (docs/rules/test-pipeline-validation.md section 2).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+MANIFEST_HEADER = "path,rows,bytes,sha256"
+
+INT_RE = re.compile(r"^-?[0-9]+$")
+FLOAT_RE = re.compile(r"^-?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
+
+
+class ValidationError(RuntimeError):
+	pass
+
+
+def coerce(s: str) -> int | float | str:
+	if INT_RE.match(s):
+		return int(s)
+	if FLOAT_RE.match(s):
+		return float(s)
+	return s
+
+
+class CorpusReader:
+	def __init__(self, csv_root: Path) -> None:
+		self.csv_root = csv_root
+		self.manifest = self._load_manifest()
+
+	def _load_manifest(self) -> dict[str, tuple[int, int, str]]:
+		manifest: dict[str, tuple[int, int, str]] = {}
+		with open(self.csv_root / "MANIFEST.csv", encoding="ascii") as fh:
+			header = fh.readline().strip()
+			if header != MANIFEST_HEADER:
+				raise ValidationError(f"MANIFEST.csv: unexpected header {header!r}")
+			for line in fh:
+				path, rows, nbytes, sha = line.strip().split(",")
+				manifest[path] = (int(rows), int(nbytes), sha)
+		logger.info("manifest loaded: %d entries", len(manifest))
+		return manifest
+
+	def read(self, relpath: str) -> tuple[list[str], list[list[str]]]:
+		"""Return (header, raw string rows), verified against MANIFEST."""
+		data = (self.csv_root / relpath).read_bytes()
+		exp_rows, exp_bytes, exp_sha = self.manifest[relpath]
+		if len(data) != exp_bytes or hashlib.sha256(data).hexdigest() != exp_sha:
+			raise ValidationError(f"{relpath}: bytes/sha256 mismatch vs MANIFEST")
+		lines = data.decode("ascii").split("\n")
+		header = lines[0].split(",")
+		rows = [ln.split(",") for ln in lines[1:] if ln]
+		if len(rows) != exp_rows:
+			raise ValidationError(f"{relpath}: {len(rows)} rows != manifest {exp_rows}")
+		return header, rows
+
+	def dicts(self, relpath: str) -> list[dict]:
+		"""Rows as dicts: lowercased CSV headers as keys, numerics coerced,
+		empty cells omitted (docs/rules/data-naming-units.md section 4)."""
+		header, rows = self.read(relpath)
+		keys = [h.lower() for h in header]
+		return [
+			{k: coerce(v) for k, v in zip(keys, row) if v != ""}
+			for row in rows
+		]

common/pipeline.py

@@ -0,0 +1,59 @@
+"""Task-orchestration helpers: CLI shape, dependency markers, completion
+markers, lab config loading (docs/rules/build-pipeline-tasks.md,
+docs/rules/code-python-style.md section 3).
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def parse_task_args(description: str) -> argparse.Namespace:
+	parser = argparse.ArgumentParser(description=description)
+	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
+	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
+	args = parser.parse_args()
+	logging.basicConfig(
+		level=args.log_level,
+		stream=sys.stderr,
+		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
+	)
+	return args
+
+
+def check_dependencies(out_root: Path, deps: list[str]) -> None:
+	for dep in deps:
+		marker = out_root / ".done" / f"{dep}.ok"
+		if not marker.exists():
+			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
+
+
+def remove_stale_marker(out_root: Path, task_id: str) -> Path:
+	marker = out_root / ".done" / f"{task_id}.ok"
+	if marker.exists():
+		logger.info("re-run: removing stale marker %s", marker)
+		marker.unlink()
+	return marker
+
+
+def write_marker(out_root: Path, task_id: str, entries: dict) -> Path:
+	marker = out_root / ".done" / f"{task_id}.ok"
+	lines = [f"task: '{task_id}'", "status: ok"] + [f"{k}: {v}" for k, v in entries.items()]
+	marker.parent.mkdir(parents=True, exist_ok=True)
+	with open(marker, "w", encoding="utf-8", newline="\n") as fh:
+		fh.write("\n".join(lines) + "\n")
+	return marker
+
+
+def load_lab_config(out_root: Path) -> dict:
+	with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
+		return yaml.safe_load(fh)

common/track_summary.py

@@ -0,0 +1,48 @@
+"""Per-track steady-state summary (docs/rules/db-sql-schema.md section 5).
+
+Both engines must produce identical values: SQLite calls this function
+during load (task 05); the PostgreSQL materialized view (task 06)
+implements the same three steps in SQL. Plain sequential float arithmetic
+keeps results within the 1e-9 checksum tolerance across implementations.
+"""
+
+from __future__ import annotations
+
+import math
+
+TAIL_START_CYCLE = 501  # steady-state proxy window: cycles 501..N
+
+
+def summarize_track(cofs: list[float]) -> tuple[float, float, int]:
+	"""Return (cof_ss_mean, cof_ss_std, run_in_cycles) for cofs[i] = cof at cycle i+1."""
+	n = len(cofs)
+	tail = cofs[TAIL_START_CYCLE - 1:]
+	m = _mean(tail)
+	s = _stddev_pop(tail, m)
+
+	run_in = TAIL_START_CYCLE - 1  # fallback when no cycle enters the 2-sigma band
+	band = 2.0 * s
+	for c in range(1, n + 1):
+		if abs(cofs[c - 1] - m) < band:
+			run_in = c
+			break
+
+	ss = cofs[run_in:]  # cycles strictly greater than run_in
+	ss_mean = _mean(ss)
+	ss_std = _stddev_pop(ss, ss_mean)
+	return ss_mean, ss_std, run_in
+
+
+def _mean(values: list[float]) -> float:
+	total = 0.0
+	for v in values:
+		total += v
+	return total / len(values)
+
+
+def _stddev_pop(values: list[float], mean: float) -> float:
+	acc = 0.0
+	for v in values:
+		d = v - mean
+		acc += d * d
+	return math.sqrt(acc / len(values))

convert_json.py

@@ -17,23 +17,26 @@ Appends both sizes to ./out/bench/storage_sizes.csv and writes
 
 from __future__ import annotations
 
-import argparse
-import hashlib
 import json
 import logging
-import re
 import shutil
 import sys
 from pathlib import Path
 
-import yaml
 from rdflib import Graph
 
+from common.corpus import CorpusReader, ValidationError
+from common.pipeline import (
+	check_dependencies,
+	load_lab_config,
+	parse_task_args,
+	remove_stale_marker,
+	write_marker,
+)
 from common.storage_sizes import update_storage_sizes
 
 logger = logging.getLogger(__name__)
 
-REPO_ROOT = Path(__file__).resolve().parent
 TASK_ID = "04"
 DEPENDS_ON = ["01", "03"]
 
@@ -60,9 +63,6 @@ CONTEXT = {
 	"started_at": {"@id": "prov:startedAtTime"},
 }
 
-INT_RE = re.compile(r"^-?[0-9]+$")
-FLOAT_RE = re.compile(r"^-?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
-
 MEASUREMENTS = [
 	# (csv file, json key, @type, instrument id, points key)
 	("xrf_map.csv", "xrf", "CompositionMeasurement", "m4_tornado", "points"),
@@ -71,61 +71,19 @@ MEASUREMENTS = [
 ]
 
 
-class ValidationError(RuntimeError):
-	pass
-
-
-def coerce(s: str) -> int | float | str:
-	if INT_RE.match(s):
-		return int(s)
-	if FLOAT_RE.match(s):
-		return float(s)
-	return s
-
-
 class JsonConverter:
 	def __init__(self, cfg: dict, csv_root: Path, json_root: Path) -> None:
 		self.cfg = cfg
-		self.csv_root = csv_root
 		self.json_root = json_root
-		self.manifest = self._load_manifest()
+		self.reader = CorpusReader(csv_root)
+		self.manifest = self.reader.manifest
 		self.graph: list[dict] = []  # hybrid metadata entities
 		self.full_bytes = 0
 		self.full_files = 0
 		self.counts = {"coupons": 0, "tracks": 0, "cycle_rows": 0, "loop_rows": 0, "xrf_rows": 0}
 
-	def _load_manifest(self) -> dict[str, tuple[int, int, str]]:
-		manifest: dict[str, tuple[int, int, str]] = {}
-		with open(self.csv_root / "MANIFEST.csv", encoding="ascii") as fh:
-			header = fh.readline().strip()
-			if header != "path,rows,bytes,sha256":
-				raise ValidationError(f"MANIFEST.csv: unexpected header {header!r}")
-			for line in fh:
-				path, rows, nbytes, sha = line.strip().split(",")
-				manifest[path] = (int(rows), int(nbytes), sha)
-		logger.info("manifest loaded: %d entries", len(manifest))
-		return manifest
-
-	def read_csv(self, relpath: str) -> tuple[list[str], list[list[str]]]:
-		"""Read one corpus file, verifying bytes and row count against MANIFEST."""
-		data = (self.csv_root / relpath).read_bytes()
-		exp_rows, exp_bytes, exp_sha = self.manifest[relpath]
-		if len(data) != exp_bytes or hashlib.sha256(data).hexdigest() != exp_sha:
-			raise ValidationError(f"{relpath}: bytes/sha256 mismatch vs MANIFEST")
-		lines = data.decode("ascii").split("\n")
-		header = lines[0].split(",")
-		rows = [ln.split(",") for ln in lines[1:] if ln]
-		if len(rows) != exp_rows:
-			raise ValidationError(f"{relpath}: {len(rows)} rows != manifest {exp_rows}")
-		return header, rows
-
 	def rows_as_dicts(self, relpath: str) -> list[dict]:
-		header, rows = self.read_csv(relpath)
-		keys = [h.lower() for h in header]
-		return [
-			{k: coerce(v) for k, v in zip(keys, row) if v != ""}
-			for row in rows
-		]
+		return self.reader.dicts(relpath)
 
 	def source_ref(self, relpath: str) -> dict:
 		rows, _, sha = self.manifest[relpath]
@@ -285,54 +243,17 @@ def validate_sample(paths: list[Path]) -> None:
 		logger.info("sample %s: valid JSON, %d triples via rdflib", path.name, len(g))
 
 
-def check_dependencies(out_root: Path) -> None:
-	for dep in DEPENDS_ON:
-		marker = out_root / ".done" / f"{dep}.ok"
-		if not marker.exists():
-			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
-
-
-def write_marker(marker_path: Path, conv: JsonConverter, hybrid_bytes: int) -> None:
-	lines = [
-		f"task: '{TASK_ID}'",
-		"status: ok",
-		f"full_files: {conv.full_files}",
-		f"full_bytes: {conv.full_bytes}",
-		f"hybrid_bytes: {hybrid_bytes}",
-		f"coupons: {conv.counts['coupons']}",
-		f"tracks: {conv.counts['tracks']}",
-		f"cycle_rows: {conv.counts['cycle_rows']}",
-		f"loop_points: {conv.counts['loop_rows']}",
-	]
-	marker_path.parent.mkdir(parents=True, exist_ok=True)
-	with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
-		fh.write("\n".join(lines) + "\n")
-
-
 def main() -> int:
-	parser = argparse.ArgumentParser(description="Task 04: convert the CSV corpus to JSON-LD")
-	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
-	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
-	args = parser.parse_args()
-	logging.basicConfig(
-		level=args.log_level,
-		stream=sys.stderr,
-		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
-	)
-
+	args = parse_task_args("Task 04: convert the CSV corpus to JSON-LD")
 	out_root: Path = args.out_root
 	json_root = out_root / "json"
-	marker_path = out_root / ".done" / f"{TASK_ID}.ok"
 	try:
-		check_dependencies(out_root)
-		if marker_path.exists():
-			logger.info("re-run: removing stale marker %s", marker_path)
-			marker_path.unlink()
+		check_dependencies(out_root, DEPENDS_ON)
+		remove_stale_marker(out_root, TASK_ID)
 		if json_root.exists():
 			logger.info("re-run: removing previous output %s", json_root)
 			shutil.rmtree(json_root)
-		with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
-			cfg = yaml.safe_load(fh)
+		cfg = load_lab_config(out_root)
 
 		conv = JsonConverter(cfg, out_root / "csv", json_root)
 		conv.convert_flat()
@@ -344,7 +265,15 @@ def main() -> int:
 			json_root / "hybrid" / "dataset.jsonld",
 		])
 		update_storage_sizes(out_root / "bench", "json", [("full", conv.full_bytes), ("hybrid", hybrid_bytes)])
-		write_marker(marker_path, conv, hybrid_bytes)
+		marker_path = write_marker(out_root, TASK_ID, {
+			"full_files": conv.full_files,
+			"full_bytes": conv.full_bytes,
+			"hybrid_bytes": hybrid_bytes,
+			"coupons": conv.counts["coupons"],
+			"tracks": conv.counts["tracks"],
+			"cycle_rows": conv.counts["cycle_rows"],
+			"loop_points": conv.counts["loop_rows"],
+		})
 	except Exception:
 		logger.critical("task %s failed", TASK_ID, exc_info=True)
 		return 1

convert_sqlite.py

@@ -0,0 +1,423 @@
+"""Task 05 - Convert the CSV corpus to SQLite.
+
+Builds ./out/sqlite/tribo.db: variant B key schema (composite natural PKs
+on bulk tables, WITHOUT ROWID, enforced foreign keys - see
+docs/research/Tribology_FK_Architecture.html), Q1-Q7 indexes created after
+the load, track_summary precomputed. Every consumed CSV is verified
+against MANIFEST.csv. Appends the db size to storage_sizes.csv and writes
+./out/.done/05.ok. Spec: docs/specs/05_convert_sqlite.md.
+"""
+
+from __future__ import annotations
+
+import logging
+import sqlite3
+import sys
+from pathlib import Path
+
+from common.corpus import CorpusReader, ValidationError
+from common.pipeline import (
+	check_dependencies,
+	load_lab_config,
+	parse_task_args,
+	remove_stale_marker,
+	write_marker,
+)
+from common.storage_sizes import update_storage_sizes
+from common.track_summary import summarize_track
+
+logger = logging.getLogger(__name__)
+
+TASK_ID = "05"
+DEPENDS_ON = ["01", "03"]
+
+BATCH_ROWS = 50000  # rows per executemany transaction, spec 05
+LOAD_PRAGMAS = [
+	"PRAGMA journal_mode=OFF",
+	"PRAGMA synchronous=OFF",
+	"PRAGMA cache_size=-262144",
+	"PRAGMA foreign_keys=ON",
+]
+
+DDL = [
+	"""CREATE TABLE instruments (
+		instrument_id INTEGER PRIMARY KEY,
+		instrument_code TEXT NOT NULL UNIQUE,
+		name TEXT NOT NULL,
+		role TEXT NOT NULL
+	)""",
+	"""CREATE TABLE batches (
+		batch_id INTEGER PRIMARY KEY,
+		batch_code TEXT NOT NULL UNIQUE,
+		pt_gun_tilt_deg INTEGER NOT NULL,
+		au_gun_tilt_deg INTEGER NOT NULL,
+		pt_power_w INTEGER NOT NULL,
+		au_power_w INTEGER NOT NULL,
+		pt_discharge_v INTEGER NOT NULL,
+		au_discharge_v INTEGER NOT NULL,
+		deposition_date TEXT NOT NULL
+	)""",
+	"""CREATE TABLE runs (
+		run_id INTEGER PRIMARY KEY,
+		run_code TEXT NOT NULL UNIQUE,
+		environment TEXT NOT NULL,
+		date TEXT NOT NULL,
+		plates INTEGER NOT NULL,
+		operator TEXT NOT NULL
+	)""",
+	"""CREATE TABLE wafers (
+		wafer_id INTEGER PRIMARY KEY,
+		wafer_code TEXT NOT NULL UNIQUE,
+		batch_id INTEGER NOT NULL REFERENCES batches(batch_id),
+		wafer_index INTEGER NOT NULL,
+		deposition_date TEXT NOT NULL,
+		coupons INTEGER NOT NULL,
+		friction_coupons INTEGER NOT NULL,
+		reserve_coupons INTEGER NOT NULL
+	)""",
+	"""CREATE TABLE coupons (
+		coupon_id INTEGER PRIMARY KEY,
+		coupon_code TEXT NOT NULL UNIQUE,
+		wafer_id INTEGER NOT NULL REFERENCES wafers(wafer_id),
+		batch_id INTEGER NOT NULL REFERENCES batches(batch_id),
+		grid_row INTEGER NOT NULL,
+		grid_col INTEGER NOT NULL,
+		thickness_um REAL NOT NULL,
+		ra_nm REAL NOT NULL,
+		au_wtpct_mean REAL NOT NULL,
+		run_id INTEGER REFERENCES runs(run_id),
+		plate INTEGER,
+		probe INTEGER,
+		square INTEGER
+	)""",
+	"""CREATE TABLE tracks (
+		track_id INTEGER PRIMARY KEY,
+		track_code TEXT NOT NULL UNIQUE,
+		coupon_id INTEGER NOT NULL REFERENCES coupons(coupon_id),
+		run_id INTEGER NOT NULL REFERENCES runs(run_id),
+		environment TEXT NOT NULL,
+		load_mn REAL NOT NULL,
+		stroke_mm REAL NOT NULL,
+		speed_mm_s REAL NOT NULL,
+		counterface_id TEXT NOT NULL,
+		started_at TEXT NOT NULL
+	)""",
+	"""CREATE TABLE simtra_profiles (
+		batch_id INTEGER NOT NULL REFERENCES batches(batch_id),
+		row_no INTEGER NOT NULL,
+		angle_deg REAL NOT NULL,
+		energy_ev REAL NOT NULL,
+		pt_flux REAL NOT NULL,
+		au_flux REAL NOT NULL,
+		PRIMARY KEY (batch_id, row_no)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE xrf_points (
+		coupon_id INTEGER NOT NULL REFERENCES coupons(coupon_id),
+		grid_x INTEGER NOT NULL,
+		grid_y INTEGER NOT NULL,
+		pt_wtpct REAL NOT NULL,
+		au_wtpct REAL NOT NULL,
+		PRIMARY KEY (coupon_id, grid_x, grid_y)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE profilometry_points (
+		coupon_id INTEGER NOT NULL REFERENCES coupons(coupon_id),
+		grid_x INTEGER NOT NULL,
+		grid_y INTEGER NOT NULL,
+		thickness_um REAL NOT NULL,
+		PRIMARY KEY (coupon_id, grid_x, grid_y)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE nanoindentation (
+		coupon_id INTEGER NOT NULL REFERENCES coupons(coupon_id),
+		indent_id INTEGER NOT NULL,
+		x_um REAL NOT NULL,
+		y_um REAL NOT NULL,
+		hardness_gpa REAL NOT NULL,
+		reduced_modulus_gpa REAL NOT NULL,
+		max_load_mn REAL NOT NULL,
+		PRIMARY KEY (coupon_id, indent_id)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE afm (
+		coupon_id INTEGER PRIMARY KEY REFERENCES coupons(coupon_id),
+		ra_nm REAL NOT NULL,
+		rq_nm REAL NOT NULL,
+		image_file TEXT NOT NULL
+	)""",
+	"""CREATE TABLE friction_cycles (
+		track_id INTEGER NOT NULL REFERENCES tracks(track_id),
+		cycle INTEGER NOT NULL,
+		cof REAL NOT NULL,
+		PRIMARY KEY (track_id, cycle)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE friction_loop_points (
+		track_id INTEGER NOT NULL REFERENCES tracks(track_id),
+		cycle INTEGER NOT NULL,
+		pt INTEGER NOT NULL,
+		position_um REAL NOT NULL,
+		friction_force_mn REAL NOT NULL,
+		PRIMARY KEY (track_id, cycle, pt)
+	) WITHOUT ROWID""",
+	"""CREATE TABLE wear (
+		track_id INTEGER PRIMARY KEY REFERENCES tracks(track_id),
+		wear_volume_um3 REAL NOT NULL,
+		k_archard REAL NOT NULL,
+		sliding_distance_m REAL NOT NULL
+	)""",
+	"""CREATE TABLE track_summary (
+		track_id INTEGER PRIMARY KEY REFERENCES tracks(track_id),
+		cof_ss_mean REAL NOT NULL,
+		cof_ss_std REAL NOT NULL,
+		run_in_cycles INTEGER NOT NULL
+	)""",
+]
+
+INDEX_DDL = [
+	"CREATE INDEX ix_coupons_au_wtpct_mean ON coupons(au_wtpct_mean)",  # Q2 selective filter
+	"CREATE INDEX ix_tracks_run_id ON tracks(run_id)",  # Q5/Q7 joins to runs
+	"CREATE INDEX ix_tracks_coupon_id ON tracks(coupon_id)",  # Q3/Q6 joins to coupons
+	"CREATE INDEX ix_runs_environment ON runs(environment)",  # Q7 grouping
+	"CREATE INDEX ix_tracks_environment_load ON tracks(environment, load_mn)",  # Q4 filter
+]
+
+INSERTS = {
+	"instruments": "INSERT INTO instruments VALUES (?,?,?,?)",
+	"batches": "INSERT INTO batches VALUES (?,?,?,?,?,?,?,?,?)",
+	"runs": "INSERT INTO runs VALUES (?,?,?,?,?,?)",
+	"wafers": "INSERT INTO wafers VALUES (?,?,?,?,?,?,?,?)",
+	"coupons": "INSERT INTO coupons VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)",
+	"tracks": "INSERT INTO tracks VALUES (?,?,?,?,?,?,?,?,?,?)",
+	"simtra_profiles": "INSERT INTO simtra_profiles VALUES (?,?,?,?,?,?)",
+	"xrf_points": "INSERT INTO xrf_points VALUES (?,?,?,?,?)",
+	"profilometry_points": "INSERT INTO profilometry_points VALUES (?,?,?,?)",
+	"nanoindentation": "INSERT INTO nanoindentation VALUES (?,?,?,?,?,?,?)",
+	"afm": "INSERT INTO afm VALUES (?,?,?,?)",
+	"friction_cycles": "INSERT INTO friction_cycles VALUES (?,?,?)",
+	"friction_loop_points": "INSERT INTO friction_loop_points VALUES (?,?,?,?,?)",
+	"wear": "INSERT INTO wear VALUES (?,?,?,?)",
+	"track_summary": "INSERT INTO track_summary VALUES (?,?,?,?)",
+}
+
+
+class SqliteLoader:
+	def __init__(self, cfg: dict, csv_root: Path, db_path: Path) -> None:
+		self.cfg = cfg
+		self.reader = CorpusReader(csv_root)
+		db_path.parent.mkdir(parents=True, exist_ok=True)
+		self.conn = sqlite3.connect(db_path)
+		for pragma in LOAD_PRAGMAS:
+			self.conn.execute(pragma)
+		for ddl in DDL:
+			self.conn.execute(ddl)
+		self.buffers: dict[str, list[tuple]] = {}
+		self.batch_ids: dict[str, int] = {}
+		self.run_ids: dict[str, int] = {}
+
+	def put(self, table: str, row: tuple) -> None:
+		buf = self.buffers.setdefault(table, [])
+		buf.append(row)
+		if len(buf) >= BATCH_ROWS:
+			self.flush(table)
+
+	def flush(self, table: str) -> None:
+		buf = self.buffers.get(table)
+		if buf:
+			self.conn.executemany(INSERTS[table], buf)
+			self.conn.commit()
+			buf.clear()
+
+	def flush_all(self) -> None:
+		for table in list(self.buffers):
+			self.flush(table)
+
+	# --- dimension + bulk loading, in FK dependency order ---
+
+	def load_flat(self) -> None:
+		for i, inst in enumerate(self.cfg["instruments"], 1):
+			self.put("instruments", (i, inst["instrument_id"], inst["name"], inst["role"]))
+		for i, row in enumerate(self.reader.dicts("batches.csv"), 1):
+			self.batch_ids[row["batch_code"]] = i
+			self.put("batches", (
+				i, row["batch_code"], row["pt_gun_tilt_deg"], row["au_gun_tilt_deg"],
+				row["pt_power_w"], row["au_power_w"], row["pt_discharge_v"],
+				row["au_discharge_v"], row["deposition_date"],
+			))
+		for i, row in enumerate(self.reader.dicts("runs.csv"), 1):
+			self.run_ids[row["run_code"]] = i
+			self.put("runs", (i, row["run_code"], row["environment"], row["date"], row["plates"], row["operator"]))
+		self.flush_all()
+		for code, batch_id in self.batch_ids.items():
+			for row_no, row in enumerate(self.reader.dicts(f"simtra/simtra_profile_{code}.csv"), 1):
+				self.put("simtra_profiles", (batch_id, row_no, row["angle_deg"], row["energy_ev"], row["pt_flux"], row["au_flux"]))
+
+	def load_hierarchy(self) -> None:
+		h = self.cfg["hierarchy"]
+		fa = self.cfg["friction_assignment"]
+		wafer_id = 0
+		coupon_id = 0
+		track_id = 0
+		for batch in self.cfg["deposition_matrix"]:
+			b_code = batch["batch_code"]
+			batch_id = self.batch_ids[b_code]
+			for w in range(1, h["wafers_per_batch"] + 1):
+				wafer_id += 1
+				wafer_dir = f"batch_{b_code}/wafer_W{w}"
+				info = self.reader.dicts(f"{wafer_dir}/wafer_info.csv")[0]
+				self.put("wafers", (
+					wafer_id, info["wafer_code"], batch_id, info["wafer_index"],
+					info["deposition_date"], info["coupons"], info["friction_coupons"], info["reserve_coupons"],
+				))
+				self.flush("wafers")
+				for c in range(1, h["coupons_per_wafer"] + 1):
+					coupon_id += 1
+					rel_dir = f"{wafer_dir}/coupon_C{c:02d}"
+					track_id = self.load_coupon(rel_dir, coupon_id, wafer_id, batch_id, track_id, fa)
+			logger.info("batch %s loaded (through coupon %d, track %d)", b_code, coupon_id, track_id)
+		self.flush_all()
+
+	def load_coupon(self, rel_dir: str, coupon_id: int, wafer_id: int, batch_id: int, track_id: int, fa: dict) -> int:
+		info = self.reader.dicts(f"{rel_dir}/coupon_info.csv")[0]
+		is_friction = info["run_code"] != "RESERVE"
+		run_id = self.run_ids[info["run_code"]] if is_friction else None
+		self.put("coupons", (
+			coupon_id, info["coupon_code"], wafer_id, batch_id,
+			info["grid_row"], info["grid_col"], info["thickness_um"], info["ra_nm"],
+			info["au_wtpct_mean"], run_id,
+			info.get("plate"), info.get("probe"), info.get("square"),
+		))
+		self.flush("coupons")
+
+		for row in self.reader.dicts(f"{rel_dir}/xrf_map.csv"):
+			self.put("xrf_points", (coupon_id, row["grid_x"], row["grid_y"], row["pt_wtpct"], row["au_wtpct"]))
+		for row in self.reader.dicts(f"{rel_dir}/profilometry.csv"):
+			self.put("profilometry_points", (coupon_id, row["grid_x"], row["grid_y"], row["thickness_um"]))
+		for row in self.reader.dicts(f"{rel_dir}/nanoindentation.csv"):
+			self.put("nanoindentation", (
+				coupon_id, row["indent_id"], row["x_um"], row["y_um"],
+				row["hardness_gpa"], row["reduced_modulus_gpa"], row["max_load_mn"],
+			))
+		afm = self.reader.dicts(f"{rel_dir}/afm.csv")[0]
+		self.put("afm", (coupon_id, afm["ra_nm"], afm["rq_nm"], afm["image_file"]))
+
+		if is_friction:
+			for t in range(1, fa["tracks_per_friction_coupon"] + 1):
+				track_id += 1
+				self.load_track(f"{rel_dir}/track_T{t}", track_id, coupon_id)
+		return track_id
+
+	def load_track(self, rel_dir: str, track_id: int, coupon_id: int) -> None:
+		info = self.reader.dicts(f"{rel_dir}/track_info.csv")[0]
+		self.put("tracks", (
+			track_id, info["track_code"], coupon_id, self.run_ids[info["run_code"]],
+			info["environment"], info["load_mn"], info["stroke_mm"], info["speed_mm_s"],
+			info["counterface_id"], info["started_at"],
+		))
+		self.flush("tracks")
+
+		cofs: list[float] = []
+		for row in self.reader.dicts(f"{rel_dir}/cof_vs_cycle.csv"):
+			cofs.append(row["cof"])
+			self.put("friction_cycles", (track_id, row["cycle"], row["cof"]))
+		ss_mean, ss_std, run_in = summarize_track(cofs)
+		self.put("track_summary", (track_id, ss_mean, ss_std, run_in))
+
+		pt = 0
+		last_cycle = None
+		for row in self.reader.dicts(f"{rel_dir}/friction_loops.csv"):
+			pt = pt + 1 if row["cycle"] == last_cycle else 1
+			last_cycle = row["cycle"]
+			self.put("friction_loop_points", (track_id, row["cycle"], pt, row["position_um"], row["friction_force_mn"]))
+
+		wear = self.reader.dicts(f"{rel_dir}/wear.csv")[0]
+		self.put("wear", (track_id, wear["wear_volume_um3"], wear["k_archard"], wear["sliding_distance_m"]))
+
+	# --- post-load ---
+
+	def finalize(self) -> None:
+		for ddl in INDEX_DDL:
+			self.conn.execute(ddl)
+		self.conn.commit()
+		self.conn.execute("PRAGMA journal_mode=WAL")
+		self.conn.execute("ANALYZE")
+		self.conn.execute("VACUUM")
+		logger.info("indexes created, WAL enabled, ANALYZE + VACUUM done")
+
+	def validate(self) -> dict[str, int]:
+		cfg = self.cfg
+		v = cfg["volumes"]
+		expected = {
+			"instruments": len(cfg["instruments"]),
+			"batches": cfg["hierarchy"]["batches"],
+			"runs": cfg["friction_assignment"]["runs"],
+			"wafers": cfg["hierarchy"]["batches"] * cfg["hierarchy"]["wafers_per_batch"],
+			"coupons": v["coupons_total"],
+			"tracks": v["tracks_total"],
+			"simtra_profiles": cfg["hierarchy"]["batches"] * v["simtra_rows_per_batch"],
+			"xrf_points": v["xrf_points_total"],
+			"profilometry_points": v["coupons_total"] * v["profilometry_points_per_coupon"],
+			"nanoindentation": v["coupons_total"] * v["nanoindentation_indents_per_coupon"],
+			"afm": v["coupons_total"],
+			"friction_cycles": v["cycle_rows_total"],
+			"friction_loop_points": v["loop_points_total"],
+			"wear": v["tracks_total"],
+			"track_summary": v["tracks_total"],
+		}
+		counts: dict[str, int] = {}
+		for table, exp in expected.items():
+			got = self.conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]
+			counts[table] = got
+			if got != exp:
+				raise ValidationError(f"row count mismatch: {table}: loaded {got} != expected {exp}")
+		reserve = self.conn.execute("SELECT COUNT(*) FROM coupons WHERE run_id IS NULL").fetchone()[0]
+		if reserve != self.cfg["friction_assignment"]["reserve_coupons_total"]:
+			raise ValidationError(f"reserve coupons: {reserve} != expected")
+		fk_violations = self.conn.execute("PRAGMA foreign_key_check").fetchall()
+		if fk_violations:
+			raise ValidationError(f"foreign_key_check reported {len(fk_violations)} violations")
+		logger.info("all %d table counts match, %d reserve coupons, foreign_key_check clean", len(expected), reserve)
+		return counts
+
+
+def main() -> int:
+	args = parse_task_args("Task 05: convert the CSV corpus to SQLite")
+	out_root: Path = args.out_root
+	db_path = out_root / "sqlite" / "tribo.db"
+	try:
+		check_dependencies(out_root, DEPENDS_ON)
+		remove_stale_marker(out_root, TASK_ID)
+		for stale in (db_path, db_path.with_suffix(".db-wal"), db_path.with_suffix(".db-shm")):
+			if stale.exists():
+				logger.info("re-run: removing %s", stale)
+				stale.unlink()
+		cfg = load_lab_config(out_root)
+
+		loader = SqliteLoader(cfg, out_root / "csv", db_path)
+		loader.load_flat()
+		loader.load_hierarchy()
+		loader.finalize()
+		counts = loader.validate()
+		loader.conn.close()
+
+		db_bytes = db_path.stat().st_size
+		update_storage_sizes(out_root / "bench", "sqlite", [("db", db_bytes)])
+		marker_path = write_marker(out_root, TASK_ID, {
+			"db_file": db_path.as_posix(),
+			"db_bytes": db_bytes,
+			"tables": len(counts),
+			"total_rows": sum(counts.values()),
+			"friction_cycles": counts["friction_cycles"],
+			"friction_loop_points": counts["friction_loop_points"],
+			"tracks": counts["tracks"],
+			"track_summary": counts["track_summary"],
+		})
+	except Exception:
+		logger.critical("task %s failed", TASK_ID, exc_info=True)
+		return 1
+
+	print(f"task 05 ok: {db_path} ({db_bytes / (1024 * 1024):.1f} MiB, {len(counts)} tables, {sum(counts.values())} rows)")
+	print(f"bulk rows: cycles={counts['friction_cycles']} loop_points={counts['friction_loop_points']} "
+		f"xrf={counts['xrf_points']} track_summary={counts['track_summary']}")
+	print(f"marker: {marker_path}")
+	return 0
+
+
+if __name__ == "__main__":
+	sys.exit(main())

docs/research/Tribology_FK_Architecture.html

@@ -0,0 +1,571 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Foreign-Key Architecture Study - Tribology Lab Data Storage</title>
+<link rel="preconnect" href="https://fonts.googleapis.com">
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
+<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600;700&display=swap" rel="stylesheet">
+<style>
+  *{box-sizing:border-box}
+  html,body{margin:0;padding:0}
+  body{font-family:"IBM Plex Sans",system-ui,sans-serif;color:#16181d;background:#eef0f2;-webkit-font-smoothing:antialiased}
+  ::-webkit-scrollbar{width:10px;height:10px}
+  ::-webkit-scrollbar-thumb{background:#cfd4da;border-radius:6px;border:2px solid #eef0f2}
+  ::-webkit-scrollbar-track{background:transparent}
+  .mono{font-family:"IBM Plex Mono",monospace}
+
+  .app{display:flex;flex-direction:column;height:100vh;width:100%;overflow:hidden;background:#eef0f2}
+  header.top{display:flex;align-items:center;gap:14px;height:56px;flex:none;padding:0 22px;background:#fff;border-bottom:1px solid #e2e5ea}
+  .logo{display:flex;align-items:center;justify-content:center;width:30px;height:30px;border-radius:7px;background:#16181d;color:#fff;font-family:"IBM Plex Mono",monospace;font-weight:600;font-size:12px}
+  .htitle{display:flex;flex-direction:column;line-height:1.25;white-space:nowrap}
+  .htitle b{font-size:14.5px;font-weight:600;letter-spacing:-.01em}
+  .htitle span{font-size:11.5px;color:#838a95}
+  .hright{margin-left:auto;display:flex;align-items:center;gap:8px}
+  .chip{font-family:"IBM Plex Mono",monospace;font-size:11px;color:#838a95;background:#f3f5f7;border:1px solid #e6e8ec;padding:4px 10px;border-radius:6px}
+  .chip-ok{display:inline-flex;align-items:center;gap:6px;font-size:12px;color:#0f7a45;background:#e7f6ee;padding:4px 10px;border-radius:6px;font-weight:600}
+  .chip-ok i{width:7px;height:7px;border-radius:7px;background:#18a05a}
+
+  .frame{display:flex;flex:1;min-height:0}
+  nav.side{width:248px;flex:none;background:#fff;border-right:1px solid #e2e5ea;padding:14px 12px;display:flex;flex-direction:column;overflow-y:auto}
+  .navcap{font-family:"IBM Plex Mono",monospace;font-size:10px;letter-spacing:.1em;color:#a4abb4;padding:6px 11px 8px}
+  nav.side a{display:flex;align-items:center;gap:11px;width:100%;padding:8px 11px;border-radius:8px;cursor:pointer;text-decoration:none;font-size:13.5px;margin:1px 0;color:#475059;font-weight:500}
+  nav.side a .tag{width:24px;height:19px;border-radius:5px;display:flex;align-items:center;justify-content:center;font-family:"IBM Plex Mono",monospace;font-size:9px;font-weight:600;flex:none;background:#eceef1;color:#8a909b}
+  nav.side a.active{background:#eaf1fe;color:#16181d;font-weight:600}
+  nav.side a.active .tag{background:#1f5fdb;color:#fff}
+  .sidefoot{margin-top:auto;padding:14px 11px 6px;border-top:1px solid #eef0f2}
+  .sidefoot .cap{font-family:"IBM Plex Mono",monospace;font-size:10px;letter-spacing:.08em;color:#a4abb4;margin-bottom:8px}
+  .sidefoot .txt{font-size:12px;color:#6b727c;line-height:1.6}
+
+  main{flex:1;min-width:0;overflow-y:auto;padding:26px 30px 60px;scroll-behavior:smooth}
+  section{margin-bottom:44px;scroll-margin-top:10px}
+  h1{margin:0;font-size:21px;font-weight:600;letter-spacing:-.02em}
+  h2{margin:0 0 4px;font-size:18px;font-weight:600;letter-spacing:-.02em}
+  .sub{margin:4px 0 18px;font-size:13px;color:#838a95;max-width:860px;line-height:1.55}
+  .card{background:#fff;border:1px solid #e6e8ec;border-radius:11px;padding:18px 20px;margin-bottom:14px}
+  .card h3{margin:0 0 10px;font-size:14px;font-weight:600}
+  .card p{margin:0 0 10px;font-size:13px;color:#3d424b;line-height:1.6;max-width:880px}
+  .card p:last-child{margin-bottom:0}
+  .card ul{margin:6px 0 4px;padding-left:20px}
+  .card li{font-size:13px;color:#3d424b;line-height:1.65;margin-bottom:6px;max-width:840px}
+  .muted{color:#838a95}
+  .kpis{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin-bottom:16px}
+  .kpi{background:#fff;border:1px solid #e6e8ec;border-radius:10px;padding:15px 16px}
+  .kpi .l{font-size:11.5px;color:#838a95;font-weight:500;text-transform:uppercase;letter-spacing:.04em}
+  .kpi .v{font-family:"IBM Plex Mono",monospace;font-size:26px;font-weight:600;margin-top:7px;line-height:1}
+  .kpi .s{font-size:11.5px;color:#a4abb4;margin-top:5px}
+  .v-bad{color:#b23b4e}.v-good{color:#0f7a45}.v-neutral{color:#16181d}
+
+  table.data{width:100%;border-collapse:collapse}
+  table.data th{font-size:10.5px;text-transform:uppercase;letter-spacing:.04em;color:#838a95;font-weight:600;text-align:left;padding:9px 12px;background:#f7f8fa;border-bottom:1px solid #eef0f2}
+  table.data td{font-size:12.5px;color:#3d424b;padding:11px 12px;border-bottom:1px solid #f4f5f7;vertical-align:top}
+  table.data td.mono, table.data th.num{font-family:"IBM Plex Mono",monospace}
+  table.data th.num, table.data td.num{text-align:right;font-family:"IBM Plex Mono",monospace}
+  .delta-bad{color:#b23b4e;font-weight:600}
+  .delta-ok{color:#0f7a45;font-weight:600}
+  .delta-zero{color:#838a95}
+
+  pre.code{background:#16181d;color:#dfe3e9;border-radius:9px;padding:14px 16px;font-family:"IBM Plex Mono",monospace;font-size:12px;line-height:1.65;overflow-x:auto;margin:12px 0 0}
+  pre.code .c{color:#7e8794}
+  pre.code .k{color:#7fb1ff}
+  pre.code .hl{color:#ff9db0}
+
+  .erd-wrap{overflow-x:auto;padding:8px 0}
+  .erd{position:relative;margin:0 auto}
+  .erd svg.wire{position:absolute;inset:0;pointer-events:none}
+  .tbl{position:absolute;background:#fff;border:1px solid #d4dae2;border-radius:9px;box-shadow:0 1px 4px rgba(20,30,50,.05);overflow:hidden}
+  .tbl.bulk{border:2px solid #1f5fdb;box-shadow:0 3px 12px rgba(31,95,219,.12)}
+  .tbl-h{background:#16181d;color:#fff;padding:9px 13px;font-family:"IBM Plex Mono",monospace;font-size:12px;font-weight:600;display:flex;justify-content:space-between;gap:8px}
+  .tbl.bulk .tbl-h{background:#1f5fdb}
+  .tbl-h .sub2{color:#7e8794;font-weight:400}
+  .tbl.bulk .tbl-h .sub2{color:#bcd2f7}
+  .tr{display:flex;justify-content:space-between;gap:8px;padding:6px 13px;font-size:11.5px;color:#3d424b;align-items:center}
+  .tr .f{font-family:"IBM Plex Mono",monospace}
+  .tr .t{font-family:"IBM Plex Mono",monospace;font-weight:600;font-size:10.5px;white-space:nowrap}
+  .tr.pk{background:#eaf1fe}.tr.pk .t{color:#1f5fdb}
+  .tr.fk{background:#fbf2e8}.tr.fk .t{color:#c08428}
+  .tr.ref .t{color:#9aa1ab}
+  .tr.idx{background:#fbe9ec}.tr.idx .t{color:#b23b4e}
+  .tr .ty{color:#a4abb4}
+  .notecard{position:absolute;background:#f7f8fa;border:1px dashed #c9cfd8;border-radius:9px;padding:12px 14px;font-size:11.5px;color:#5b6472;line-height:1.55}
+  .notecard b{font-family:"IBM Plex Mono",monospace;font-size:11px;color:#3d424b}
+
+  .hbars{display:flex;flex-direction:column;gap:12px;margin-top:6px}
+  .hbar{display:flex;align-items:center;gap:14px}
+  .hbar .lab{width:250px;flex:none;font-size:12.5px;color:#3d424b}
+  .hbar .lab small{display:block;color:#a4abb4;font-size:11px}
+  .hbar .trk{flex:1;background:#f1f3f6;border-radius:6px;overflow:hidden}
+  .hbar .fill{height:14px;border-radius:6px}
+  .hbar .val{font-family:"IBM Plex Mono",monospace;font-size:12.5px;color:#16181d;width:120px;flex:none;text-align:right}
+  .col-a{background:#9aa6b6}.col-b{background:#1f5fdb}.col-c{background:#c0476b}
+  .legend{display:flex;gap:16px;flex-wrap:wrap;margin:2px 0 10px}
+  .legend span{display:inline-flex;align-items:center;gap:6px;font-size:11.5px;color:#5b6472}
+  .legend i{width:10px;height:10px;border-radius:3px;display:inline-block}
+
+  .callout{border-left:3px solid #1f5fdb;background:#f5f8fe;border-radius:0 9px 9px 0;padding:12px 16px;font-size:13px;color:#2b3038;line-height:1.6;margin:12px 0;max-width:880px}
+  .callout.warn{border-left-color:#b23b4e;background:#fdf3f5}
+  .pill{display:inline-flex;align-items:center;gap:6px;padding:2px 9px;border-radius:20px;font-size:11px;font-weight:600}
+  .pill.rec{color:#0f7a45;background:#e7f6ee}
+  .pill.base{color:#5b6472;background:#eef0f2}
+  .pill.costly{color:#b23b4e;background:#fbe9ec}
+  .foot{font-size:11.5px;color:#a4abb4;border-top:1px solid #e6e8ec;padding-top:14px;margin-top:30px;line-height:1.7}
+</style>
+</head>
+<body>
+<div class="app">
+
+  <header class="top">
+    <div class="logo">FK</div>
+    <div class="htitle">
+      <b>Foreign-Key Architecture Study</b>
+      <span>Tribology lab data storage &middot; bulk-table key design</span>
+    </div>
+    <div class="hright">
+      <span class="chip">SQLite &middot; 4.32M rows measured</span>
+      <span class="chip-ok"><i></i>Measured 2026-07-11</span>
+    </div>
+  </header>
+
+  <div class="frame">
+
+    <nav class="side" id="sidenav">
+      <div class="navcap">NAVIGATE</div>
+      <a href="#overview" class="active"><span class="tag">OV</span><span>Overview</span></a>
+      <a href="#dataset"><span class="tag">DS</span><span>Sample Dataset</span></a>
+      <a href="#variant-a"><span class="tag">A</span><span>Variant A &middot; Composite PK</span></a>
+      <a href="#variant-b"><span class="tag">B</span><span>Variant B &middot; Enforced FK</span></a>
+      <a href="#variant-c"><span class="tag">C</span><span>Variant C &middot; Surrogate id</span></a>
+      <a href="#method"><span class="tag">BM</span><span>Benchmark Method</span></a>
+      <a href="#storage"><span class="tag">ST</span><span>Storage &amp; Load</span></a>
+      <a href="#reads"><span class="tag">RD</span><span>Read Performance</span></a>
+      <a href="#analysis"><span class="tag">AN</span><span>Why It Differs</span></a>
+      <a href="#verdict"><span class="tag">RC</span><span>Recommendation</span></a>
+      <div class="sidefoot">
+        <div class="cap">CONTEXT</div>
+        <div class="txt">Key-schema study for the bulk measurement tables of the LabDataStorageEvaluation pipeline (SQLite / PostgreSQL benchmark targets).</div>
+      </div>
+    </nav>
+
+    <main id="main">
+
+      <!-- ================= OVERVIEW ================= -->
+      <section id="overview">
+        <h1>Overview</h1>
+        <p class="sub">Three candidate key schemas for the two largest measurement tables (<span class="mono">friction_cycles</span>, 1.44M rows and <span class="mono">friction_loop_points</span>, 2.88M rows) were built from the same simulated corpus and measured for disk footprint, load time and six read patterns. In every variant the <b>relations between tables are single-column integer foreign keys</b> (<span class="mono">track_id</span>, <span class="mono">coupon_id</span>); the variants differ only in how a bulk row is identified inside its own table.</p>
+
+        <div class="kpis">
+          <div class="kpi"><div class="l">Disk, variant C vs A</div><div class="v v-bad">+74%</div><div class="s">190.2 vs 109.1 MiB</div></div>
+          <div class="kpi"><div class="l">GROUP BY scan, C vs A</div><div class="v v-bad">+59%</div><div class="s">135.9 vs 85.7 ms (Q5 pattern)</div></div>
+          <div class="kpi"><div class="l">Enforced FK cost (B)</div><div class="v v-good">+0.5 s</div><div class="s">load only; disk and reads = A</div></div>
+          <div class="kpi"><div class="l">Point lookups</div><div class="v v-neutral">&plusmn;2%</div><div class="s">identical across A / B / C</div></div>
+        </div>
+
+        <div class="card">
+          <h3>The three variants</h3>
+          <ul>
+            <li><b>Variant A - composite natural PK.</b> Row identity of a bulk row is its natural composite key <span class="mono">(track_id, cycle)</span>, stored as a clustered <span class="mono">WITHOUT ROWID</span> table. FK columns are plain integers, validated by the loader against the corpus manifest (no engine constraint).</li>
+            <li><b>Variant B - composite natural PK + enforced FK.</b> Same physical layout as A, plus <span class="mono">REFERENCES</span> constraints with <span class="mono">PRAGMA foreign_keys=ON</span>, so the engine itself guarantees referential integrity.</li>
+            <li><b>Variant C - surrogate key on every table.</b> Classic style: each table gets a single-column auto-increment key named <span class="mono">&lt;singular&gt;_id</span> (<span class="mono">friction_cycle_id</span>, never a bare <span class="mono">id</span>). Correctness then requires an additional <span class="mono">UNIQUE(track_id, cycle)</span> index - a second B-tree.</li>
+          </ul>
+          <div class="callout">Referential integrity is <b>not</b> what differs between the variants: every relation is a single integer FK in all three. What differs is the number of B-trees the engine must store and traverse per bulk table: one (A, B) or two (C).</div>
+        </div>
+      </section>
+
+      <!-- ================= DATASET ================= -->
+      <section id="dataset">
+        <h2>Sample Dataset</h2>
+        <p class="sub">All measurements ran against the real generated corpus of the evaluation pipeline - deterministic, physically plausible tribology data (seed 20260711). The two bulk tables and their parent dimension were loaded in full.</p>
+        <div class="card" style="padding:0;overflow:hidden">
+          <table class="data">
+            <thead><tr><th>Table</th><th>Grain</th><th class="num">Rows loaded</th><th>Columns (beyond keys)</th><th>Source files</th></tr></thead>
+            <tbody>
+              <tr><td class="mono">tracks</td><td>one friction track (3 per tested coupon)</td><td class="num">1,440</td><td>track_code, coupon_id, run_id, environment, load_mN, ...</td><td class="mono">track_info.csv &times; 1,440</td></tr>
+              <tr><td class="mono">friction_cycles</td><td>one reciprocating cycle of one track</td><td class="num">1,440,000</td><td>cof</td><td class="mono">cof_vs_cycle.csv &times; 1,440</td></tr>
+              <tr><td class="mono">friction_loop_points</td><td>one position-resolved loop point (10 loops &times; 200 pts per track)</td><td class="num">2,880,000</td><td>position_um, friction_force_mN</td><td class="mono">friction_loops.csv &times; 1,440</td></tr>
+            </tbody>
+          </table>
+        </div>
+        <div class="card">
+          <h3>Corpus context</h3>
+          <p>The corpus models a Pt-Au coating study: 4 deposition batches &rarr; 12 wafers &rarr; 588 coupons (480 friction-tested) &rarr; 1,440 tracks &rarr; 1.44M cycle records. Generated size: 141.8 MiB of CSV; the production target for extrapolation is 600 GB - 6 TB, which is why per-row bytes on the bulk tables matter.</p>
+        </div>
+      </section>
+
+      <!-- ================= VARIANT A ================= -->
+      <section id="variant-a">
+        <h2>Variant A &middot; Composite Natural Primary Key <span class="pill base" style="vertical-align:2px">baseline</span></h2>
+        <p class="sub">The bulk row is identified by what it physically is: cycle <span class="mono">n</span> of track <span class="mono">t</span>. The composite PK doubles as the clustering order, so the table itself is the only B-tree. FK columns are unenforced integers; integrity is guaranteed by row-count and checksum validation against the corpus manifest.</p>
+
+        <div class="card">
+          <div class="erd-wrap"><div class="erd" style="width:1000px;height:440px">
+            <svg class="wire" viewBox="0 0 1000 440" width="1000" height="440">
+              <path d="M270 200 H360 V77 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <path d="M270 240 H360 V269 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <circle cx="270" cy="200" r="3.5" fill="#1f5fdb"/><circle cx="270" cy="240" r="3.5" fill="#1f5fdb"/>
+              <text x="280" y="193" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="280" y="233" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="432" y="70" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+              <text x="432" y="262" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+            </svg>
+            <div class="tbl" style="left:20px;top:120px;width:250px">
+              <div class="tbl-h"><span>tracks</span><span class="sub2">dimension</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">track_code</span><span class="t" style="color:#1f5fdb">UK</span></div>
+              <div class="tr fk"><span class="f">coupon_id</span><span class="t">FK</span></div>
+              <div class="tr fk"><span class="f">run_id</span><span class="t">FK</span></div>
+              <div class="tr"><span class="f">environment</span><span class="ty">text</span></div>
+              <div class="tr"><span class="f">load_mn</span><span class="ty">real</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:20px;width:280px">
+              <div class="tbl-h"><span>friction_cycles</span><span class="sub2">1.44M rows</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK &middot; ref</span></div>
+              <div class="tr pk"><span class="f">cycle</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">cof</span><span class="ty">real</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:185px;width:280px">
+              <div class="tbl-h"><span>friction_loop_points</span><span class="sub2">2.88M rows</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK &middot; ref</span></div>
+              <div class="tr pk"><span class="f">cycle</span><span class="t">PK</span></div>
+              <div class="tr pk"><span class="f">pt</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">position_um</span><span class="ty">real</span></div>
+              <div class="tr"><span class="f">friction_force_mn</span><span class="ty">real</span></div>
+            </div>
+            <div class="notecard" style="left:770px;top:130px;width:210px">
+              <b>wear</b> and <b>track_summary</b> follow the same pattern: single-column PK = FK <b>track_id</b> (1:1 with tracks).<br><br>
+              <b>ref</b> = integer reference to tracks(track_id), validated by the loader, no engine constraint.
+            </div>
+          </div></div>
+          <pre class="code">CREATE TABLE friction_cycles (
+  track_id INTEGER NOT NULL,   <span class="c">-- reference to tracks(track_id), loader-validated</span>
+  cycle    INTEGER NOT NULL,
+  cof      REAL    NOT NULL,
+  <span class="k">PRIMARY KEY (track_id, cycle)</span>
+) <span class="k">WITHOUT ROWID</span>;              <span class="c">-- clustered: the PK is the table's only B-tree</span></pre>
+        </div>
+      </section>
+
+      <!-- ================= VARIANT B ================= -->
+      <section id="variant-b">
+        <h2>Variant B &middot; Composite PK + Enforced Foreign Keys <span class="pill rec" style="vertical-align:2px">recommended</span></h2>
+        <p class="sub">Physically identical to Variant A - same clustered layout, same single B-tree per bulk table. The only change: FK columns carry real <span class="mono">REFERENCES</span> constraints and every connection runs <span class="mono">PRAGMA foreign_keys=ON</span>, so the DBMS itself rejects an orphan row. Measured cost: +0.5 s of load time on 4.32M rows; zero cost on disk and reads.</p>
+
+        <div class="card">
+          <div class="erd-wrap"><div class="erd" style="width:1000px;height:440px">
+            <svg class="wire" viewBox="0 0 1000 440" width="1000" height="440">
+              <path d="M270 200 H360 V77 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <path d="M270 240 H360 V269 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <circle cx="270" cy="200" r="3.5" fill="#1f5fdb"/><circle cx="270" cy="240" r="3.5" fill="#1f5fdb"/>
+              <text x="280" y="193" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="280" y="233" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="432" y="70" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+              <text x="432" y="262" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+            </svg>
+            <div class="tbl" style="left:20px;top:120px;width:250px">
+              <div class="tbl-h"><span>tracks</span><span class="sub2">dimension</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">track_code</span><span class="t" style="color:#1f5fdb">UK</span></div>
+              <div class="tr fk"><span class="f">coupon_id</span><span class="t">FK</span></div>
+              <div class="tr fk"><span class="f">run_id</span><span class="t">FK</span></div>
+              <div class="tr"><span class="f">environment</span><span class="ty">text</span></div>
+              <div class="tr"><span class="f">load_mn</span><span class="ty">real</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:20px;width:280px">
+              <div class="tbl-h"><span>friction_cycles</span><span class="sub2">1.44M rows</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK</span></div>
+              <div class="tr fk"><span class="f">track_id &rarr; tracks</span><span class="t">FK &#10003;</span></div>
+              <div class="tr pk"><span class="f">cycle</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">cof</span><span class="ty">real</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:190px;width:280px">
+              <div class="tbl-h"><span>friction_loop_points</span><span class="sub2">2.88M rows</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK</span></div>
+              <div class="tr fk"><span class="f">track_id &rarr; tracks</span><span class="t">FK &#10003;</span></div>
+              <div class="tr pk"><span class="f">cycle</span><span class="t">PK</span></div>
+              <div class="tr pk"><span class="f">pt</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">position_um, friction_force_mn</span><span class="ty">real</span></div>
+            </div>
+            <div class="notecard" style="left:770px;top:130px;width:210px">
+              <b>PRAGMA foreign_keys = ON</b> on every connection - SQLite silently ignores FK clauses without it.<br><br>
+              FK checks are lookups into the already-indexed parent PK, which is why the measured overhead is only ~6% of load time.
+            </div>
+          </div></div>
+          <pre class="code">PRAGMA foreign_keys = ON;
+
+CREATE TABLE friction_cycles (
+  track_id INTEGER NOT NULL <span class="k">REFERENCES tracks(track_id)</span>,
+  cycle    INTEGER NOT NULL,
+  cof      REAL    NOT NULL,
+  <span class="k">PRIMARY KEY (track_id, cycle)</span>
+) <span class="k">WITHOUT ROWID</span>;</pre>
+        </div>
+      </section>
+
+      <!-- ================= VARIANT C ================= -->
+      <section id="variant-c">
+        <h2>Variant C &middot; Surrogate <span class="mono" style="font-size:16px">&lt;singular&gt;_id</span> on Every Table <span class="pill costly" style="vertical-align:2px">+74% disk</span></h2>
+        <p class="sub">The uniform classic design: every table, including the bulk fact tables, gets a single-column auto-increment key (<span class="mono">friction_cycle_id</span>, <span class="mono">friction_loop_point_id</span> - by convention always <span class="mono">&lt;table name in singular&gt;_id</span>, never a bare <span class="mono">id</span>). In SQLite the id column itself is nearly free (it aliases the rowid), but correctness still demands <span class="mono">UNIQUE(track_id, cycle)</span> - and that unique index is a second B-tree holding a copy of almost every column.</p>
+
+        <div class="card">
+          <div class="erd-wrap"><div class="erd" style="width:1000px;height:520px">
+            <svg class="wire" viewBox="0 0 1000 520" width="1000" height="520">
+              <path d="M270 240 H360 V100 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <path d="M270 280 H360 V345 H450" stroke="#c5cdd8" stroke-width="1.5" fill="none"/>
+              <circle cx="270" cy="240" r="3.5" fill="#1f5fdb"/><circle cx="270" cy="280" r="3.5" fill="#1f5fdb"/>
+              <text x="280" y="233" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="280" y="273" font-family="IBM Plex Mono, monospace" font-size="11" fill="#1f5fdb">1</text>
+              <text x="432" y="93" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+              <text x="432" y="338" font-family="IBM Plex Mono, monospace" font-size="11" fill="#9aa1ab">&#8734;</text>
+            </svg>
+            <div class="tbl" style="left:20px;top:160px;width:250px">
+              <div class="tbl-h"><span>tracks</span><span class="sub2">dimension</span></div>
+              <div class="tr pk"><span class="f">track_id</span><span class="t">PK</span></div>
+              <div class="tr"><span class="f">track_code</span><span class="t" style="color:#1f5fdb">UK</span></div>
+              <div class="tr fk"><span class="f">coupon_id</span><span class="t">FK</span></div>
+              <div class="tr fk"><span class="f">run_id</span><span class="t">FK</span></div>
+              <div class="tr"><span class="f">environment</span><span class="ty">text</span></div>
+              <div class="tr"><span class="f">load_mn</span><span class="ty">real</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:20px;width:280px">
+              <div class="tbl-h"><span>friction_cycles</span><span class="sub2">1.44M rows</span></div>
+              <div class="tr pk"><span class="f">friction_cycle_id</span><span class="t">PK</span></div>
+              <div class="tr fk"><span class="f">track_id &rarr; tracks</span><span class="t">FK &#10003;</span></div>
+              <div class="tr"><span class="f">cycle</span><span class="ty">int</span></div>
+              <div class="tr"><span class="f">cof</span><span class="ty">real</span></div>
+              <div class="tr idx"><span class="f">UNIQUE (track_id, cycle)</span><span class="t">2nd B-tree</span></div>
+            </div>
+            <div class="tbl bulk" style="left:450px;top:235px;width:280px">
+              <div class="tbl-h"><span>friction_loop_points</span><span class="sub2">2.88M rows</span></div>
+              <div class="tr pk"><span class="f">friction_loop_point_id</span><span class="t">PK</span></div>
+              <div class="tr fk"><span class="f">track_id &rarr; tracks</span><span class="t">FK &#10003;</span></div>
+              <div class="tr"><span class="f">cycle</span><span class="ty">int</span></div>
+              <div class="tr"><span class="f">pt</span><span class="ty">int</span></div>
+              <div class="tr"><span class="f">position_um</span><span class="ty">real</span></div>
+              <div class="tr"><span class="f">friction_force_mn</span><span class="ty">real</span></div>
+              <div class="tr idx"><span class="f">UNIQUE (track_id, cycle, pt)</span><span class="t">2nd B-tree</span></div>
+            </div>
+            <div class="notecard" style="left:770px;top:180px;width:210px">
+              The surrogate key is <b>never referenced</b> by any other table - no row in the schema points at an individual cycle.<br><br>
+              The UNIQUE index is <b>mandatory for correctness</b> (it prevents duplicate cycles), so its cost cannot be avoided in this variant.
+            </div>
+          </div></div>
+          <pre class="code">CREATE TABLE friction_cycles (
+  <span class="hl">friction_cycle_id INTEGER PRIMARY KEY</span>,        <span class="c">-- rowid alias; unused by any FK</span>
+  track_id INTEGER NOT NULL REFERENCES tracks(track_id),
+  cycle    INTEGER NOT NULL,
+  cof      REAL    NOT NULL,
+  <span class="hl">UNIQUE (track_id, cycle)</span>                      <span class="c">-- required; duplicates the table</span>
+);</pre>
+        </div>
+      </section>
+
+      <!-- ================= METHOD ================= -->
+      <section id="method">
+        <h2>Benchmark Method</h2>
+        <p class="sub">Identical protocol for all three variants; only the DDL differs.</p>
+        <div class="card">
+          <ul>
+            <li><b>Data:</b> the full generated corpus subset - <span class="mono">tracks</span> (1,440), <span class="mono">friction_cycles</span> (1,440,000), <span class="mono">friction_loop_points</span> (2,880,000) - read from the same CSV files in the same order for every variant.</li>
+            <li><b>Load:</b> SQLite, <span class="mono">PRAGMA journal_mode=OFF, synchronous=OFF, cache_size=-262144</span>; inserts via <span class="mono">executemany</span> in 50k-row batches; <span class="mono">VACUUM</span> before measuring the file size.</li>
+            <li><b>Reads:</b> warm OS page cache, best of 3 runs per pattern; fixed deterministic probe sets (39 tracks for range reads, 150 <span class="mono">(track_id, cycle)</span> pairs for point lookups); result-set sizes asserted, so every variant answers the identical question.</li>
+            <li><b>Caveat:</b> at 150 MB everything fits in RAM, which <i>flatters</i> variant C - at the 600 GB production target the working set exceeds RAM, and twice the pages means twice the cache pressure and disk reads. Warm-cache deltas are therefore a lower bound.</li>
+          </ul>
+        </div>
+      </section>
+
+      <!-- ================= STORAGE ================= -->
+      <section id="storage">
+        <h2>Results &middot; Storage &amp; Load</h2>
+        <p class="sub">File size after VACUUM; load time includes CSV parsing (identical work in all variants).</p>
+
+        <div class="card" style="padding:0;overflow:hidden">
+          <table class="data">
+            <thead><tr><th>Variant</th><th>Bulk-table keys</th><th class="num">Load time (s)</th><th class="num">DB size (MiB)</th><th class="num">Size vs A</th></tr></thead>
+            <tbody>
+              <tr><td><b>A</b> &middot; composite PK</td><td>PK (track_id, cycle), unenforced refs</td><td class="num">7.5</td><td class="num">109.1</td><td class="num delta-zero">baseline</td></tr>
+              <tr><td><b>B</b> &middot; composite PK + FK</td><td>PK (track_id, cycle), REFERENCES enforced</td><td class="num">8.0</td><td class="num">109.1</td><td class="num delta-ok">&plusmn;0%</td></tr>
+              <tr><td><b>C</b> &middot; surrogate id</td><td>friction_cycle_id PK + UNIQUE(track_id, cycle)</td><td class="num">8.8</td><td class="num">190.2</td><td class="num delta-bad">+74%</td></tr>
+            </tbody>
+          </table>
+        </div>
+
+        <div class="card">
+          <h3>Database size after VACUUM (MiB, scale 0-200)</h3>
+          <div class="hbars">
+            <div class="hbar"><span class="lab">Variant A<small>composite PK, unenforced refs</small></span><div class="trk"><div class="fill col-a" style="width:54.6%"></div></div><span class="val">109.1 MiB</span></div>
+            <div class="hbar"><span class="lab">Variant B<small>composite PK + enforced FK</small></span><div class="trk"><div class="fill col-b" style="width:54.6%"></div></div><span class="val">109.1 MiB</span></div>
+            <div class="hbar"><span class="lab">Variant C<small>surrogate id + UNIQUE index</small></span><div class="trk"><div class="fill col-c" style="width:95.1%"></div></div><span class="val">190.2 MiB</span></div>
+          </div>
+          <h3 style="margin-top:20px">Load time, 4.32M rows (s, scale 0-10)</h3>
+          <div class="hbars">
+            <div class="hbar"><span class="lab">Variant A</span><div class="trk"><div class="fill col-a" style="width:75%"></div></div><span class="val">7.5 s</span></div>
+            <div class="hbar"><span class="lab">Variant B</span><div class="trk"><div class="fill col-b" style="width:80%"></div></div><span class="val">8.0 s</span></div>
+            <div class="hbar"><span class="lab">Variant C</span><div class="trk"><div class="fill col-c" style="width:88%"></div></div><span class="val">8.8 s</span></div>
+          </div>
+          <div class="callout" style="margin-top:16px">Extrapolated to the 600 GB production target, the +74% of variant C on the two hottest tables translates into <b>hundreds of gigabytes of extra disk</b> and a doubled page count competing for the same RAM cache.</div>
+        </div>
+      </section>
+
+      <!-- ================= READS ================= -->
+      <section id="reads">
+        <h2>Results &middot; Read Performance</h2>
+        <p class="sub">Warm OS cache, best of 3 runs. Q1 / Q5 refer to the benchmark scenarios of the evaluation pipeline.</p>
+
+        <div class="card" style="padding:0;overflow:hidden">
+          <table class="data">
+            <thead><tr><th>Read pattern</th><th>Unit</th><th class="num">A</th><th class="num">B</th><th class="num">C</th><th class="num">C vs A</th></tr></thead>
+            <tbody>
+              <tr><td>Point lookup of one row by <span class="mono">(track_id, cycle)</span></td><td class="mono">&micro;s/op</td><td class="num">24.8</td><td class="num">23.6</td><td class="num">24.3</td><td class="num delta-zero">~0%</td></tr>
+              <tr><td>Q1: all 1,000 cycles of one track</td><td class="mono">ms/track</td><td class="num">0.473</td><td class="num">0.485</td><td class="num">0.533</td><td class="num delta-bad">+13%</td></tr>
+              <tr><td>All 2,000 loop points of one track</td><td class="mono">ms/track</td><td class="num">1.267</td><td class="num">1.299</td><td class="num">1.336</td><td class="num delta-bad">+5%</td></tr>
+              <tr><td>Full scan + AVG over 1.44M cycles</td><td class="mono">ms</td><td class="num">64.9</td><td class="num">64.9</td><td class="num">70.7</td><td class="num delta-bad">+9%</td></tr>
+              <tr><td>AVG with GROUP BY track_id over 1.44M cycles (Q5 pattern)</td><td class="mono">ms</td><td class="num">85.7</td><td class="num">89.1</td><td class="num">135.9</td><td class="num delta-bad">+59%</td></tr>
+              <tr><td>Full scan over 2.88M loop points</td><td class="mono">ms</td><td class="num">152.5</td><td class="num">150.8</td><td class="num">165.4</td><td class="num delta-bad">+8%</td></tr>
+            </tbody>
+          </table>
+        </div>
+
+        <div class="card">
+          <h3>Relative read time, variant A = 100% (lower is better)</h3>
+          <div class="legend">
+            <span><i class="col-a"></i>A &middot; composite PK</span>
+            <span><i class="col-b"></i>B &middot; + enforced FK</span>
+            <span><i class="col-c"></i>C &middot; surrogate id + UNIQUE</span>
+          </div>
+          <svg viewBox="0 0 1080 300" style="width:100%;height:auto;display:block">
+            <line x1="60" y1="246" x2="1080" y2="246" stroke="#d8dce1" stroke-width="1"/>
+            <line x1="60" y1="183.2" x2="1080" y2="183.2" stroke="#eef0f2" stroke-width="1"/>
+            <line x1="60" y1="120.4" x2="1080" y2="120.4" stroke="#c9d4e6" stroke-width="1" stroke-dasharray="4 3"/>
+            <line x1="60" y1="57.7" x2="1080" y2="57.7" stroke="#eef0f2" stroke-width="1"/>
+            <text x="54" y="246" text-anchor="end" dominant-baseline="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#a4abb4">0%</text>
+            <text x="54" y="183.2" text-anchor="end" dominant-baseline="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#a4abb4">50%</text>
+            <text x="54" y="120.4" text-anchor="end" dominant-baseline="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#7e8ba1">100%</text>
+            <text x="54" y="57.7" text-anchor="end" dominant-baseline="middle" font-family="IBM Plex Mono, monospace" font-size="9" fill="#a4abb4">150%</text>
+
+            <!-- group 0: point lookup -->
+            <rect x="78"  y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="124" y="126.5" width="38" height="119.5" rx="2.5" fill="#1f5fdb"/>
+            <rect x="170" y="123.0" width="38" height="123.0" rx="2.5" fill="#c0476b"/>
+            <text x="143" y="262" text-anchor="middle" font-size="10" fill="#5b6472">Point lookup</text>
+            <!-- group 1: Q1 cycles -->
+            <rect x="245" y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="291" y="117.3" width="38" height="128.7" rx="2.5" fill="#1f5fdb"/>
+            <rect x="337" y="104.5" width="38" height="141.5" rx="2.5" fill="#c0476b"/>
+            <text x="310" y="262" text-anchor="middle" font-size="10" fill="#5b6472">Q1: track cycles</text>
+            <!-- group 2: loop points range -->
+            <rect x="411" y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="457" y="117.3" width="38" height="128.7" rx="2.5" fill="#1f5fdb"/>
+            <rect x="503" y="113.7" width="38" height="132.3" rx="2.5" fill="#c0476b"/>
+            <text x="476" y="262" text-anchor="middle" font-size="10" fill="#5b6472">Track loop points</text>
+            <!-- group 3: full scan avg -->
+            <rect x="578" y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="624" y="120.4" width="38" height="125.6" rx="2.5" fill="#1f5fdb"/>
+            <rect x="670" y="109.3" width="38" height="136.7" rx="2.5" fill="#c0476b"/>
+            <text x="643" y="262" text-anchor="middle" font-size="10" fill="#5b6472">Full scan + AVG</text>
+            <!-- group 4: group by -->
+            <rect x="745" y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="791" y="115.4" width="38" height="130.6" rx="2.5" fill="#1f5fdb"/>
+            <rect x="837" y="46.9"  width="38" height="199.1" rx="2.5" fill="#c0476b"/>
+            <text x="856" y="40" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="10" font-weight="600" fill="#b23b4e">+59%</text>
+            <text x="810" y="262" text-anchor="middle" font-size="10" fill="#5b6472">GROUP BY track (Q5)</text>
+            <!-- group 5: loops scan -->
+            <rect x="911"  y="120.4" width="38" height="125.6" rx="2.5" fill="#9aa6b6"/>
+            <rect x="957"  y="121.8" width="38" height="124.2" rx="2.5" fill="#1f5fdb"/>
+            <rect x="1003" y="109.8" width="38" height="136.2" rx="2.5" fill="#c0476b"/>
+            <text x="976" y="262" text-anchor="middle" font-size="10" fill="#5b6472">Loop points scan</text>
+
+            <text x="570" y="290" text-anchor="middle" font-size="10" fill="#838a95">dashed line = variant A baseline (100%)</text>
+          </svg>
+        </div>
+      </section>
+
+      <!-- ================= ANALYSIS ================= -->
+      <section id="analysis">
+        <h2>Why It Differs</h2>
+        <p class="sub">The whole difference reduces to one thing: how many B-trees per bulk table, and in what order the rows physically live.</p>
+
+        <div class="card">
+          <div class="erd-wrap"><div class="erd" style="width:1000px;height:230px">
+            <svg class="wire" viewBox="0 0 1000 230" width="1000" height="230">
+              <path d="M700 115 H760" stroke="#b23b4e" stroke-width="1.6" fill="none" stroke-dasharray="5 4"/>
+              <text x="730" y="105" text-anchor="middle" font-family="IBM Plex Mono, monospace" font-size="10" fill="#b23b4e">extra hop</text>
+            </svg>
+            <div class="tbl" style="left:20px;top:30px;width:380px;border-color:#0f9d8a">
+              <div class="tbl-h" style="background:#0f9d8a"><span>Variants A / B &middot; one B-tree</span></div>
+              <div class="tr"><span class="f">clustered table keyed by (track_id, cycle)</span></div>
+              <div class="tr"><span class="f">descent &rarr; leaf &rarr; sequential range scan</span></div>
+              <div class="tr"><span class="f muted">rows of one track are physically adjacent</span></div>
+            </div>
+            <div class="tbl" style="left:450px;top:10px;width:250px;border-color:#c0476b">
+              <div class="tbl-h" style="background:#c0476b"><span>Variant C &middot; B-tree 1</span></div>
+              <div class="tr"><span class="f">UNIQUE index (track_id, cycle)</span></div>
+              <div class="tr"><span class="f muted">stores key columns + rowid again</span></div>
+            </div>
+            <div class="tbl" style="left:760px;top:80px;width:220px;border-color:#c0476b">
+              <div class="tbl-h" style="background:#c0476b"><span>Variant C &middot; B-tree 2</span></div>
+              <div class="tr"><span class="f">table keyed by friction_cycle_id</span></div>
+              <div class="tr"><span class="f muted">per-row lookup to fetch cof</span></div>
+            </div>
+          </div></div>
+          <ul>
+            <li><b>+74% disk.</b> In SQLite the surrogate id column is a rowid alias and costs nothing in the table itself - but the mandatory <span class="mono">UNIQUE(track_id, cycle)</span> index is a full second B-tree whose entries contain almost every column of the row. Two trees instead of one on a 3-column table is close to a doubling.</li>
+            <li><b>+59% on grouped aggregation (the pipeline's forced-full-scan Q5).</b> In A/B the table is clustered by <span class="mono">track_id</span>, so <span class="mono">GROUP BY track_id</span> streams the tree in group order. In C the engine walks the UNIQUE index for ordering and hops into the table B-tree for every <span class="mono">cof</span> value - the classic double access path.</li>
+            <li><b>+5-13% on ranges and plain scans.</b> More pages (190 vs 109 MiB) means more page traversals and cell decoding, even fully cached.</li>
+            <li><b>~0% on point lookups.</b> Tree depth is the same either way; a single-row probe does not expose the second tree.</li>
+            <li><b>PostgreSQL outlook.</b> PG has no WITHOUT ROWID: the composite-PK table is a heap plus one PK index either way, but variant C still adds the id column to every heap tuple <i>and</i> the second index - expect +30-50% on the bulk tables, plus higher COPY-time FK validation cost.</li>
+          </ul>
+          <div class="callout warn"><b>Integrity is a constant across all variants.</b> Every relation is a single-column integer FK. Choosing C buys naming uniformity, not more integrity - and the surrogate key it adds is never referenced by anything.</div>
+        </div>
+      </section>
+
+      <!-- ================= RECOMMENDATION ================= -->
+      <section id="verdict">
+        <h2>Recommendation</h2>
+        <div class="card" style="border-left:4px solid #0f7a45">
+          <h3><span class="pill rec">Variant B</span> &nbsp;Composite natural PK on bulk tables + enforced foreign keys</h3>
+          <ul>
+            <li>Referential integrity is guaranteed by the DBMS itself at a measured cost of <b>+0.5 s of load time</b> and nothing else.</li>
+            <li>Disk footprint and read profile are identical to the leanest variant A: <b>109.1 MiB</b>, single clustered B-tree per bulk table.</li>
+            <li>Dimension tables (<span class="mono">batches</span>, <span class="mono">wafers</span>, <span class="mono">coupons</span>, <span class="mono">runs</span>, <span class="mono">tracks</span>, <span class="mono">instruments</span>) keep their classic single-column surrogate keys - <span class="mono">&lt;singular&gt;_id</span>, never a bare <span class="mono">id</span> - because other tables reference their rows.</li>
+            <li>Variant C remains a valid design where fact rows must be individually addressable from outside (annotations, reprocessing links). No table in this schema references an individual cycle or loop point, so its cost (+74% disk, up to +59% on aggregation, growing at 600 GB scale) buys nothing here.</li>
+          </ul>
+        </div>
+        <div class="foot">
+          Measured 2026-07-11 on the deterministic simulated corpus (seed 20260711, 141.8 MiB CSV) &middot; SQLite, warm OS cache, best of 3 runs &middot;
+          scripts: <span class="mono">measure_key_variants.py</span> (build + size + load + Q1) and <span class="mono">read_benchmarks.py</span> (six read patterns) &middot;
+          companion page: <span class="mono">Tribology_Ontology.html</span> (measurement-result registry).
+        </div>
+      </section>
+
+    </main>
+  </div>
+</div>
+
+<script>
+(function(){
+  var links = Array.prototype.slice.call(document.querySelectorAll('#sidenav a[href^="#"]'));
+  var byId = {};
+  links.forEach(function(a){ byId[a.getAttribute('href').slice(1)] = a; });
+  var sections = Array.prototype.slice.call(document.querySelectorAll('main section'));
+  var main = document.getElementById('main');
+  function setActive(id){
+    links.forEach(function(a){ a.classList.remove('active'); });
+    if(byId[id]) byId[id].classList.add('active');
+  }
+  links.forEach(function(a){
+    a.addEventListener('click', function(){ setActive(a.getAttribute('href').slice(1)); });
+  });
+  var ticking = false;
+  main.addEventListener('scroll', function(){
+    if(ticking) return; ticking = true;
+    requestAnimationFrame(function(){
+      var top = main.scrollTop + 90, current = sections[0].id;
+      sections.forEach(function(s){ if(s.offsetTop <= top) current = s.id; });
+      setActive(current);
+      ticking = false;
+    });
+  });
+})();
+</script>
+</body>
+</html>

docs/rules/db-sql-schema.md

@@ -74,11 +74,14 @@ and it wins where a spec detail is ambiguous.
 - Loads are all-or-nothing per table and validated against `MANIFEST.csv`
   row counts - see [code-error-handling.md](code-error-handling.md)
   section 3 and [test-pipeline-validation.md](test-pipeline-validation.md).
-- On every SQLite connection that relies on FK behaviour:
-  `PRAGMA foreign_keys = ON` (SQLite silently ignores FK clauses without
-  it). Bulk tables MAY declare FK columns without enforced constraints for
-  load speed - row-count validation is the integrity gate; whichever choice
-  is made must be the same in both engines.
+- **Foreign keys are ENFORCED in both engines on every table, including
+  bulk tables** (owner decision 2026-07-11, after measurement on the real
+  corpus: +0.5 s load on 4.3M rows in SQLite, zero disk and read cost;
+  full study with ER diagrams and measurements:
+  [docs/research/Tribology_FK_Architecture.html](../research/Tribology_FK_Architecture.html)).
+  Every SQLite connection sets `PRAGMA foreign_keys = ON` (SQLite silently
+  ignores FK clauses without it). Row-count validation against
+  `MANIFEST.csv` remains a second, independent integrity gate.
 
 ## 5. `track_summary` - precomputed, and off-limits to Q5
 
@@ -89,6 +92,20 @@ never read it** - Q5 is the forced full-scan benchmark over raw
 `friction_cycles` in every format (spec
 [08](../specs/08_benchmark_queries.md)).
 
+Binding computation (identical semantics in both engines; float results
+agree within the 1e-9 checksum tolerance):
+
+1. Steady-state proxy window = cycles 501-1000 (the second half);
+   `m` = avg(cof), `s` = stddev_pop(cof) over that window.
+2. `run_in_cycles` = the first cycle `c` with `|cof(c) - m| < 2*s`;
+   fallback 500 if no cycle qualifies.
+3. `cof_ss_mean` / `cof_ss_std` = avg / stddev_pop of cof over cycles
+   `> run_in_cycles`.
+
+SQLite computes this in Python during load
+([common/track_summary.py](../../common/track_summary.py)); PostgreSQL as
+a materialized view implementing the same three steps.
+
 ## 6. Engine parity
 
 - Same logical schema, same table names, same column names, same row

docs/specs/05_convert_sqlite.md

@@ -10,17 +10,19 @@ Build the single-file relational variant optimized for size and single-user quer
 CSV tree.
 
 ## Processing instructions
-- One database `./out/sqlite/tribo.db`. PRAGMAs during load: `journal_mode=OFF, synchronous=OFF, cache_size=-262144`; after load: `journal_mode=WAL`, `ANALYZE`, `VACUUM`.
-- Schema (16-table style, integer surrogate keys everywhere; no TEXT foreign keys in bulk tables):
+- One database `./out/sqlite/tribo.db`. PRAGMAs during load: `journal_mode=OFF, synchronous=OFF, cache_size=-262144, foreign_keys=ON`; after load: `journal_mode=WAL`, `ANALYZE`, `VACUUM`.
+- Foreign keys are enforced on every table, including bulk tables (variant B of the measured key-schema study; owner decision 2026-07-11, see db-sql-schema rule). Full study with ER diagrams and measurements: [../research/Tribology_FK_Architecture.html](../research/Tribology_FK_Architecture.html).
+- Schema (16-table style, integer surrogate keys everywhere; no TEXT foreign keys in bulk tables; SQL column names are the lowercase of the CSV headers per data-naming-units):
 	- `batches, wafers, coupons, runs, instruments, tracks` - dimension tables, TEXT natural keys kept as unique columns.
-	- `xrf_points(coupon_id INT, gx INT, gy INT, pt_wtpct REAL, au_wtpct REAL)`.
-	- `profilometry_points(coupon_id INT, gx INT, gy INT, thickness_um REAL)`.
-	- `nanoindentation(coupon_id INT, indent_id INT, x_um REAL, y_um REAL, hardness_gpa REAL, reduced_modulus_gpa REAL)`.
-	- `afm(coupon_id INT PRIMARY KEY, ra_nm REAL, rq_nm REAL)`.
+	- `simtra_profiles(batch_id INT, row_no INT, angle_deg REAL, energy_ev REAL, pt_flux REAL, au_flux REAL, PRIMARY KEY(batch_id, row_no)) WITHOUT ROWID`.
+	- `xrf_points(coupon_id INT, grid_x INT, grid_y INT, pt_wtpct REAL, au_wtpct REAL, PRIMARY KEY(coupon_id, grid_x, grid_y)) WITHOUT ROWID`.
+	- `profilometry_points(coupon_id INT, grid_x INT, grid_y INT, thickness_um REAL, PRIMARY KEY(coupon_id, grid_x, grid_y)) WITHOUT ROWID`.
+	- `nanoindentation(coupon_id INT, indent_id INT, x_um REAL, y_um REAL, hardness_gpa REAL, reduced_modulus_gpa REAL, max_load_mn REAL, PRIMARY KEY(coupon_id, indent_id)) WITHOUT ROWID`.
+	- `afm(coupon_id INT PRIMARY KEY, ra_nm REAL, rq_nm REAL, image_file TEXT)`.
 	- `friction_cycles(track_id INT, cycle INT, cof REAL, PRIMARY KEY(track_id, cycle)) WITHOUT ROWID`.
-	- `friction_loop_points(track_id INT, cycle INT, pt INT, position_um REAL, force_mn REAL, PRIMARY KEY(track_id, cycle, pt)) WITHOUT ROWID`.
+	- `friction_loop_points(track_id INT, cycle INT, pt INT, position_um REAL, friction_force_mn REAL, PRIMARY KEY(track_id, cycle, pt)) WITHOUT ROWID`.
 	- `wear(track_id INT PRIMARY KEY, wear_volume_um3 REAL, k_archard REAL, sliding_distance_m REAL)`.
-	- `track_summary(track_id INT PRIMARY KEY, cof_ss_mean REAL, cof_ss_std REAL, run_in_cycles INT)` - precomputed during load.
+	- `track_summary(track_id INT PRIMARY KEY, cof_ss_mean REAL, cof_ss_std REAL, run_in_cycles INT)` - precomputed during load (algorithm fixed in db-sql-schema rule, section 5).
 - Indexes for Q1-Q7: `coupons(au_wtpct_mean)`, `tracks(run_id)`, `tracks(coupon_id)`, `runs(environment)`, composite `tracks(environment, load_mn)` (denormalize environment/load onto tracks).
 - Batch inserts with `executemany`, 50k rows per transaction.
 - Validate row counts against MANIFEST.

generate_data.py

@@ -9,21 +9,26 @@ instruments would write it, driven entirely by out/config/lab_config.yaml
 
 from __future__ import annotations
 
-import argparse
 import hashlib
 import logging
 import math
 import shutil
 import sys
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from pathlib import Path
 
 import numpy as np
-import yaml
+
+from common.pipeline import (
+	check_dependencies,
+	load_lab_config,
+	parse_task_args,
+	remove_stale_marker,
+	write_marker,
+)
 
 logger = logging.getLogger(__name__)
 
-REPO_ROOT = Path(__file__).resolve().parent
 TASK_ID = "03"
 DEPENDS_ON = ["01"]
 
@@ -361,56 +366,17 @@ class CorpusGenerator:
 		return len(data)
 
 
-def check_dependencies(out_root: Path) -> None:
-	for dep in DEPENDS_ON:
-		marker = out_root / ".done" / f"{dep}.ok"
-		if not marker.exists():
-			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
-
-
-def write_marker(marker_path: Path, gen: CorpusGenerator, tree_bytes: int) -> None:
-	lines = [
-		f"task: '{TASK_ID}'",
-		"status: ok",
-		f"csv_root: {gen.csv_root.as_posix()}",
-		f"files: {len(gen.manifest)}",
-		f"total_rows: {gen.total_rows}",
-		f"total_bytes: {tree_bytes}",
-		f"coupons: {gen.counts['coupons']}",
-		f"tracks: {gen.counts['tracks']}",
-		f"cycle_rows: {gen.counts['cycle_rows']}",
-		f"loop_points: {gen.counts['loop_rows']}",
-		f"xrf_points: {gen.counts['xrf_rows']}",
-	]
-	marker_path.parent.mkdir(parents=True, exist_ok=True)
-	with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
-		fh.write("\n".join(lines) + "\n")
-
-
 def main() -> int:
-	parser = argparse.ArgumentParser(description="Task 03: generate the CSV corpus")
-	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
-	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
-	args = parser.parse_args()
-	logging.basicConfig(
-		level=args.log_level,
-		stream=sys.stderr,
-		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
-	)
-
+	args = parse_task_args("Task 03: generate the CSV corpus")
 	out_root: Path = args.out_root
 	csv_root = out_root / "csv"
-	marker_path = out_root / ".done" / f"{TASK_ID}.ok"
 	try:
-		check_dependencies(out_root)
-		if marker_path.exists():
-			logger.info("re-run: removing stale marker %s", marker_path)
-			marker_path.unlink()
+		check_dependencies(out_root, DEPENDS_ON)
+		remove_stale_marker(out_root, TASK_ID)
 		if csv_root.exists():
 			logger.info("re-run: removing previous corpus %s", csv_root)
 			shutil.rmtree(csv_root)
-		with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
-			cfg = yaml.safe_load(fh)
+		cfg = load_lab_config(out_root)
 
 		gen = CorpusGenerator(cfg, csv_root)
 		gen.generate()
@@ -424,7 +390,17 @@ def main() -> int:
 			raise ValidationError(
 				f"tree size {tree_mib:.1f} MiB outside gate {SIZE_GATE_MIB[0]}-{SIZE_GATE_MIB[1]} MiB"
 			)
-		write_marker(marker_path, gen, tree_bytes)
+		marker_path = write_marker(out_root, TASK_ID, {
+			"csv_root": gen.csv_root.as_posix(),
+			"files": len(gen.manifest),
+			"total_rows": gen.total_rows,
+			"total_bytes": tree_bytes,
+			"coupons": gen.counts["coupons"],
+			"tracks": gen.counts["tracks"],
+			"cycle_rows": gen.counts["cycle_rows"],
+			"loop_points": gen.counts["loop_rows"],
+			"xrf_points": gen.counts["xrf_rows"],
+		})
 	except Exception:
 		logger.critical("task %s failed", TASK_ID, exc_info=True)
 		return 1

make_lab_config.py

@@ -8,16 +8,16 @@ completion marker. Spec: docs/specs/01_lab_configuration.md.
 
 from __future__ import annotations
 
-import argparse
 import logging
 import sys
 from pathlib import Path
 
 import yaml
 
+from common.pipeline import parse_task_args, remove_stale_marker, write_marker
+
 logger = logging.getLogger(__name__)
 
-REPO_ROOT = Path(__file__).resolve().parent
 TASK_ID = "01"
 
 # Transcription of docs/specs/01_lab_configuration.md. This literal is the
@@ -301,51 +301,30 @@ def write_yaml(cfg: dict, config_path: Path) -> int:
 	return len(text.encode("utf-8"))
 
 
-def write_marker(marker_path: Path, config_path: Path, config_bytes: int, cfg: dict) -> None:
-	v = cfg["volumes"]
-	f = cfg["friction_assignment"]
-	h = cfg["hierarchy"]
-	lines = [
-		f"task: '{TASK_ID}'",
-		"status: ok",
-		f"config_file: {config_path.as_posix()}",
-		f"config_bytes: {config_bytes}",
-		f"seed: {cfg['seed']}",
-		f"batches: {h['batches']}",
-		f"wafers: {h['batches'] * h['wafers_per_batch']}",
-		f"coupons: {v['coupons_total']}",
-		f"friction_coupons: {f['friction_coupons_total']}",
-		f"tracks: {v['tracks_total']}",
-		f"cycle_rows: {v['cycle_rows_total']}",
-		f"loop_points: {v['loop_points_total']}",
-		f"xrf_points: {v['xrf_points_total']}",
-	]
-	marker_path.parent.mkdir(parents=True, exist_ok=True)
-	with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
-		fh.write("\n".join(lines) + "\n")
-
-
 def main() -> int:
-	parser = argparse.ArgumentParser(description="Task 01: write out/config/lab_config.yaml")
-	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
-	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
-	args = parser.parse_args()
-	logging.basicConfig(
-		level=args.log_level,
-		stream=sys.stderr,
-		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
-	)
-
+	args = parse_task_args("Task 01: write out/config/lab_config.yaml")
 	out_root: Path = args.out_root
 	config_path = out_root / "config" / "lab_config.yaml"
-	marker_path = out_root / ".done" / f"{TASK_ID}.ok"
 	try:
-		if marker_path.exists():
-			logger.info("re-run: removing stale marker %s", marker_path)
-			marker_path.unlink()
+		remove_stale_marker(out_root, TASK_ID)
 		validate_config(LAB_CONFIG)
 		config_bytes = write_yaml(LAB_CONFIG, config_path)
-		write_marker(marker_path, config_path, config_bytes, LAB_CONFIG)
+		v = LAB_CONFIG["volumes"]
+		f = LAB_CONFIG["friction_assignment"]
+		h = LAB_CONFIG["hierarchy"]
+		marker_path = write_marker(out_root, TASK_ID, {
+			"config_file": config_path.as_posix(),
+			"config_bytes": config_bytes,
+			"seed": LAB_CONFIG["seed"],
+			"batches": h["batches"],
+			"wafers": h["batches"] * h["wafers_per_batch"],
+			"coupons": v["coupons_total"],
+			"friction_coupons": f["friction_coupons_total"],
+			"tracks": v["tracks_total"],
+			"cycle_rows": v["cycle_rows_total"],
+			"loop_points": v["loop_points_total"],
+			"xrf_points": v["xrf_points_total"],
+		})
 	except Exception:
 		logger.critical("task %s failed", TASK_ID, exc_info=True)
 		return 1

make_process_flow.py

@@ -8,16 +8,20 @@ the laboratory workflow: ./out/report/process_flow.md plus one standalone
 
 from __future__ import annotations
 
-import argparse
 import logging
 import sys
 from pathlib import Path
 
-import yaml
+from common.pipeline import (
+	check_dependencies,
+	load_lab_config,
+	parse_task_args,
+	remove_stale_marker,
+	write_marker,
+)
 
 logger = logging.getLogger(__name__)
 
-REPO_ROOT = Path(__file__).resolve().parent
 TASK_ID = "02"
 DEPENDS_ON = ["01"]
 
@@ -190,13 +194,6 @@ def build_diagrams(cfg: dict) -> list[tuple[str, str, str, str]]:
 	]
 
 
-def check_dependencies(out_root: Path) -> None:
-	for dep in DEPENDS_ON:
-		marker = out_root / ".done" / f"{dep}.ok"
-		if not marker.exists():
-			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
-
-
 def validate_diagram(stem: str, text: str) -> None:
 	first = text.splitlines()[0].strip() if text.strip() else ""
 	if not first.startswith(DIAGRAM_TYPES):
@@ -235,42 +232,20 @@ def write_outputs(cfg: dict, out_root: Path) -> tuple[Path, int, int]:
 	return md_path, len(md_text.encode("utf-8")), total_diagram_bytes
 
 
-def write_marker(marker_path: Path, md_path: Path, md_bytes: int, diagram_bytes: int) -> None:
-	lines = [
-		f"task: '{TASK_ID}'",
-		"status: ok",
-		f"process_flow_file: {md_path.as_posix()}",
-		f"process_flow_bytes: {md_bytes}",
-		"diagrams: 6",
-		f"diagram_bytes: {diagram_bytes}",
-	]
-	marker_path.parent.mkdir(parents=True, exist_ok=True)
-	with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
-		fh.write("\n".join(lines) + "\n")
-
-
 def main() -> int:
-	parser = argparse.ArgumentParser(description="Task 02: write process flow diagrams")
-	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
-	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
-	args = parser.parse_args()
-	logging.basicConfig(
-		level=args.log_level,
-		stream=sys.stderr,
-		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
-	)
-
+	args = parse_task_args("Task 02: write process flow diagrams")
 	out_root: Path = args.out_root
-	marker_path = out_root / ".done" / f"{TASK_ID}.ok"
 	try:
-		check_dependencies(out_root)
-		if marker_path.exists():
-			logger.info("re-run: removing stale marker %s", marker_path)
-			marker_path.unlink()
-		with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
-			cfg = yaml.safe_load(fh)
+		check_dependencies(out_root, DEPENDS_ON)
+		remove_stale_marker(out_root, TASK_ID)
+		cfg = load_lab_config(out_root)
 		md_path, md_bytes, diagram_bytes = write_outputs(cfg, out_root)
-		write_marker(marker_path, md_path, md_bytes, diagram_bytes)
+		marker_path = write_marker(out_root, TASK_ID, {
+			"process_flow_file": md_path.as_posix(),
+			"process_flow_bytes": md_bytes,
+			"diagrams": 6,
+			"diagram_bytes": diagram_bytes,
+		})
 	except Exception:
 		logger.critical("task %s failed", TASK_ID, exc_info=True)
 		return 1