feat(convert): add task 05 sqlite converter with variant B keys" -m "- convert: convert_sqlite.py builds tribo.db (composite PKs, enforced FKs, Q1-Q7 indexes, track_summary)

- tools: common/track_summary.py fixes the cross-engine steady-state algorithm
- rules: db-sql-schema records the FK enforcement decision and track_summary definition
- specs: 05 aligned with data-naming-units, simtra_profiles added
- docs: FK key-schema study added under docs/research/, README updated
- replace manual CSV reading with CorpusReader for better data handling
- streamline argument parsing and dependency checks using common pipeline functions
- enhance marker writing for task completion tracking
- remove unused regex and validation error classes for cleaner code

common/corpus.py

@@ -0,0 +1,73 @@
+"""Reading the CSV corpus with MANIFEST validation (converters, tasks 04-07).
+
+Every file consumed from ./out/csv/ is verified against MANIFEST.csv (byte
+size, sha256, row count) before its content is trusted - a mismatch aborts
+the converter (docs/rules/test-pipeline-validation.md section 2).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import re
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+MANIFEST_HEADER = "path,rows,bytes,sha256"
+
+INT_RE = re.compile(r"^-?[0-9]+$")
+FLOAT_RE = re.compile(r"^-?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
+
+
+class ValidationError(RuntimeError):
+	pass
+
+
+def coerce(s: str) -> int | float | str:
+	if INT_RE.match(s):
+		return int(s)
+	if FLOAT_RE.match(s):
+		return float(s)
+	return s
+
+
+class CorpusReader:
+	def __init__(self, csv_root: Path) -> None:
+		self.csv_root = csv_root
+		self.manifest = self._load_manifest()
+
+	def _load_manifest(self) -> dict[str, tuple[int, int, str]]:
+		manifest: dict[str, tuple[int, int, str]] = {}
+		with open(self.csv_root / "MANIFEST.csv", encoding="ascii") as fh:
+			header = fh.readline().strip()
+			if header != MANIFEST_HEADER:
+				raise ValidationError(f"MANIFEST.csv: unexpected header {header!r}")
+			for line in fh:
+				path, rows, nbytes, sha = line.strip().split(",")
+				manifest[path] = (int(rows), int(nbytes), sha)
+		logger.info("manifest loaded: %d entries", len(manifest))
+		return manifest
+
+	def read(self, relpath: str) -> tuple[list[str], list[list[str]]]:
+		"""Return (header, raw string rows), verified against MANIFEST."""
+		data = (self.csv_root / relpath).read_bytes()
+		exp_rows, exp_bytes, exp_sha = self.manifest[relpath]
+		if len(data) != exp_bytes or hashlib.sha256(data).hexdigest() != exp_sha:
+			raise ValidationError(f"{relpath}: bytes/sha256 mismatch vs MANIFEST")
+		lines = data.decode("ascii").split("\n")
+		header = lines[0].split(",")
+		rows = [ln.split(",") for ln in lines[1:] if ln]
+		if len(rows) != exp_rows:
+			raise ValidationError(f"{relpath}: {len(rows)} rows != manifest {exp_rows}")
+		return header, rows
+
+	def dicts(self, relpath: str) -> list[dict]:
+		"""Rows as dicts: lowercased CSV headers as keys, numerics coerced,
+		empty cells omitted (docs/rules/data-naming-units.md section 4)."""
+		header, rows = self.read(relpath)
+		keys = [h.lower() for h in header]
+		return [
+			{k: coerce(v) for k, v in zip(keys, row) if v != ""}
+			for row in rows
+		]

common/pipeline.py

@@ -0,0 +1,59 @@
+"""Task-orchestration helpers: CLI shape, dependency markers, completion
+markers, lab config loading (docs/rules/build-pipeline-tasks.md,
+docs/rules/code-python-style.md section 3).
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def parse_task_args(description: str) -> argparse.Namespace:
+	parser = argparse.ArgumentParser(description=description)
+	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
+	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
+	args = parser.parse_args()
+	logging.basicConfig(
+		level=args.log_level,
+		stream=sys.stderr,
+		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
+	)
+	return args
+
+
+def check_dependencies(out_root: Path, deps: list[str]) -> None:
+	for dep in deps:
+		marker = out_root / ".done" / f"{dep}.ok"
+		if not marker.exists():
+			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
+
+
+def remove_stale_marker(out_root: Path, task_id: str) -> Path:
+	marker = out_root / ".done" / f"{task_id}.ok"
+	if marker.exists():
+		logger.info("re-run: removing stale marker %s", marker)
+		marker.unlink()
+	return marker
+
+
+def write_marker(out_root: Path, task_id: str, entries: dict) -> Path:
+	marker = out_root / ".done" / f"{task_id}.ok"
+	lines = [f"task: '{task_id}'", "status: ok"] + [f"{k}: {v}" for k, v in entries.items()]
+	marker.parent.mkdir(parents=True, exist_ok=True)
+	with open(marker, "w", encoding="utf-8", newline="\n") as fh:
+		fh.write("\n".join(lines) + "\n")
+	return marker
+
+
+def load_lab_config(out_root: Path) -> dict:
+	with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
+		return yaml.safe_load(fh)

common/track_summary.py

@@ -0,0 +1,48 @@
+"""Per-track steady-state summary (docs/rules/db-sql-schema.md section 5).
+
+Both engines must produce identical values: SQLite calls this function
+during load (task 05); the PostgreSQL materialized view (task 06)
+implements the same three steps in SQL. Plain sequential float arithmetic
+keeps results within the 1e-9 checksum tolerance across implementations.
+"""
+
+from __future__ import annotations
+
+import math
+
+TAIL_START_CYCLE = 501  # steady-state proxy window: cycles 501..N
+
+
+def summarize_track(cofs: list[float]) -> tuple[float, float, int]:
+	"""Return (cof_ss_mean, cof_ss_std, run_in_cycles) for cofs[i] = cof at cycle i+1."""
+	n = len(cofs)
+	tail = cofs[TAIL_START_CYCLE - 1:]
+	m = _mean(tail)
+	s = _stddev_pop(tail, m)
+
+	run_in = TAIL_START_CYCLE - 1  # fallback when no cycle enters the 2-sigma band
+	band = 2.0 * s
+	for c in range(1, n + 1):
+		if abs(cofs[c - 1] - m) < band:
+			run_in = c
+			break
+
+	ss = cofs[run_in:]  # cycles strictly greater than run_in
+	ss_mean = _mean(ss)
+	ss_std = _stddev_pop(ss, ss_mean)
+	return ss_mean, ss_std, run_in
+
+
+def _mean(values: list[float]) -> float:
+	total = 0.0
+	for v in values:
+		total += v
+	return total / len(values)
+
+
+def _stddev_pop(values: list[float], mean: float) -> float:
+	acc = 0.0
+	for v in values:
+		d = v - mean
+		acc += d * d
+	return math.sqrt(acc / len(values))