feat(convert): add task 05 sqlite converter with variant B keys" -m "- convert: convert_sqlite.py builds tribo.db (composite PKs, enforced FKs, Q1-Q7 indexes, track_summary)

- tools: common/track_summary.py fixes the cross-engine steady-state algorithm
- rules: db-sql-schema records the FK enforcement decision and track_summary definition
- specs: 05 aligned with data-naming-units, simtra_profiles added
- docs: FK key-schema study added under docs/research/, README updated
- replace manual CSV reading with CorpusReader for better data handling
- streamline argument parsing and dependency checks using common pipeline functions
- enhance marker writing for task completion tracking
- remove unused regex and validation error classes for cleaner code

generate_data.py

@@ -9,21 +9,26 @@ instruments would write it, driven entirely by out/config/lab_config.yaml
 
 from __future__ import annotations
 
-import argparse
 import hashlib
 import logging
 import math
 import shutil
 import sys
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from pathlib import Path
 
 import numpy as np
-import yaml
+
+from common.pipeline import (
+	check_dependencies,
+	load_lab_config,
+	parse_task_args,
+	remove_stale_marker,
+	write_marker,
+)
 
 logger = logging.getLogger(__name__)
 
-REPO_ROOT = Path(__file__).resolve().parent
 TASK_ID = "03"
 DEPENDS_ON = ["01"]
 
@@ -361,56 +366,17 @@ class CorpusGenerator:
 		return len(data)
 
 
-def check_dependencies(out_root: Path) -> None:
-	for dep in DEPENDS_ON:
-		marker = out_root / ".done" / f"{dep}.ok"
-		if not marker.exists():
-			raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
-
-
-def write_marker(marker_path: Path, gen: CorpusGenerator, tree_bytes: int) -> None:
-	lines = [
-		f"task: '{TASK_ID}'",
-		"status: ok",
-		f"csv_root: {gen.csv_root.as_posix()}",
-		f"files: {len(gen.manifest)}",
-		f"total_rows: {gen.total_rows}",
-		f"total_bytes: {tree_bytes}",
-		f"coupons: {gen.counts['coupons']}",
-		f"tracks: {gen.counts['tracks']}",
-		f"cycle_rows: {gen.counts['cycle_rows']}",
-		f"loop_points: {gen.counts['loop_rows']}",
-		f"xrf_points: {gen.counts['xrf_rows']}",
-	]
-	marker_path.parent.mkdir(parents=True, exist_ok=True)
-	with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
-		fh.write("\n".join(lines) + "\n")
-
-
 def main() -> int:
-	parser = argparse.ArgumentParser(description="Task 03: generate the CSV corpus")
-	parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
-	parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
-	args = parser.parse_args()
-	logging.basicConfig(
-		level=args.log_level,
-		stream=sys.stderr,
-		format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
-	)
-
+	args = parse_task_args("Task 03: generate the CSV corpus")
 	out_root: Path = args.out_root
 	csv_root = out_root / "csv"
-	marker_path = out_root / ".done" / f"{TASK_ID}.ok"
 	try:
-		check_dependencies(out_root)
-		if marker_path.exists():
-			logger.info("re-run: removing stale marker %s", marker_path)
-			marker_path.unlink()
+		check_dependencies(out_root, DEPENDS_ON)
+		remove_stale_marker(out_root, TASK_ID)
 		if csv_root.exists():
 			logger.info("re-run: removing previous corpus %s", csv_root)
 			shutil.rmtree(csv_root)
-		with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
-			cfg = yaml.safe_load(fh)
+		cfg = load_lab_config(out_root)
 
 		gen = CorpusGenerator(cfg, csv_root)
 		gen.generate()
@@ -424,7 +390,17 @@ def main() -> int:
 			raise ValidationError(
 				f"tree size {tree_mib:.1f} MiB outside gate {SIZE_GATE_MIB[0]}-{SIZE_GATE_MIB[1]} MiB"
 			)
-		write_marker(marker_path, gen, tree_bytes)
+		marker_path = write_marker(out_root, TASK_ID, {
+			"csv_root": gen.csv_root.as_posix(),
+			"files": len(gen.manifest),
+			"total_rows": gen.total_rows,
+			"total_bytes": tree_bytes,
+			"coupons": gen.counts["coupons"],
+			"tracks": gen.counts["tracks"],
+			"cycle_rows": gen.counts["cycle_rows"],
+			"loop_points": gen.counts["loop_rows"],
+			"xrf_points": gen.counts["xrf_rows"],
+		})
 	except Exception:
 		logger.critical("task %s failed", TASK_ID, exc_info=True)
 		return 1