96ed7bc918
- tools: common/track_summary.py fixes the cross-engine steady-state algorithm - rules: db-sql-schema records the FK enforcement decision and track_summary definition - specs: 05 aligned with data-naming-units, simtra_profiles added - docs: FK key-schema study added under docs/research/, README updated - replace manual CSV reading with CorpusReader for better data handling - streamline argument parsing and dependency checks using common pipeline functions - enhance marker writing for task completion tracking - remove unused regex and validation error classes for cleaner code
74 lines
2.4 KiB
Python
74 lines
2.4 KiB
Python
"""Reading the CSV corpus with MANIFEST validation (converters, tasks 04-07).
|
|
|
|
Every file consumed from ./out/csv/ is verified against MANIFEST.csv (byte
|
|
size, sha256, row count) before its content is trusted - a mismatch aborts
|
|
the converter (docs/rules/test-pipeline-validation.md section 2).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import re
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MANIFEST_HEADER = "path,rows,bytes,sha256"
|
|
|
|
INT_RE = re.compile(r"^-?[0-9]+$")
|
|
FLOAT_RE = re.compile(r"^-?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
|
|
|
|
|
|
class ValidationError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def coerce(s: str) -> int | float | str:
|
|
if INT_RE.match(s):
|
|
return int(s)
|
|
if FLOAT_RE.match(s):
|
|
return float(s)
|
|
return s
|
|
|
|
|
|
class CorpusReader:
|
|
def __init__(self, csv_root: Path) -> None:
|
|
self.csv_root = csv_root
|
|
self.manifest = self._load_manifest()
|
|
|
|
def _load_manifest(self) -> dict[str, tuple[int, int, str]]:
|
|
manifest: dict[str, tuple[int, int, str]] = {}
|
|
with open(self.csv_root / "MANIFEST.csv", encoding="ascii") as fh:
|
|
header = fh.readline().strip()
|
|
if header != MANIFEST_HEADER:
|
|
raise ValidationError(f"MANIFEST.csv: unexpected header {header!r}")
|
|
for line in fh:
|
|
path, rows, nbytes, sha = line.strip().split(",")
|
|
manifest[path] = (int(rows), int(nbytes), sha)
|
|
logger.info("manifest loaded: %d entries", len(manifest))
|
|
return manifest
|
|
|
|
def read(self, relpath: str) -> tuple[list[str], list[list[str]]]:
|
|
"""Return (header, raw string rows), verified against MANIFEST."""
|
|
data = (self.csv_root / relpath).read_bytes()
|
|
exp_rows, exp_bytes, exp_sha = self.manifest[relpath]
|
|
if len(data) != exp_bytes or hashlib.sha256(data).hexdigest() != exp_sha:
|
|
raise ValidationError(f"{relpath}: bytes/sha256 mismatch vs MANIFEST")
|
|
lines = data.decode("ascii").split("\n")
|
|
header = lines[0].split(",")
|
|
rows = [ln.split(",") for ln in lines[1:] if ln]
|
|
if len(rows) != exp_rows:
|
|
raise ValidationError(f"{relpath}: {len(rows)} rows != manifest {exp_rows}")
|
|
return header, rows
|
|
|
|
def dicts(self, relpath: str) -> list[dict]:
|
|
"""Rows as dicts: lowercased CSV headers as keys, numerics coerced,
|
|
empty cells omitted (docs/rules/data-naming-units.md section 4)."""
|
|
header, rows = self.read(relpath)
|
|
keys = [h.lower() for h in header]
|
|
return [
|
|
{k: coerce(v) for k, v in zip(keys, row) if v != ""}
|
|
for row in rows
|
|
]
|