feat(convert): add task 05 sqlite converter with variant B keys" -m "- convert: convert_sqlite.py builds tribo.db (composite PKs, enforced FKs, Q1-Q7 indexes, track_summary)
- tools: common/track_summary.py fixes the cross-engine steady-state algorithm - rules: db-sql-schema records the FK enforcement decision and track_summary definition - specs: 05 aligned with data-naming-units, simtra_profiles added - docs: FK key-schema study added under docs/research/, README updated - replace manual CSV reading with CorpusReader for better data handling - streamline argument parsing and dependency checks using common pipeline functions - enhance marker writing for task completion tracking - remove unused regex and validation error classes for cleaner code
This commit is contained in:
administrator
119
convert_json.py
119
convert_json.py
@@ -17,23 +17,26 @@ Appends both sizes to ./out/bench/storage_sizes.csv and writes
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
from rdflib import Graph
|
||||
|
||||
from common.corpus import CorpusReader, ValidationError
|
||||
from common.pipeline import (
|
||||
check_dependencies,
|
||||
load_lab_config,
|
||||
parse_task_args,
|
||||
remove_stale_marker,
|
||||
write_marker,
|
||||
)
|
||||
from common.storage_sizes import update_storage_sizes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent
|
||||
TASK_ID = "04"
|
||||
DEPENDS_ON = ["01", "03"]
|
||||
|
||||
@@ -60,9 +63,6 @@ CONTEXT = {
|
||||
"started_at": {"@id": "prov:startedAtTime"},
|
||||
}
|
||||
|
||||
INT_RE = re.compile(r"^-?[0-9]+$")
|
||||
FLOAT_RE = re.compile(r"^-?([0-9]+\.[0-9]*|\.[0-9]+|[0-9]+)([eE][+-]?[0-9]+)?$")
|
||||
|
||||
MEASUREMENTS = [
|
||||
# (csv file, json key, @type, instrument id, points key)
|
||||
("xrf_map.csv", "xrf", "CompositionMeasurement", "m4_tornado", "points"),
|
||||
@@ -71,61 +71,19 @@ MEASUREMENTS = [
|
||||
]
|
||||
|
||||
|
||||
class ValidationError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def coerce(s: str) -> int | float | str:
|
||||
if INT_RE.match(s):
|
||||
return int(s)
|
||||
if FLOAT_RE.match(s):
|
||||
return float(s)
|
||||
return s
|
||||
|
||||
|
||||
class JsonConverter:
|
||||
def __init__(self, cfg: dict, csv_root: Path, json_root: Path) -> None:
|
||||
self.cfg = cfg
|
||||
self.csv_root = csv_root
|
||||
self.json_root = json_root
|
||||
self.manifest = self._load_manifest()
|
||||
self.reader = CorpusReader(csv_root)
|
||||
self.manifest = self.reader.manifest
|
||||
self.graph: list[dict] = [] # hybrid metadata entities
|
||||
self.full_bytes = 0
|
||||
self.full_files = 0
|
||||
self.counts = {"coupons": 0, "tracks": 0, "cycle_rows": 0, "loop_rows": 0, "xrf_rows": 0}
|
||||
|
||||
def _load_manifest(self) -> dict[str, tuple[int, int, str]]:
|
||||
manifest: dict[str, tuple[int, int, str]] = {}
|
||||
with open(self.csv_root / "MANIFEST.csv", encoding="ascii") as fh:
|
||||
header = fh.readline().strip()
|
||||
if header != "path,rows,bytes,sha256":
|
||||
raise ValidationError(f"MANIFEST.csv: unexpected header {header!r}")
|
||||
for line in fh:
|
||||
path, rows, nbytes, sha = line.strip().split(",")
|
||||
manifest[path] = (int(rows), int(nbytes), sha)
|
||||
logger.info("manifest loaded: %d entries", len(manifest))
|
||||
return manifest
|
||||
|
||||
def read_csv(self, relpath: str) -> tuple[list[str], list[list[str]]]:
|
||||
"""Read one corpus file, verifying bytes and row count against MANIFEST."""
|
||||
data = (self.csv_root / relpath).read_bytes()
|
||||
exp_rows, exp_bytes, exp_sha = self.manifest[relpath]
|
||||
if len(data) != exp_bytes or hashlib.sha256(data).hexdigest() != exp_sha:
|
||||
raise ValidationError(f"{relpath}: bytes/sha256 mismatch vs MANIFEST")
|
||||
lines = data.decode("ascii").split("\n")
|
||||
header = lines[0].split(",")
|
||||
rows = [ln.split(",") for ln in lines[1:] if ln]
|
||||
if len(rows) != exp_rows:
|
||||
raise ValidationError(f"{relpath}: {len(rows)} rows != manifest {exp_rows}")
|
||||
return header, rows
|
||||
|
||||
def rows_as_dicts(self, relpath: str) -> list[dict]:
|
||||
header, rows = self.read_csv(relpath)
|
||||
keys = [h.lower() for h in header]
|
||||
return [
|
||||
{k: coerce(v) for k, v in zip(keys, row) if v != ""}
|
||||
for row in rows
|
||||
]
|
||||
return self.reader.dicts(relpath)
|
||||
|
||||
def source_ref(self, relpath: str) -> dict:
|
||||
rows, _, sha = self.manifest[relpath]
|
||||
@@ -285,54 +243,17 @@ def validate_sample(paths: list[Path]) -> None:
|
||||
logger.info("sample %s: valid JSON, %d triples via rdflib", path.name, len(g))
|
||||
|
||||
|
||||
def check_dependencies(out_root: Path) -> None:
|
||||
for dep in DEPENDS_ON:
|
||||
marker = out_root / ".done" / f"{dep}.ok"
|
||||
if not marker.exists():
|
||||
raise FileNotFoundError(f"dependency marker missing: {marker} - run task {dep} first")
|
||||
|
||||
|
||||
def write_marker(marker_path: Path, conv: JsonConverter, hybrid_bytes: int) -> None:
|
||||
lines = [
|
||||
f"task: '{TASK_ID}'",
|
||||
"status: ok",
|
||||
f"full_files: {conv.full_files}",
|
||||
f"full_bytes: {conv.full_bytes}",
|
||||
f"hybrid_bytes: {hybrid_bytes}",
|
||||
f"coupons: {conv.counts['coupons']}",
|
||||
f"tracks: {conv.counts['tracks']}",
|
||||
f"cycle_rows: {conv.counts['cycle_rows']}",
|
||||
f"loop_points: {conv.counts['loop_rows']}",
|
||||
]
|
||||
marker_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(marker_path, "w", encoding="utf-8", newline="\n") as fh:
|
||||
fh.write("\n".join(lines) + "\n")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Task 04: convert the CSV corpus to JSON-LD")
|
||||
parser.add_argument("--log-level", default="INFO", choices=["DEBUG", "INFO", "WARNING", "ERROR"])
|
||||
parser.add_argument("--out-root", type=Path, default=REPO_ROOT / "out", help="artifact tree root (default: ./out)")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(
|
||||
level=args.log_level,
|
||||
stream=sys.stderr,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
)
|
||||
|
||||
args = parse_task_args("Task 04: convert the CSV corpus to JSON-LD")
|
||||
out_root: Path = args.out_root
|
||||
json_root = out_root / "json"
|
||||
marker_path = out_root / ".done" / f"{TASK_ID}.ok"
|
||||
try:
|
||||
check_dependencies(out_root)
|
||||
if marker_path.exists():
|
||||
logger.info("re-run: removing stale marker %s", marker_path)
|
||||
marker_path.unlink()
|
||||
check_dependencies(out_root, DEPENDS_ON)
|
||||
remove_stale_marker(out_root, TASK_ID)
|
||||
if json_root.exists():
|
||||
logger.info("re-run: removing previous output %s", json_root)
|
||||
shutil.rmtree(json_root)
|
||||
with open(out_root / "config" / "lab_config.yaml", encoding="utf-8") as fh:
|
||||
cfg = yaml.safe_load(fh)
|
||||
cfg = load_lab_config(out_root)
|
||||
|
||||
conv = JsonConverter(cfg, out_root / "csv", json_root)
|
||||
conv.convert_flat()
|
||||
@@ -344,7 +265,15 @@ def main() -> int:
|
||||
json_root / "hybrid" / "dataset.jsonld",
|
||||
])
|
||||
update_storage_sizes(out_root / "bench", "json", [("full", conv.full_bytes), ("hybrid", hybrid_bytes)])
|
||||
write_marker(marker_path, conv, hybrid_bytes)
|
||||
marker_path = write_marker(out_root, TASK_ID, {
|
||||
"full_files": conv.full_files,
|
||||
"full_bytes": conv.full_bytes,
|
||||
"hybrid_bytes": hybrid_bytes,
|
||||
"coupons": conv.counts["coupons"],
|
||||
"tracks": conv.counts["tracks"],
|
||||
"cycle_rows": conv.counts["cycle_rows"],
|
||||
"loop_points": conv.counts["loop_rows"],
|
||||
})
|
||||
except Exception:
|
||||
logger.critical("task %s failed", TASK_ID, exc_info=True)
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user