48c102d2a4
- add process_metrics function to record peak RSS and CPU time - update JSON converter to include metrics in completion marker - modify SQLite converter to utilize new metrics for performance tracking - enhance storage size updates with locking mechanism for concurrent access
74 lines
2.3 KiB
Python
74 lines
2.3 KiB
Python
"""Canonical benchmark result handling (spec 08, test-pipeline-validation).
|
|
|
|
Every implementation of a benchmark query returns the same canonical rows:
|
|
tuples of str / int / float columns. Rows are compared SORTED; floats agree
|
|
within 1e-9. The canonical expected artifacts are produced from PostgreSQL
|
|
and cross-validated against SQLite (task 08); every measured run (task 09)
|
|
re-validates its rows against them.
|
|
|
|
Two serializations:
|
|
- full: floats via repr() - lossless, used for stdout and expected/*.rows.csv;
|
|
- rounded (6 decimals): used ONLY for the sha256 fingerprint, so engines
|
|
whose aggregate arithmetic differs in the last bits hash identically.
|
|
The tolerant row comparison is the authoritative check; the sha256 is a
|
|
compact fingerprint for reports and markers.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import sys
|
|
|
|
from common.corpus import coerce
|
|
|
|
FLOAT_TOL = 1e-9
|
|
|
|
Row = tuple
|
|
|
|
|
|
def sort_rows(rows: list[Row]) -> list[Row]:
|
|
return sorted(rows)
|
|
|
|
|
|
def _fmt(value, rounded: bool) -> str:
|
|
if isinstance(value, float):
|
|
return f"{round(value, 6):.6f}" if rounded else repr(value)
|
|
return str(value)
|
|
|
|
|
|
def serialize(rows: list[Row], rounded: bool = False) -> str:
|
|
return "\n".join(",".join(_fmt(v, rounded) for v in row) for row in rows) + "\n"
|
|
|
|
|
|
def sha256_of(rows: list[Row]) -> str:
|
|
return hashlib.sha256(serialize(sort_rows(rows), rounded=True).encode("ascii")).hexdigest()
|
|
|
|
|
|
def print_rows(rows: list[Row]) -> None:
|
|
sys.stdout.write(serialize(sort_rows(rows)))
|
|
|
|
|
|
def parse_rows(text: str) -> list[Row]:
|
|
rows = []
|
|
for line in text.splitlines():
|
|
if line:
|
|
rows.append(tuple(coerce(cell) for cell in line.split(",")))
|
|
return rows
|
|
|
|
|
|
def compare(expected: list[Row], actual: list[Row], tol: float = FLOAT_TOL) -> str | None:
|
|
"""Return None when equal within tolerance, else a first-difference message."""
|
|
exp, act = sort_rows(expected), sort_rows(actual)
|
|
if len(exp) != len(act):
|
|
return f"row count {len(act)} != expected {len(exp)}"
|
|
for i, (er, ar) in enumerate(zip(exp, act)):
|
|
if len(er) != len(ar):
|
|
return f"row {i}: arity {len(ar)} != {len(er)}"
|
|
for j, (ev, av) in enumerate(zip(er, ar)):
|
|
if isinstance(ev, float) or isinstance(av, float):
|
|
if abs(float(ev) - float(av)) > tol:
|
|
return f"row {i} col {j}: {av!r} != {ev!r} (tol {tol})"
|
|
elif ev != av:
|
|
return f"row {i} col {j}: {av!r} != {ev!r}"
|
|
return None
|