eda11aa63e
- implement SQLite and PostgreSQL wrappers to execute benchmark queries - update README to reflect task 09 as implemented - enhance storage size updates to handle (format, variant) replacements - clarify benchmarking methodology for remote PostgreSQL setup
56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
"""Idempotent per-(format, variant) updates to out/bench/storage_sizes.csv.
|
|
|
|
Contract (docs/rules/build-pipeline-tasks.md section 4): columns
|
|
format,variant,bytes; a task re-run REPLACES its own (format, variant) rows
|
|
and never touches other rows. Replacement is per variant, not per format:
|
|
converters 04-07 own the working-footprint variants while task 09 appends
|
|
archival variants of the same formats.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
import time
|
|
from pathlib import Path
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
HEADER = "format,variant,bytes"
|
|
LOCK_TIMEOUT_S = 30.0 # converters 04-07 may run in parallel; the update is a
|
|
LOCK_POLL_S = 0.2 # read-modify-write, so it is serialized via a lock file
|
|
|
|
|
|
def update_storage_sizes(bench_dir: Path, fmt: str, variants: list[tuple[str, int]]) -> Path:
|
|
bench_dir.mkdir(parents=True, exist_ok=True)
|
|
path = bench_dir / "storage_sizes.csv"
|
|
lock = bench_dir / "storage_sizes.lock"
|
|
fd = _acquire(lock)
|
|
try:
|
|
replaced = {(fmt, variant) for variant, _ in variants}
|
|
kept: list[str] = []
|
|
if path.exists():
|
|
lines = path.read_text(encoding="ascii").splitlines()
|
|
if lines and lines[0] != HEADER:
|
|
raise ValueError(f"{path}: unexpected header {lines[0]!r}")
|
|
kept = [ln for ln in lines[1:] if ln and tuple(ln.split(",")[:2]) not in replaced]
|
|
rows = kept + [f"{fmt},{variant},{nbytes}" for variant, nbytes in variants]
|
|
with open(path, "w", encoding="ascii", newline="\n") as fh:
|
|
fh.write(HEADER + "\n" + "\n".join(rows) + "\n")
|
|
finally:
|
|
os.close(fd)
|
|
lock.unlink()
|
|
logger.info("storage_sizes.csv: %s -> %s", fmt, ", ".join(f"{v}={b}" for v, b in variants))
|
|
return path
|
|
|
|
|
|
def _acquire(lock: Path) -> int:
|
|
deadline = time.monotonic() + LOCK_TIMEOUT_S
|
|
while True:
|
|
try:
|
|
return os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
|
except FileExistsError:
|
|
if time.monotonic() > deadline:
|
|
raise TimeoutError(f"storage_sizes lock held too long: {lock}")
|
|
time.sleep(LOCK_POLL_S)
|