feat(convert): integrate process metrics into JSON and SQLite converters
- add process_metrics function to record peak RSS and CPU time - update JSON converter to include metrics in completion marker - modify SQLite converter to utilize new metrics for performance tracking - enhance storage size updates with locking mechanism for concurrent access
This commit is contained in:
@@ -8,24 +8,45 @@ touches other formats' rows.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HEADER = "format,variant,bytes"
|
||||
LOCK_TIMEOUT_S = 30.0 # converters 04-07 may run in parallel; the update is a
|
||||
LOCK_POLL_S = 0.2 # read-modify-write, so it is serialized via a lock file
|
||||
|
||||
|
||||
def update_storage_sizes(bench_dir: Path, fmt: str, variants: list[tuple[str, int]]) -> Path:
|
||||
path = bench_dir / "storage_sizes.csv"
|
||||
kept: list[str] = []
|
||||
if path.exists():
|
||||
lines = path.read_text(encoding="ascii").splitlines()
|
||||
if lines and lines[0] != HEADER:
|
||||
raise ValueError(f"{path}: unexpected header {lines[0]!r}")
|
||||
kept = [ln for ln in lines[1:] if ln and ln.split(",", 1)[0] != fmt]
|
||||
rows = kept + [f"{fmt},{variant},{nbytes}" for variant, nbytes in variants]
|
||||
bench_dir.mkdir(parents=True, exist_ok=True)
|
||||
with open(path, "w", encoding="ascii", newline="\n") as fh:
|
||||
fh.write(HEADER + "\n" + "\n".join(rows) + "\n")
|
||||
path = bench_dir / "storage_sizes.csv"
|
||||
lock = bench_dir / "storage_sizes.lock"
|
||||
fd = _acquire(lock)
|
||||
try:
|
||||
kept: list[str] = []
|
||||
if path.exists():
|
||||
lines = path.read_text(encoding="ascii").splitlines()
|
||||
if lines and lines[0] != HEADER:
|
||||
raise ValueError(f"{path}: unexpected header {lines[0]!r}")
|
||||
kept = [ln for ln in lines[1:] if ln and ln.split(",", 1)[0] != fmt]
|
||||
rows = kept + [f"{fmt},{variant},{nbytes}" for variant, nbytes in variants]
|
||||
with open(path, "w", encoding="ascii", newline="\n") as fh:
|
||||
fh.write(HEADER + "\n" + "\n".join(rows) + "\n")
|
||||
finally:
|
||||
os.close(fd)
|
||||
lock.unlink()
|
||||
logger.info("storage_sizes.csv: %s -> %s", fmt, ", ".join(f"{v}={b}" for v, b in variants))
|
||||
return path
|
||||
|
||||
|
||||
def _acquire(lock: Path) -> int:
|
||||
deadline = time.monotonic() + LOCK_TIMEOUT_S
|
||||
while True:
|
||||
try:
|
||||
return os.open(lock, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
|
||||
except FileExistsError:
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(f"storage_sizes lock held too long: {lock}")
|
||||
time.sleep(LOCK_POLL_S)
|
||||
|
||||
Reference in New Issue
Block a user