feat(convert): integrate process metrics into JSON and SQLite converters

- add process_metrics function to record peak RSS and CPU time
- update JSON converter to include metrics in completion marker
- modify SQLite converter to utilize new metrics for performance tracking
- enhance storage size updates with locking mechanism for concurrent access
This commit is contained in:
administrator
2026-07-11 21:39:08 -04:00
parent 96ed7bc918
commit 48c102d2a4
17 changed files with 1928 additions and 187 deletions

View File

@@ -30,6 +30,7 @@ from common.pipeline import (
check_dependencies,
load_lab_config,
parse_task_args,
process_metrics,
remove_stale_marker,
write_marker,
)
@@ -265,7 +266,7 @@ def main() -> int:
json_root / "hybrid" / "dataset.jsonld",
])
update_storage_sizes(out_root / "bench", "json", [("full", conv.full_bytes), ("hybrid", hybrid_bytes)])
marker_path = write_marker(out_root, TASK_ID, {
entries = {
"full_files": conv.full_files,
"full_bytes": conv.full_bytes,
"hybrid_bytes": hybrid_bytes,
@@ -273,7 +274,9 @@ def main() -> int:
"tracks": conv.counts["tracks"],
"cycle_rows": conv.counts["cycle_rows"],
"loop_points": conv.counts["loop_rows"],
})
}
entries.update(process_metrics())
marker_path = write_marker(out_root, TASK_ID, entries)
except Exception:
logger.critical("task %s failed", TASK_ID, exc_info=True)
return 1