feat(convert): integrate process metrics into JSON and SQLite converters
- add process_metrics function to record peak RSS and CPU time - update JSON converter to include metrics in completion marker - modify SQLite converter to utilize new metrics for performance tracking - enhance storage size updates with locking mechanism for concurrent access
This commit is contained in:
@@ -30,6 +30,7 @@ from common.pipeline import (
|
||||
check_dependencies,
|
||||
load_lab_config,
|
||||
parse_task_args,
|
||||
process_metrics,
|
||||
remove_stale_marker,
|
||||
write_marker,
|
||||
)
|
||||
@@ -265,7 +266,7 @@ def main() -> int:
|
||||
json_root / "hybrid" / "dataset.jsonld",
|
||||
])
|
||||
update_storage_sizes(out_root / "bench", "json", [("full", conv.full_bytes), ("hybrid", hybrid_bytes)])
|
||||
marker_path = write_marker(out_root, TASK_ID, {
|
||||
entries = {
|
||||
"full_files": conv.full_files,
|
||||
"full_bytes": conv.full_bytes,
|
||||
"hybrid_bytes": hybrid_bytes,
|
||||
@@ -273,7 +274,9 @@ def main() -> int:
|
||||
"tracks": conv.counts["tracks"],
|
||||
"cycle_rows": conv.counts["cycle_rows"],
|
||||
"loop_points": conv.counts["loop_rows"],
|
||||
})
|
||||
}
|
||||
entries.update(process_metrics())
|
||||
marker_path = write_marker(out_root, TASK_ID, entries)
|
||||
except Exception:
|
||||
logger.critical("task %s failed", TASK_ID, exc_info=True)
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user