48c102d2a4
- add process_metrics function to record peak RSS and CPU time - update JSON converter to include metrics in completion marker - modify SQLite converter to utilize new metrics for performance tracking - enhance storage size updates with locking mechanism for concurrent access
67 lines
2.4 KiB
Python
67 lines
2.4 KiB
Python
"""Remote host metrics via Prometheus / node_exporter (stdlib urllib only).
|
|
|
|
The PostgreSQL instance is remote, so psutil cannot attribute its CPU/RAM
|
|
(spec 09 assumed a local instance). The database host runs Prometheus;
|
|
its base URL comes from the TRIBO_PROM_URL environment variable
|
|
(e.g. http://192.168.10.73:9090). Unset = monitoring disabled.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
QUERY_STEP_S = 5
|
|
HTTP_TIMEOUT_S = 15
|
|
# rate() window must span at least two scrapes (15 s default interval)
|
|
CPU_QUERY = '100 * (1 - avg(rate(node_cpu_seconds_total{mode="idle"}[30s])))'
|
|
RAM_QUERY = "node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes"
|
|
|
|
|
|
def get_prom_url() -> str:
|
|
return os.environ.get("TRIBO_PROM_URL", "")
|
|
|
|
|
|
def query_range(base_url: str, promql: str, start: float, end: float) -> list[float]:
|
|
params = urllib.parse.urlencode({"query": promql, "start": start, "end": end, "step": QUERY_STEP_S})
|
|
url = base_url.rstrip("/") + "/api/v1/query_range?" + params
|
|
with urllib.request.urlopen(url, timeout=HTTP_TIMEOUT_S) as resp:
|
|
data = json.load(resp)
|
|
if data.get("status") != "success":
|
|
raise RuntimeError(f"prometheus query failed: {data.get('errorType')}: {data.get('error')}")
|
|
result = data["data"]["result"]
|
|
if not result:
|
|
return []
|
|
return [float(v) for _, v in result[0]["values"]]
|
|
|
|
|
|
def host_window_metrics(base_url: str, start: float, end: float) -> dict | None:
|
|
"""CPU/RAM of the monitored host over [start, end] wall-clock seconds.
|
|
|
|
Returns None (with a warning) on any failure - monitoring is best-effort
|
|
and must never fail the owning task.
|
|
"""
|
|
try:
|
|
if end - start < 2 * QUERY_STEP_S:
|
|
end = start + 2 * QUERY_STEP_S
|
|
cpu = query_range(base_url, CPU_QUERY, start, end)
|
|
ram = query_range(base_url, RAM_QUERY, start, end)
|
|
if not cpu or not ram:
|
|
logger.warning("prometheus returned no samples for the window (expected condition: short window or scrape lag)")
|
|
return None
|
|
return {
|
|
"host_cpu_avg_pct": round(sum(cpu) / len(cpu), 1),
|
|
"host_cpu_max_pct": round(max(cpu), 1),
|
|
"host_ram_used_baseline_mb": round(ram[0] / 1048576),
|
|
"host_ram_used_peak_mb": round(max(ram) / 1048576),
|
|
"host_ram_used_delta_mb": round((max(ram) - ram[0]) / 1048576),
|
|
}
|
|
except Exception:
|
|
logger.warning("host metrics collection skipped (prometheus unreachable or query failed)", exc_info=True)
|
|
return None
|