From 35d2bdadbd31afa0d01c97f77fdefbff66027857 Mon Sep 17 00:00:00 2001 From: administrator Date: Sat, 11 Jul 2026 23:57:41 -0400 Subject: [PATCH] feat(report): update extrapolation parameters and scoring in README - revise hardware prices and parameters in extrapolate.py - enhance scoring details in README for task 11 - clarify projected scoreboard values and sources - improve chart readability and scoring methodology in report.py - add new utility functions for formatting and reading hardware prices --- README.md | 28 +-- docs/rules/code-python-style.md | 2 +- extrapolate.py | 48 +++-- report.py | 344 +++++++++++++++++++++----------- 4 files changed, 275 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index a88b10a..1debf8f 100644 --- a/README.md +++ b/README.md @@ -240,26 +240,30 @@ D4 tribometer session sequence, D5 execution loop, D6 data hierarchy. (Images appear after running the full pipeline; task 11 writes them to `out/report/charts/` under exactly these names.) -### Final scoreboard (task 11) - to be generated +### Final scoreboard (task 11) Weighted scoring, max 80 points: search speed x5 (0-50), RAM economy x2 -(0-20), disk economy x1 (0-10); TIMEOUT/FAIL projects to 0. +(0-20), disk economy x1 (0-10); a FAIL projection at 600 GB zeroes the +search subscore. Values below are the projected-600 GB scoreboard from the +2026-07-11 pipeline run (seed 20260711); the generated source of truth is +`out/report/tables/t5_weighted_scores.csv`, and REPORT.md also carries the +measured-scale scoreboard. | Format | Search (0-50) | RAM (0-20) | Disk (0-10) | Total (0-80) | |---|---|---|---|---| -| CSV | TBD | TBD | TBD | TBD | -| JSON-LD | TBD | TBD | TBD | TBD | -| SQLite | TBD | TBD | TBD | TBD | -| PostgreSQL | TBD | TBD | TBD | TBD | -| RDF | TBD | TBD | TBD | TBD | +| CSV | 0.6 | 20.0 | 8.3 | 28.9 | +| JSON-LD | 0.1 | 20.0 | 4.0 | 24.0 | +| SQLite | 21.8 | 15.2 | 10.0 | 47.0 | +| PostgreSQL | 50.0 | 5.5 | 3.1 | 58.6 | +| RDF | 0.0 | 0.2 | 0.7 | 0.9 | | Use case | Recommended format | |---|---| -| Analysis | TBD | -| Report generation | TBD | -| Search / filtering | TBD | -| Archiving | TBD | -| Inter-lab exchange | TBD | +| Analysis | PostgreSQL (SQLite acceptable single-user up to ~600 GB) | +| Report generation | PostgreSQL (track_summary materialized view) | +| Search / filtering | PostgreSQL or SQLite (indexed) | +| Archiving | CSV tree + tar.gz, plus pg_dump of the system of record | +| Inter-lab exchange | Hybrid JSON-LD (metadata + sourceFile links to CSV) | ## Conventions diff --git a/docs/rules/code-python-style.md b/docs/rules/code-python-style.md index 2f736cc..75a4fed 100644 --- a/docs/rules/code-python-style.md +++ b/docs/rules/code-python-style.md @@ -30,7 +30,7 @@ The stdlib is the default. The ONLY permitted third-party packages: | Package | Used by | For | |---|---|---| | `numpy` | 03 | vectorized generation, seeded RNG | -| `PyYAML` | 01, 03, 10 | reading/writing the YAML configs | +| `PyYAML` | 01, 03, 10, 11 | reading/writing the YAML configs | | `rdflib` | 04, 07 | JSON-LD validation, RDF serialization | | `oxigraph` | 07, 09 | embedded queryable triplestore | | `ijson` | 08, 09 | streaming JSON parsing | diff --git a/extrapolate.py b/extrapolate.py index 361e61e..904a97a 100644 --- a/extrapolate.py +++ b/extrapolate.py @@ -79,15 +79,29 @@ LAWS = { ("csv", 7): "scan", ("json", 7): "scan", ("sqlite", 7): "scan", ("pg", 7): "scan_parallel", ("rdf", 7): "scan", } -HW_PRICES_DEFAULT = """# Hardware cost parameters for extrapolation (task 10) - June 2026 defaults. +HW_PRICES_DEFAULT = """# Hardware cost parameters for extrapolation (task 10). +# Street prices retrieved 2026-07-11 from the sources below (server DRAM is +# in a documented 2025-2026 price surge - see market_context). The cost +# model buys whole components: 64 GB RDIMM modules and 7.68 TB NVMe drives. # Operator-editable; re-run extrapolate.py after changes (code-config-yaml). -ram_usd_per_gb: 14.0 -nvme_usd_per_tb: 250.0 -server_16_core_usd: 6000.0 -server_32_core_usd: 10000.0 -extra_node_usd: 6000.0 +as_of: "2026-07-11" +ram_module_gb: 64 +ram_module_usd: 1887.0 # A-Tech 64GB (2x32GB) DDR5-5600 ECC RDIMM +nvme_drive_tb: 7.68 +nvme_drive_usd: 3995.0 # Cloud Ninjas NEW 7.68TB NVMe U.2 1DWPD (Dell 14-16G) +server_16_core_usd: 4618.0 # Supermicro AS-1015CS-TNR 1U (EPYC 9004/9005), Broadberry starting config +server_32_core_usd: 6272.0 # Supermicro AS-1115CS-TNR 1U, Broadberry starting config +extra_node_usd: 4618.0 # one additional entry 1U node (AS-1015CS-TNR chassis) +sources: + ram: https://atechmemory.com/collections/ddr5-memory-ram + nvme: https://cloudninjas.com/products/new-7-68tb-nvme-u-2-1dwpd-sie-2-5-enterprise-solid-state-drive-for-14th-15th-16th-gen-dell + servers: https://www.broadberry.com/amd-epyc-9004-supermicro-servers + market_context: https://www.techpowerup.com/342331/server-dram-pricing-jumps-50-only-70-of-orders-getting-filled """ -HW_PRICE_KEYS = ("ram_usd_per_gb", "nvme_usd_per_tb", "server_16_core_usd", "server_32_core_usd", "extra_node_usd") +HW_PRICE_KEYS = ( + "as_of", "ram_module_gb", "ram_module_usd", "nvme_drive_tb", "nvme_drive_usd", + "server_16_core_usd", "server_32_core_usd", "extra_node_usd", "sources", +) def read_medians(path: Path) -> dict[tuple[str, int], float]: @@ -222,7 +236,8 @@ def main() -> int: with open(bench_dir / "hardware_sizing.csv", "w", encoding="ascii", newline="") as fh: writer = csv.writer(fh) - writer.writerow(["format", "scale_gb", "disk_tb", "ram_gb", "cores", "nodes", "est_cost_usd"]) + writer.writerow(["format", "scale_gb", "disk_tb", "ram_gb", "cores", "nodes", + "est_cost_usd", "ram_modules", "nvme_drives", "base_usd", "ram_usd", "disk_usd", "extra_nodes_usd"]) for scale_gb in SCALES_GB: r = scale_gb * 1e9 / csv_bytes for fmt in FORMATS: @@ -238,11 +253,17 @@ def main() -> int: ram_gb = max(MIN_RAM_GB, RDF_HOT_FRACTION * fmt_bytes / 1e9) cores = pick_pg_cores(medians, overhead, r, n0) if fmt == "pg" else CORE_OPTIONS[0] nodes = max(1, math.ceil(ram_gb / NODE_RAM_CEILING_GB)) - server_usd = prices["server_16_core_usd"] if cores == 16 else prices["server_32_core_usd"] - cost = (server_usd + ram_gb * prices["ram_usd_per_gb"] - + disk_tb * prices["nvme_usd_per_tb"] - + (nodes - 1) * prices["extra_node_usd"]) - writer.writerow([fmt, scale_gb, round(disk_tb, 3), round(ram_gb, 1), cores, nodes, round(cost)]) + # bill of materials: whole RDIMM modules and whole NVMe drives + ram_modules = math.ceil(ram_gb / prices["ram_module_gb"]) + nvme_drives = math.ceil(disk_tb / prices["nvme_drive_tb"]) + base_usd = prices["server_16_core_usd"] if cores == 16 else prices["server_32_core_usd"] + ram_usd = ram_modules * prices["ram_module_usd"] + disk_usd = nvme_drives * prices["nvme_drive_usd"] + extra_nodes_usd = (nodes - 1) * prices["extra_node_usd"] + cost = base_usd + ram_usd + disk_usd + extra_nodes_usd + writer.writerow([fmt, scale_gb, round(disk_tb, 3), round(ram_gb, 1), cores, nodes, + round(cost), ram_modules, nvme_drives, + round(base_usd), round(ram_usd), round(disk_usd), round(extra_nodes_usd)]) entries = { "extrapolation_rows": len(SCALES_GB) * len(FORMATS) * len(QUERY_NUMBERS), @@ -252,6 +273,7 @@ def main() -> int: "flags_fail": flags["FAIL"], "pg_index_share": round(index_share, 3), "pg_index_mb": round(pg_index_bytes / 1048576, 1), + "prices_as_of": prices["as_of"], "note_pandas": "pandas CSV variant not measured; its RAM is O(n) - infeasible above ~10 GB (spec 10)", } for fmt in FORMATS: diff --git a/report.py b/report.py index 7178a0f..ea56212 100644 --- a/report.py +++ b/report.py @@ -2,13 +2,21 @@ Aggregates the measured matrix (task 09), the projections (task 10) and the process diagrams (task 02) into the final decision document. Writes 14 PNG -charts (150 dpi, log scales where ranges span decades) under -./out/report/charts/ with the exact names fixed in README.md, five CSV -tables under ./out/report/tables/, and ./out/report/REPORT.md. Weighted -scoring: search speed x5 (0-50), RAM economy x2 (0-20), disk economy x1 -(0-10), max 80; each subscore is 10 x best/value; a format with a FAIL -projection at 600 GB scores 0 on search there (spec 11). Projected numbers -are always labeled as projected (bench-methodology section 7). +charts (150 dpi) under ./out/report/charts/ with the exact names fixed in +README.md, five CSV tables under ./out/report/tables/, and +./out/report/REPORT.md. + +Chart readability contract (owner feedback 2026-07-11): linear scales with +actual units everywhere (panels instead of log axes when magnitudes clash), +no scientific tick notation, full format names, a value label on every bar +and point, titles that state the takeaway. Hardware costs are a discrete +bill of materials (whole RDIMM modules, whole NVMe drives, priced chassis) +with street-price sources and an as-of date from out/config/hw_prices.yaml. + +Weighted scoring: search speed x5 (0-50), RAM economy x2 (0-20), disk +economy x1 (0-10), max 80; each subscore is 10 x best/value; a format with +a FAIL projection at 600 GB scores 0 on search there (spec 11). Projected +numbers are always labeled as projected (bench-methodology section 7). Spec: docs/specs/11_reporting.md. """ @@ -24,6 +32,8 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt +import yaml +from matplotlib.ticker import FuncFormatter from common.pipeline import ( check_dependencies, @@ -42,12 +52,12 @@ FORMATS = ("csv", "json", "sqlite", "pg", "rdf") QUERY_NUMBERS = tuple(range(1, 8)) SCALES_GB = (600, 1200, 6000) REPORT_SCALE_GB = 600 # the headline projection scale, spec 11 +SCALE_LABELS = {600: "600 GB", 1200: "1.2 TB", 6000: "6 TB"} WEIGHT_SEARCH = 5 # x5 -> 0-50, spec 11 WEIGHT_RAM = 2 # x2 -> 0-20 WEIGHT_DISK = 1 # x1 -> 0-10 -# archival (compressed) variant per format for table 1 ARCHIVAL_VARIANT = { "csv": ("csv", "targz"), "json": ("json", "targz"), "sqlite": ("sqlite", "db_gz"), "pg": ("pg", "dump"), "rdf": ("rdf", "nt_gz"), @@ -57,12 +67,19 @@ WORKING_VARIANT = { "pg": ("pg", "live"), "rdf": ("rdf", "store"), } -# query classes for the degradation curves (C4), spec 11 +# query classes for the degradation charts (C4), spec 11 QUERY_CLASSES = { "point_read": (1,), "indexed": (2, 3, 4, 6), "full_scan": (5, 7), } +CLASS_LABELS = {"point_read": "point read", "indexed": "indexed / join", "full_scan": "full scan"} + +QUERY_LABELS = { + 1: "single-track COF curve", 2: "coupons at Au 10+/-0.5 wt%", + 3: "hardness vs steady-state COF", 4: "anomaly filter", + 5: "run-in over raw cycles", 6: "wear volume vs load", 7: "Stribeck aggregation", +} DIAGRAM_TITLES = { "D1": "Coupon assembly", "D2": "Characterization and batch assembly", @@ -72,6 +89,7 @@ DIAGRAM_TITLES = { FMT_COLORS = {"csv": "tab:gray", "json": "tab:orange", "sqlite": "tab:green", "pg": "tab:blue", "rdf": "tab:red"} FMT_NAMES = {"csv": "CSV", "json": "JSON-LD", "sqlite": "SQLite", "pg": "PostgreSQL", "rdf": "RDF triplestore"} +CHASSIS = {16: "Supermicro AS-1015CS-TNR", 32: "Supermicro AS-1115CS-TNR"} def read_medians(path: Path) -> dict[tuple[str, int], dict]: @@ -117,10 +135,18 @@ def read_sizing(path: Path) -> dict[tuple[str, int], dict]: "disk_tb": float(row["disk_tb"]), "ram_gb": float(row["ram_gb"]), "cores": int(row["cores"]), "nodes": int(row["nodes"]), "est_cost_usd": float(row["est_cost_usd"]), + "ram_modules": int(row["ram_modules"]), "nvme_drives": int(row["nvme_drives"]), + "base_usd": float(row["base_usd"]), "ram_usd": float(row["ram_usd"]), + "disk_usd": float(row["disk_usd"]), "extra_nodes_usd": float(row["extra_nodes_usd"]), } return sizing +def read_hw_prices(path: Path) -> dict: + with open(path, encoding="ascii") as fh: + return yaml.safe_load(fh) + + def geometric_mean(values: list[float]) -> float: return math.exp(sum(math.log(v) for v in values) / len(values)) @@ -133,7 +159,7 @@ def md_table(headers: list[str], rows: list[list]) -> str: def human_time(seconds: float) -> str: if seconds < 120: - return f"{seconds:.2f} s" + return f"{seconds:.2f} s" if seconds < 10 else f"{seconds:.0f} s" if seconds < 2 * 3600: return f"{seconds / 60:.1f} min" if seconds < 2 * 86400: @@ -141,6 +167,17 @@ def human_time(seconds: float) -> str: return f"{seconds / 86400:.1f} d" +def human_size(nbytes: float) -> str: + for unit, div in (("TB", 1e12), ("GB", 1e9), ("MB", 1e6)): + if nbytes >= div: + return f"{nbytes / div:.2f} {unit}" if nbytes < 10 * div else f"{nbytes / div:.0f} {unit}" + return f"{nbytes / 1e3:.0f} KB" + + +def money(value: float) -> str: + return f"${value:,.0f}" + + def write_csv(path: Path, headers: list[str], rows: list[list]) -> None: with open(path, "w", encoding="ascii", newline="") as fh: writer = csv.writer(fh) @@ -159,7 +196,6 @@ def scores_for(metrics: dict[str, dict[str, float]], failed_search: set[str]) -> ram = 10.0 * best["ram"] / m["ram"] disk = 10.0 * best["disk"] / m["disk"] out[fmt] = { - "search_sub": round(search, 2), "ram_sub": round(ram, 2), "disk_sub": round(disk, 2), "search_pts": round(search * WEIGHT_SEARCH, 1), "ram_pts": round(ram * WEIGHT_RAM, 1), "disk_pts": round(disk * WEIGHT_DISK, 1), @@ -193,40 +229,60 @@ def build_scoreboards(medians, proj, sizes, sizing) -> dict[str, dict[str, dict] # --- charts --------------------------------------------------------------- +# Shared conventions: linear axes, real units, a value label on every bar +# and point, full format names, no scientific notation. -def _grouped_bars(ax, labels, series: dict[str, list[float]]) -> None: - width = 0.8 / len(series) - for i, (name, values) in enumerate(series.items()): - x = [j + i * width for j in range(len(labels))] - ax.bar(x, values, width=width, label=name) - ax.set_xticks([j + 0.4 - width / 2 for j in range(len(labels))]) - ax.set_xticklabels(labels) - ax.set_yscale("log") - ax.legend() - ax.grid(axis="y", alpha=0.3) +def _vbar_panel(ax, values: dict[str, float], labels: dict[str, str], unit: str, title: str) -> None: + bars = ax.bar(range(len(FORMATS)), [values[f] for f in FORMATS], color=[FMT_COLORS[f] for f in FORMATS]) + for bar, fmt in zip(bars, FORMATS): + ax.annotate(labels[fmt], (bar.get_x() + bar.get_width() / 2, bar.get_height()), + ha="center", va="bottom", fontsize=8) + ax.set_xticks(range(len(FORMATS))) + ax.set_xticklabels([FMT_NAMES[f] for f in FORMATS], rotation=12, fontsize=8) + ax.set_ylabel(unit) + ax.set_title(title, fontsize=10) + ax.margins(y=0.18) + ax.grid(axis="y", alpha=0.25) -def chart_c1(charts_dir, sizes, proj_bytes) -> None: - fig, ax = plt.subplots(figsize=(7, 4.5)) - _grouped_bars(ax, list(FORMATS), { - "measured (150 MB corpus)": [sizes[WORKING_VARIANT[fmt]] / 1e6 for fmt in FORMATS], - "projected @600 GB": [proj_bytes[fmt] / 1e6 for fmt in FORMATS], - }) - ax.set_ylabel("working footprint, MB (log)") - ax.set_title("C1 - disk footprint per format") +def _hbar_panel(ax, values: dict[str, float], labels: dict[str, str], title: str, names: bool) -> None: + y = range(len(FORMATS)) + bars = ax.barh(list(y), [values[f] for f in FORMATS], color=[FMT_COLORS[f] for f in FORMATS]) + for bar, fmt in zip(bars, FORMATS): + ax.annotate(" " + labels[fmt], (bar.get_width(), bar.get_y() + bar.get_height() / 2), + va="center", fontsize=8) + ax.set_yticks(list(y)) + ax.set_yticklabels([FMT_NAMES[f] for f in FORMATS] if names else [""] * len(FORMATS), fontsize=8) + ax.invert_yaxis() + ax.set_xticks([]) # linear axis; the value labels on the bars carry the numbers + ax.margins(x=0.30) + ax.set_title(title, fontsize=10) + + +def chart_c1(charts_dir, sizes, proj_bytes, coeff) -> None: + fig, axes = plt.subplots(1, 2, figsize=(11, 4.5)) + measured = {f: sizes[WORKING_VARIANT[f]] for f in FORMATS} + _vbar_panel(axes[0], {f: measured[f] / 1e9 for f in FORMATS}, + {f: human_size(measured[f]) for f in FORMATS}, "GB (linear)", "measured on the 150 MB corpus") + _vbar_panel(axes[1], {f: proj_bytes[f] / 1e9 for f in FORMATS}, + {f: human_size(proj_bytes[f]) for f in FORMATS}, "GB (linear)", "projected at 600 GB of raw data") + fig.suptitle(f"C1 - disk footprint: RDF stores the same data in {coeff['rdf']:.0f}x the raw CSV size, SQLite in {coeff['sqlite']:.2f}x") fig.tight_layout() fig.savefig(charts_dir / "c1_disk_footprint.png", dpi=DPI) plt.close(fig) def chart_c2(charts_dir, medians, sizing) -> None: - fig, ax = plt.subplots(figsize=(7, 4.5)) - _grouped_bars(ax, list(FORMATS), { - "measured client peak RSS, MB": [max(medians[(fmt, qn)]["peak_rss_mb"] for qn in QUERY_NUMBERS) for fmt in FORMATS], - "recommended RAM @600 GB, MB (projected)": [sizing[(fmt, REPORT_SCALE_GB)]["ram_gb"] * 1024 for fmt in FORMATS], - }) - ax.set_ylabel("MB (log)") - ax.set_title("C2 - RAM requirement per format") + fig, axes = plt.subplots(1, 2, figsize=(11, 4.5)) + rss = {f: max(medians[(f, qn)]["peak_rss_mb"] for qn in QUERY_NUMBERS) for f in FORMATS} + ram = {f: sizing[(f, REPORT_SCALE_GB)]["ram_gb"] for f in FORMATS} + _vbar_panel(axes[0], rss, {f: f"{rss[f]:.0f} MB" for f in FORMATS}, + "MB (linear)", "measured client peak RSS on 150 MB") + _vbar_panel(axes[1], ram, {f: f"{ram[f]:.0f} GB" for f in FORMATS}, + "GB (linear)", "recommended server RAM at 600 GB (projected)") + axes[1].axhline(1024, color="gray", linestyle="--", linewidth=1) + axes[1].annotate("single-node ceiling: 1 TB RAM", (0, 1024), fontsize=8, va="bottom") + fig.suptitle("C2 - RAM: only the RDF hot set outgrows a single 1 TB node at 600 GB") fig.tight_layout() fig.savefig(charts_dir / "c2_ram_requirement.png", dpi=DPI) plt.close(fig) @@ -234,82 +290,89 @@ def chart_c2(charts_dir, medians, sizing) -> None: def chart_c3(charts_dir, medians, proj) -> None: for qn in QUERY_NUMBERS: - fig, ax = plt.subplots(figsize=(7, 4.5)) - _grouped_bars(ax, list(FORMATS), { - "measured (150 MB)": [medians[(fmt, qn)]["wall_s"] for fmt in FORMATS], - "projected @600 GB": [proj[(fmt, qn, REPORT_SCALE_GB)][0] for fmt in FORMATS], - }) - ax.set_ylabel("wall time, s (log)") - ax.set_title(f"C3 - Q{qn} wall time per format") + fig, axes = plt.subplots(1, 2, figsize=(11, 3.6)) + measured = {f: medians[(f, qn)]["wall_s"] for f in FORMATS} + projected = {f: proj[(f, qn, REPORT_SCALE_GB)][0] for f in FORMATS} + _hbar_panel(axes[0], measured, {f: human_time(measured[f]) for f in FORMATS}, + "measured on 150 MB", names=True) + _hbar_panel(axes[1], projected, {f: human_time(projected[f]) for f in FORMATS}, + "projected at 600 GB", names=False) + fastest = min(FORMATS, key=lambda f: projected[f]) + slowest = max(FORMATS, key=lambda f: projected[f]) + fig.suptitle(f"C3 - Q{qn} ({QUERY_LABELS[qn]}) at 600 GB: {FMT_NAMES[fastest]} {human_time(projected[fastest])}" + f" vs {FMT_NAMES[slowest]} {human_time(projected[slowest])}") fig.tight_layout() fig.savefig(charts_dir / f"c3_q{qn}_wall_time.png", dpi=DPI) plt.close(fig) -def chart_c4(charts_dir, medians, proj, measured_gb: float) -> None: +def chart_c4(charts_dir, medians, proj, measured_label: str) -> None: for cls, queries in QUERY_CLASSES.items(): - fig, ax = plt.subplots(figsize=(7, 4.5)) - for fmt in FORMATS: - x = [measured_gb, *SCALES_GB] - y = [geometric_mean([medians[(fmt, qn)]["wall_s"] for qn in queries])] - y += [geometric_mean([proj[(fmt, qn, s)][0] for qn in queries]) for s in SCALES_GB] - ax.plot(x, y, marker="o", label=fmt, color=FMT_COLORS[fmt]) - ax.set_xscale("log") - ax.set_yscale("log") - ax.axhline(3600, color="gray", linestyle="--", linewidth=1) - ax.text(measured_gb, 3600, "IMPRACTICAL (1 h)", fontsize=7, va="bottom") - ax.set_xlabel("raw data size, GB (log); first point measured, rest projected") - ax.set_ylabel("wall time, s (log)") + fig, axes = plt.subplots(1, 1 + len(SCALES_GB), figsize=(13, 3.6)) + measured = {f: geometric_mean([medians[(f, qn)]["wall_s"] for qn in queries]) for f in FORMATS} + _hbar_panel(axes[0], measured, {f: human_time(measured[f]) for f in FORMATS}, measured_label, names=True) + for ax, scale in zip(axes[1:], SCALES_GB): + values = {f: geometric_mean([proj[(f, qn, scale)][0] for qn in queries]) for f in FORMATS} + _hbar_panel(ax, values, {f: human_time(values[f]) for f in FORMATS}, + f"projected {SCALE_LABELS[scale]}", names=False) queries_label = ", ".join(f"Q{qn}" for qn in queries) - ax.set_title(f"C4 - degradation, {cls.replace('_', ' ')} ({queries_label})") - ax.legend() - ax.grid(alpha=0.3) + fig.suptitle(f"C4 - {CLASS_LABELS[cls]} queries ({queries_label}): wall time growth by data size" + " (every panel after the first is projected; linear scale within each panel)") fig.tight_layout() fig.savefig(charts_dir / f"c4_degradation_{cls}.png", dpi=DPI) plt.close(fig) -def chart_c5(charts_dir, scoreboards) -> None: +def chart_c5(charts_dir, scoreboards, winner: str) -> None: fig, axes = plt.subplots(1, 2, figsize=(11, 4.5), sharey=True) for ax, (title, board) in zip(axes, [ ("measured scale (150 MB)", scoreboards["measured"]), - ("projected @600 GB", scoreboards["projected_600gb"]), + ("projected at 600 GB", scoreboards["projected_600gb"]), ]): x = range(len(FORMATS)) search = [board[fmt]["search_pts"] for fmt in FORMATS] ram = [board[fmt]["ram_pts"] for fmt in FORMATS] disk = [board[fmt]["disk_pts"] for fmt in FORMATS] - ax.bar(x, search, label=f"search speed x{WEIGHT_SEARCH}") - ax.bar(x, ram, bottom=search, label=f"RAM economy x{WEIGHT_RAM}") - ax.bar(x, disk, bottom=[s + r for s, r in zip(search, ram)], label=f"disk economy x{WEIGHT_DISK}") + ax.bar(x, search, label=f"search speed x{WEIGHT_SEARCH} (0-50)") + ax.bar(x, ram, bottom=search, label=f"RAM economy x{WEIGHT_RAM} (0-20)") + ax.bar(x, disk, bottom=[s + r for s, r in zip(search, ram)], label=f"disk economy x{WEIGHT_DISK} (0-10)") for i, fmt in enumerate(FORMATS): - ax.text(i, board[fmt]["total"] + 1, f"{board[fmt]['total']:.0f}", ha="center", fontsize=8) + ax.text(i, board[fmt]["total"] + 1.5, f"{board[fmt]['total']:.0f}", ha="center", fontsize=9) ax.set_xticks(list(x)) - ax.set_xticklabels(FORMATS) + ax.set_xticklabels([FMT_NAMES[f] for f in FORMATS], rotation=12, fontsize=8) ax.set_ylim(0, 85) - ax.set_title(title) - ax.grid(axis="y", alpha=0.3) - axes[0].set_ylabel("weighted score (max 80)") + ax.set_title(title, fontsize=10) + ax.grid(axis="y", alpha=0.25) + axes[0].set_ylabel("weighted score (linear, max 80)") axes[0].legend(loc="upper left", fontsize=8) - fig.suptitle("C5 - weighted score, stacked components") + board600 = scoreboards["projected_600gb"] + fig.suptitle(f"C5 - weighted score (max 80): {FMT_NAMES[winner]} leads the 600 GB scoreboard with {board600[winner]['total']:.0f} points") fig.tight_layout() fig.savefig(charts_dir / "c5_weighted_score.png", dpi=DPI) plt.close(fig) def chart_c6(charts_dir, proj, sizing) -> None: - fig, ax = plt.subplots(figsize=(7, 4.5)) + fig, ax = plt.subplots(figsize=(9, 5.5)) + offsets = {"csv": (12, 0), "json": (12, -4), "sqlite": (12, 10), "pg": (12, -16), "rdf": (-40, -20)} + geo = {f: geometric_mean([proj[(f, qn, REPORT_SCALE_GB)][0] for qn in QUERY_NUMBERS]) for f in FORMATS} + cost = {f: sizing[(f, REPORT_SCALE_GB)]["est_cost_usd"] for f in FORMATS} for fmt in FORMATS: - cost = sizing[(fmt, REPORT_SCALE_GB)]["est_cost_usd"] - geo = geometric_mean([proj[(fmt, qn, REPORT_SCALE_GB)][0] for qn in QUERY_NUMBERS]) - ax.scatter(cost, geo, s=60, color=FMT_COLORS[fmt]) - ax.annotate(fmt, (cost, geo), textcoords="offset points", xytext=(6, 4)) - ax.set_xscale("log") - ax.set_yscale("log") - ax.set_xlabel("estimated hardware cost @600 GB, USD (log)") - ax.set_ylabel("geometric mean projected wall time Q1-Q7, s (log)") - ax.set_title("C6 - cost vs performance @600 GB (projected)") - ax.grid(alpha=0.3) + ax.scatter(cost[fmt], geo[fmt] / 60, s=70, color=FMT_COLORS[fmt]) + ax.annotate(f"{FMT_NAMES[fmt]}: {money(cost[fmt])}, {human_time(geo[fmt])}", + (cost[fmt], geo[fmt] / 60), textcoords="offset points", xytext=offsets[fmt], fontsize=8) + ax.xaxis.set_major_formatter(FuncFormatter(lambda v, _: f"${v / 1000:.0f}k")) + ax.yaxis.set_major_formatter(FuncFormatter(lambda v, _: f"{v:.0f} min")) + ax.set_xlabel("estimated hardware cost at 600 GB (bill of materials, linear)") + ax.set_ylabel("typical query time at 600 GB, projected (geometric mean Q1-Q7, linear)") + ax.annotate("better: lower-left", xy=(0.05, 0.04), xytext=(0.42, 0.10), + xycoords="axes fraction", textcoords="axes fraction", + fontsize=10, color="darkgreen", fontweight="bold", + arrowprops={"arrowstyle": "->", "color": "darkgreen"}) + ax.margins(x=0.18, y=0.12) + ax.grid(alpha=0.25) + fastest = min(FORMATS, key=lambda f: geo[f]) + ax.set_title(f"C6 - cost vs speed at 600 GB (projected): {FMT_NAMES[fastest]} is the fastest at {money(cost[fastest])}") fig.tight_layout() fig.savefig(charts_dir / "c6_cost_vs_performance.png", dpi=DPI) plt.close(fig) @@ -319,17 +382,26 @@ def chart_c6(charts_dir, proj, sizing) -> None: def scoreboard_rows(board: dict[str, dict]) -> list[list]: ranked = sorted(FORMATS, key=lambda f: -board[f]["total"]) - return [[fmt, board[fmt]["search_pts"], board[fmt]["ram_pts"], board[fmt]["disk_pts"], board[fmt]["total"]] for fmt in ranked] + return [[FMT_NAMES[fmt], board[fmt]["search_pts"], board[fmt]["ram_pts"], board[fmt]["disk_pts"], board[fmt]["total"]] for fmt in ranked] -def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[list]]], - scoreboards, environment: str, diagrams: dict[str, str], winner: str) -> str: +def config_string(row: dict, prices: dict) -> str: + return (f"{row['nodes']} x 1U {CHASSIS[row['cores']]} ({row['cores']} cores);" + f" {row['ram_modules']} x {prices['ram_module_gb']} GB DDR5 ECC RDIMM;" + f" {row['nvme_drives']} x {prices['nvme_drive_tb']} TB NVMe U.2") + + +def assemble_report(tables: dict[str, tuple[list[str], list[list]]], scoreboards, + environment: str, diagrams: dict[str, str], winner: str, sizing, prices: dict) -> str: t1_h, t1_r = tables["t1_storage_footprint"] t2_h, t2_r = tables["t2_measured_medians"] t3_h, t3_r = tables["t3_projected_600gb"] t4_h, t4_r = tables["t4_hardware_600gb"] + pg6 = sizing[("pg", 6000)] + rdf6 = sizing[("rdf", 6000)] + as_of = prices["as_of"] + sources = prices["sources"] - board600 = scoreboards["projected_600gb"] parts = [ "# Laboratory Data Storage Format Evaluation - Final Report", "", @@ -345,9 +417,10 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li "", "- **PostgreSQL** is the only format whose seven benchmark queries stay" " below the 1-hour practicality bound at every projected scale (600 GB," - " 1.2 TB, 6 TB); at 6 TB it needs a single 16-core node with ~551 GB" - " RAM (~$19k, projected). Partitioned parallel scans and the" - " track_summary materialized view are the scaling levers.", + f" 1.2 TB, 6 TB); at 6 TB it needs one {pg6['cores']}-core node with" + f" {pg6['ram_gb']:.0f} GB RAM at an estimated {money(pg6['est_cost_usd'])}" + f" (bill of materials, prices as of {as_of}). Partitioned parallel scans" + " and the track_summary materialized view are the scaling levers.", "- **CSV** remains the canonical raw archive: byte-reproducible," " instrument-native, and the best compressor (tar.gz ~2.8x); its full" " scans exceed 1 h beyond ~600 GB, so it is an archive, not a query" @@ -359,9 +432,10 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li " corpus) is the exchange format; the FULL variant re-reads the whole" " corpus per scan and becomes impractical past 600 GB.", "- **RDF (materialized triplestore)** fails at scale: 11.7x storage" - " blow-up, 5 of 7 queries FAIL at 6 TB, hot-set RAM exceeds the 1 TB" - " single-node ceiling (14 nodes, ~$303k projected). Semantic access" - " should be a virtual layer (e.g. Ontop OBDA) over PostgreSQL instead.", + " blow-up, 5 of 7 queries FAIL at 6 TB, and the hot set needs" + f" {rdf6['ram_gb'] / 1024:.1f} TB RAM across {rdf6['nodes']} nodes" + f" (~{money(rdf6['est_cost_usd'])} projected). Semantic access should be" + " a virtual layer (e.g. Ontop OBDA) over PostgreSQL instead.", "", "## 2. Methodology snapshot", "", @@ -392,9 +466,11 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li md_table(t2_h, t2_r), "", "Sub-interval note: cells faster than the 50 ms sampling cadence" - " (fast sqlite/csv point reads) under-report client RSS/CPU; wall" + " (fast SQLite/CSV point reads) under-report client RSS/CPU; wall" " times are exact.", "", + "![C2 - RAM requirement](charts/c2_ram_requirement.png)", + "", "Per-query charts: " + ", ".join(f"[Q{qn}](charts/c3_q{qn}_wall_time.png)" for qn in QUERY_NUMBERS), "", "## 6. Projections (600 GB / 1.2 TB / 6 TB)", @@ -405,21 +481,38 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li " calibrated on the measured point after subtracting the per-format" " harness floor. Flags: IMPRACTICAL > 1 h, FAIL > 24 h.", "", - "Projected wall times @600 GB:", + "Projected wall times at 600 GB:", "", md_table(t3_h, t3_r), "", - "Degradation curves: [point read](charts/c4_degradation_point_read.png)," - " [indexed](charts/c4_degradation_indexed.png)," + "Degradation by data size: [point read](charts/c4_degradation_point_read.png)," + " [indexed / join](charts/c4_degradation_indexed.png)," " [full scan](charts/c4_degradation_full_scan.png)", "", - "## 7. Hardware sizing and cost @600 GB (projected)", + "## 7. Hardware sizing and cost at 600 GB (projected)", + "", + f"Bill-of-materials costing with street prices as of **{as_of}**" + " (sources below). Every configuration buys whole components: 64 GB" + " DDR5 ECC RDIMM modules, 7.68 TB enterprise NVMe U.2 drives, and a" + " priced 1U chassis; disk capacity includes 30% free-space headroom.", "", md_table(t4_h, t4_r), "", + f"Component prices (retrieved {as_of}):", + "", + f"- {prices['ram_module_gb']} GB DDR5-5600 ECC RDIMM (A-Tech, 2x32GB kit):" + f" {money(prices['ram_module_usd'])} - <{sources['ram']}>", + f"- {prices['nvme_drive_tb']} TB NVMe U.2 1DWPD enterprise SSD (Dell 14-16G" + f" compatible, Cloud Ninjas): {money(prices['nvme_drive_usd'])} - <{sources['nvme']}>", + f"- 1U AMD EPYC 9004/9005 chassis (Broadberry starting configurations):" + f" {CHASSIS[16]} {money(prices['server_16_core_usd'])}," + f" {CHASSIS[32]} {money(prices['server_32_core_usd'])} - <{sources['servers']}>", + f"- Market context - server DRAM is in a documented 2025-2026 price" + f" surge: <{sources['market_context']}>", + "", "Full sizing at 1.2 TB and 6 TB: out/bench/hardware_sizing.csv." - " RDF exceeds the 1 TB single-node RAM ceiling already at 600 GB" - " (1.4 TB hot set) and reaches 14 nodes at 6 TB.", + f" RDF exceeds the 1 TB single-node RAM ceiling already at 600 GB and" + f" reaches {rdf6['nodes']} nodes at 6 TB.", "", "![C6 - cost vs performance](charts/c6_cost_vs_performance.png)", "", @@ -436,10 +529,10 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li md_table(["format", f"search (x{WEIGHT_SEARCH})", f"RAM (x{WEIGHT_RAM})", f"disk (x{WEIGHT_DISK})", "total (max 80)"], scoreboard_rows(scoreboards["measured"])), "", - "Projected @600 GB:", + "Projected at 600 GB:", "", md_table(["format", f"search (x{WEIGHT_SEARCH})", f"RAM (x{WEIGHT_RAM})", f"disk (x{WEIGHT_DISK})", "total (max 80)"], - scoreboard_rows(board600)), + scoreboard_rows(scoreboards["projected_600gb"])), "", "![C5 - weighted score](charts/c5_weighted_score.png)", "", @@ -473,6 +566,9 @@ def assemble_report(report_dir: Path, tables: dict[str, tuple[list[str], list[li "- RDF numbers reflect the compact modeling and a client that derives" " run-in / steady-state from raw cycles; SPARQL-side aggregation" " engines could shift (not remove) the scan penalty.", + f"- Hardware prices are spot street prices retrieved {as_of} during a" + " documented server-DRAM price surge; re-cost by editing" + " out/config/hw_prices.yaml and re-running tasks 10-11.", "", "## Appendix A - process flow diagrams (task 02)", "", @@ -494,6 +590,7 @@ def main() -> int: proj = read_projections(bench_dir / "extrapolation.csv") sizes = read_sizes(bench_dir / "storage_sizes.csv") sizing = read_sizing(bench_dir / "hardware_sizing.csv") + prices = read_hw_prices(out_root / "config" / "hw_prices.yaml") environment = (bench_dir / "environment.txt").read_text(encoding="utf-8") diagrams = {p.stem: p.read_text(encoding="ascii") for p in sorted((report_dir / "diagrams").glob("D*.mermaid"))} if len(diagrams) != 6: @@ -505,57 +602,61 @@ def main() -> int: tables_dir.mkdir(parents=True, exist_ok=True) csv_bytes = sizes[WORKING_VARIANT["csv"]] - measured_gb = csv_bytes / 1e9 coeff = {fmt: sizes[WORKING_VARIANT[fmt]] / csv_bytes for fmt in FORMATS} proj_bytes_600 = {fmt: coeff[fmt] * REPORT_SCALE_GB * 1e9 for fmt in FORMATS} tables: dict[str, tuple[list[str], list[list]]] = {} tables["t1_storage_footprint"] = ( - ["format", "measured_mb", "coeff_vs_csv", "archival_mb (compressed)", - "projected_600gb_gb", "projected_1200gb_gb", "projected_6000gb_gb (all projected)"], - [[fmt, round(sizes[WORKING_VARIANT[fmt]] / 1e6, 1), round(coeff[fmt], 2), - round(sizes[ARCHIVAL_VARIANT[fmt]] / 1e6, 1), - *[round(coeff[fmt] * s * 1e9 / 1e9, 0) for s in SCALES_GB]] for fmt in FORMATS], + ["format", "measured", "vs CSV", "archival (compressed)", + "projected @600 GB", "projected @1.2 TB", "projected @6 TB"], + [[FMT_NAMES[fmt], human_size(sizes[WORKING_VARIANT[fmt]]), f"{coeff[fmt]:.2f}x", + human_size(sizes[ARCHIVAL_VARIANT[fmt]]), + *[human_size(coeff[fmt] * s * 1e9) for s in SCALES_GB]] for fmt in FORMATS], ) tables["t2_measured_medians"] = ( - ["format", "query", "wall_s", "peak_rss_mb", "cpu_util_pct", "read_mb"], - [[fmt, f"q{qn}", medians[(fmt, qn)]["wall_s"], medians[(fmt, qn)]["peak_rss_mb"], - medians[(fmt, qn)]["cpu_util_pct"], medians[(fmt, qn)]["read_mb"]] + ["format", "query", "wall time", "peak RSS", "CPU util", "read"], + [[FMT_NAMES[fmt], f"q{qn}", human_time(medians[(fmt, qn)]["wall_s"]), + f"{medians[(fmt, qn)]['peak_rss_mb']:.0f} MB", + f"{medians[(fmt, qn)]['cpu_util_pct']:.0f}%", + f"{medians[(fmt, qn)]['read_mb']:.1f} MB"] for fmt in FORMATS for qn in QUERY_NUMBERS], ) tables["t3_projected_600gb"] = ( - ["format", "query", "projected_wall", "flag"], - [[fmt, f"q{qn}", human_time(proj[(fmt, qn, REPORT_SCALE_GB)][0]), proj[(fmt, qn, REPORT_SCALE_GB)][1]] + ["format", "query", "projected wall time", "flag"], + [[FMT_NAMES[fmt], f"q{qn}", human_time(proj[(fmt, qn, REPORT_SCALE_GB)][0]), proj[(fmt, qn, REPORT_SCALE_GB)][1]] for fmt in FORMATS for qn in QUERY_NUMBERS], ) tables["t4_hardware_600gb"] = ( - ["format", "disk_tb", "ram_gb", "cores", "nodes", "est_cost_usd"], - [[fmt, sizing[(fmt, REPORT_SCALE_GB)]["disk_tb"], sizing[(fmt, REPORT_SCALE_GB)]["ram_gb"], - sizing[(fmt, REPORT_SCALE_GB)]["cores"], sizing[(fmt, REPORT_SCALE_GB)]["nodes"], - round(sizing[(fmt, REPORT_SCALE_GB)]["est_cost_usd"])] for fmt in FORMATS], + ["format", "configuration", "chassis", "RAM", "disk", "extra nodes", "total"], + [[FMT_NAMES[fmt], config_string(sizing[(fmt, REPORT_SCALE_GB)], prices), + money(sizing[(fmt, REPORT_SCALE_GB)]["base_usd"]), + money(sizing[(fmt, REPORT_SCALE_GB)]["ram_usd"]), + money(sizing[(fmt, REPORT_SCALE_GB)]["disk_usd"]), + money(sizing[(fmt, REPORT_SCALE_GB)]["extra_nodes_usd"]), + money(sizing[(fmt, REPORT_SCALE_GB)]["est_cost_usd"])] for fmt in FORMATS], ) scoreboards = build_scoreboards(medians, proj, sizes, sizing) tables["t5_weighted_scores"] = ( ["scale", "format", "search_pts", "ram_pts", "disk_pts", "total"], - [[scale, fmt, board[fmt]["search_pts"], board[fmt]["ram_pts"], board[fmt]["disk_pts"], board[fmt]["total"]] + [[scale, FMT_NAMES[fmt], board[fmt]["search_pts"], board[fmt]["ram_pts"], board[fmt]["disk_pts"], board[fmt]["total"]] for scale, board in scoreboards.items() for fmt in FORMATS], ) for name, (headers, rows) in tables.items(): write_csv(tables_dir / f"{name}.csv", headers, rows) - chart_c1(charts_dir, sizes, proj_bytes_600) + winner = max(FORMATS, key=lambda f: scoreboards["projected_600gb"][f]["total"]) + chart_c1(charts_dir, sizes, proj_bytes_600, coeff) chart_c2(charts_dir, medians, sizing) chart_c3(charts_dir, medians, proj) - chart_c4(charts_dir, medians, proj, measured_gb) - chart_c5(charts_dir, scoreboards) + chart_c4(charts_dir, medians, proj, f"measured {human_size(csv_bytes)}") + chart_c5(charts_dir, scoreboards, winner) chart_c6(charts_dir, proj, sizing) charts = sorted(p.name for p in charts_dir.glob("*.png")) if len(charts) != 14: raise ValueError(f"expected 14 charts, wrote {len(charts)}: {charts}") - winner = max(FORMATS, key=lambda f: scoreboards["projected_600gb"][f]["total"]) - report_text = assemble_report(report_dir, tables, scoreboards, environment, diagrams, winner) + report_text = assemble_report(tables, scoreboards, environment, diagrams, winner, sizing, prices) report_path = report_dir / "REPORT.md" report_path.write_text(report_text, encoding="ascii", newline="\n") @@ -564,6 +665,7 @@ def main() -> int: "tables": len(tables), "report_bytes": report_path.stat().st_size, "winner_600gb": winner, + "prices_as_of": prices["as_of"], } for fmt in FORMATS: entries[f"score600_{fmt}"] = scoreboards["projected_600gb"][fmt]["total"] -- 2.53.0.windows.2