diff --git a/.gitignore b/.gitignore index d07d4cb..4e1c633 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ # Generated pipeline artifacts - reproducible from specs + rules + code # (see docs/rules/build-pipeline-tasks.md) -out/ +out/* +!out/report/ +!out/report/** +!out/config/ +!out/config/** +!out/.done/ +!out/.done/** # Python __pycache__/ diff --git a/out/.done/01.ok b/out/.done/01.ok new file mode 100644 index 0000000..839d008 --- /dev/null +++ b/out/.done/01.ok @@ -0,0 +1,13 @@ +task: '01' +status: ok +config_file: C:/Projects/Public/LabDataStorageEvaluation/out/config/lab_config.yaml +config_bytes: 5827 +seed: 20260711 +batches: 4 +wafers: 12 +coupons: 588 +friction_coupons: 480 +tracks: 1440 +cycle_rows: 1440000 +loop_points: 2880000 +xrf_points: 235200 diff --git a/out/.done/02.ok b/out/.done/02.ok new file mode 100644 index 0000000..6f22362 --- /dev/null +++ b/out/.done/02.ok @@ -0,0 +1,6 @@ +task: '02' +status: ok +process_flow_file: C:/Projects/Public/LabDataStorageEvaluation/out/report/process_flow.md +process_flow_bytes: 4205 +diagrams: 6 +diagram_bytes: 2924 diff --git a/out/.done/03.ok b/out/.done/03.ok new file mode 100644 index 0000000..742ef91 --- /dev/null +++ b/out/.done/03.ok @@ -0,0 +1,11 @@ +task: '03' +status: ok +csv_root: C:/Projects/Public/LabDataStorageEvaluation/out/csv +files: 8718 +total_rows: 4636776 +total_bytes: 148654472 +coupons: 588 +tracks: 1440 +cycle_rows: 1440000 +loop_points: 2880000 +xrf_points: 235200 diff --git a/out/.done/04.ok b/out/.done/04.ok new file mode 100644 index 0000000..993026b --- /dev/null +++ b/out/.done/04.ok @@ -0,0 +1,11 @@ +task: '04' +status: ok +full_files: 588 +full_bytes: 311300222 +hybrid_bytes: 1887322 +coupons: 588 +tracks: 1440 +cycle_rows: 1440000 +loop_points: 2880000 +proc_peak_rss_mb: 119.1 +proc_cpu_seconds: 19.1 diff --git a/out/.done/05.ok b/out/.done/05.ok new file mode 100644 index 0000000..e2c6b06 --- /dev/null +++ b/out/.done/05.ok @@ -0,0 +1,12 @@ +task: '05' +status: ok +db_file: C:/Projects/Public/LabDataStorageEvaluation/out/sqlite/tribo.db +db_bytes: 123400192 +tables: 15 +total_rows: 4638223 +friction_cycles: 1440000 +friction_loop_points: 2880000 +tracks: 1440 +track_summary: 1440 +proc_peak_rss_mb: 323.0 +proc_cpu_seconds: 19.9 diff --git a/out/.done/06.ok b/out/.done/06.ok new file mode 100644 index 0000000..92304f5 --- /dev/null +++ b/out/.done/06.ok @@ -0,0 +1,18 @@ +task: '06' +status: ok +database: lab_data +tables: 15 +total_rows: 4638223 +live_bytes: 397467648 +friction_cycles_bytes: 99753984 +friction_loop_points_bytes: 267796480 +copy_seconds: 19.8 +total_seconds: 23.0 +dump: skipped (pg_dump unavailable on this host) +proc_peak_rss_mb: 58.6 +proc_cpu_seconds: 18.7 +host_cpu_avg_pct: 2.4 +host_cpu_max_pct: 3.7 +host_ram_used_baseline_mb: 617 +host_ram_used_peak_mb: 681 +host_ram_used_delta_mb: 64 diff --git a/out/.done/07.ok b/out/.done/07.ok new file mode 100644 index 0000000..1f2fa9d --- /dev/null +++ b/out/.done/07.ok @@ -0,0 +1,13 @@ +task: '07' +status: ok +triples: 17419713 +nt_gz_bytes: 126487650 +ttl_bytes: 596401229 +store_bytes: 1739414085 +store_load_seconds: 33.6 +coupons: 588 +tracks: 1440 +cycle_nodes: 1440000 +loop_nodes: 2880000 +proc_peak_rss_mb: 2185.2 +proc_cpu_seconds: 151.8 diff --git a/out/.done/08.ok b/out/.done/08.ok new file mode 100644 index 0000000..d9bf08e --- /dev/null +++ b/out/.done/08.ok @@ -0,0 +1,19 @@ +task: '08' +status: ok +query_files: 56 +q1_rows: 1000 +q1_sha256: bb6d871275d0 +q2_rows: 33 +q2_sha256: 62ca30e729a8 +q3_rows: 480 +q3_sha256: a3a73414201f +q4_rows: 469 +q4_sha256: 38281dcf7896 +q5_rows: 4 +q5_sha256: f3ee32dbc981 +q6_rows: 480 +q6_sha256: 9cc28d51ee35 +q7_rows: 2 +q7_sha256: 1991f8bf2217 +proc_peak_rss_mb: 61.1 +proc_cpu_seconds: 1.7 diff --git a/out/.done/09.ok b/out/.done/09.ok new file mode 100644 index 0000000..e58d042 --- /dev/null +++ b/out/.done/09.ok @@ -0,0 +1,14 @@ +task: '09' +status: ok +cells_total: 35 +cells_ok: 35 +cells_invalid: 0 +cells_timeout: 0 +cells_error: 0 +session_wall_s: 915.5 +cold_cache_pass: skipped (Windows host) +session_host_cpu_avg_pct: 0.4 +session_host_cpu_max_pct: 9.2 +session_host_ram_used_baseline_mb: 909 +session_host_ram_used_peak_mb: 914 +session_host_ram_used_delta_mb: 5 diff --git a/out/.done/10.ok b/out/.done/10.ok new file mode 100644 index 0000000..5e5d819 --- /dev/null +++ b/out/.done/10.ok @@ -0,0 +1,16 @@ +task: '10' +status: ok +extrapolation_rows: 105 +sizing_rows: 15 +flags_ok: 53 +flags_impractical: 35 +flags_fail: 17 +pg_index_share: 0.341 +pg_index_mb: 128.5 +prices_as_of: 2026-07-11 +note_pandas: pandas CSV variant not measured; its RAM is O(n) - infeasible above ~10 GB (spec 10) +coeff_csv: 1.0 +coeff_json: 2.094 +coeff_sqlite: 0.83 +coeff_pg: 2.674 +coeff_rdf: 11.701 diff --git a/out/.done/11.ok b/out/.done/11.ok new file mode 100644 index 0000000..ccb2f2a --- /dev/null +++ b/out/.done/11.ok @@ -0,0 +1,12 @@ +task: '11' +status: ok +charts: 14 +tables: 5 +report_bytes: 16091 +winner_600gb: pg +prices_as_of: 2026-07-11 +score600_csv: 28.9 +score600_json: 24.0 +score600_sqlite: 47.0 +score600_pg: 58.6 +score600_rdf: 0.9 diff --git a/out/config/hw_prices.yaml b/out/config/hw_prices.yaml new file mode 100644 index 0000000..5a40c97 --- /dev/null +++ b/out/config/hw_prices.yaml @@ -0,0 +1,18 @@ +# Hardware cost parameters for extrapolation (task 10). +# Street prices retrieved 2026-07-11 from the sources below (server DRAM is +# in a documented 2025-2026 price surge - see market_context). The cost +# model buys whole components: 64 GB RDIMM modules and 7.68 TB NVMe drives. +# Operator-editable; re-run extrapolate.py after changes (code-config-yaml). +as_of: "2026-07-11" +ram_module_gb: 64 +ram_module_usd: 1887.0 # A-Tech 64GB (2x32GB) DDR5-5600 ECC RDIMM +nvme_drive_tb: 7.68 +nvme_drive_usd: 3995.0 # Cloud Ninjas NEW 7.68TB NVMe U.2 1DWPD (Dell 14-16G) +server_16_core_usd: 4618.0 # Supermicro AS-1015CS-TNR 1U (EPYC 9004/9005), Broadberry starting config +server_32_core_usd: 6272.0 # Supermicro AS-1115CS-TNR 1U, Broadberry starting config +extra_node_usd: 4618.0 # one additional entry 1U node (AS-1015CS-TNR chassis) +sources: + ram: https://atechmemory.com/collections/ddr5-memory-ram + nvme: https://cloudninjas.com/products/new-7-68tb-nvme-u-2-1dwpd-sie-2-5-enterprise-solid-state-drive-for-14th-15th-16th-gen-dell + servers: https://www.broadberry.com/amd-epyc-9004-supermicro-servers + market_context: https://www.techpowerup.com/342331/server-dram-pricing-jumps-50-only-70-of-orders-getting-filled diff --git a/out/config/lab_config.yaml b/out/config/lab_config.yaml new file mode 100644 index 0000000..1887ffc --- /dev/null +++ b/out/config/lab_config.yaml @@ -0,0 +1,246 @@ +# Generated by make_lab_config.py (task 01) from docs/specs/01_lab_configuration.md. +# Do not hand-edit: regenerate via `python make_lab_config.py`. +project: LabDataStorageEvaluation +description: Simulated tribology laboratory (Sandia Pt-Au LDRD context) +spec: docs/specs/01_lab_configuration.md +seed: 20260711 +material_system: + substrate: + material: Ti-6Al-4V + dimensions_mm: + - 10 + - 10 + - 3 + adhesion_layer: + material: Cr + process: sputtered + coating: + material: Pt-Au + deposition: composition gradient across each wafer + thickness_um_range: + - 0.3 + - 1.1 +instruments: +- instrument_id: rapid + role: friction + name: RAPID custom high-throughput parallelized 6-probe tribometer + data_produced: COF vs cycle per track; test conditions +- instrument_id: ti980 + role: nanoindentation + name: Bruker TI980 TriboIndenter + data_produced: hardness, reduced modulus (25 indents per coupon) +- instrument_id: m4_tornado + role: composition + name: Bruker M4 Tornado micro-XRF + data_produced: Pt/Au wt% map per coupon (20x20 grid) +- instrument_id: pvd200 + role: deposition + name: Kurt J. Lesker PVD 200 sputter-down + data_produced: batch deposition parameters +- instrument_id: afm + role: surface_roughness + name: AFM + data_produced: topography image metadata + Ra/Rq per coupon +- instrument_id: profilometer + role: film_thickness + name: Optical profilometry + data_produced: thickness map per coupon (10x10 grid) +- instrument_id: simtra + role: simulation + name: SIMTRA sputter transport Monte-Carlo + data_produced: deposition atom-energy / composition profiles per deposition run +deposition_matrix: +- batch_code: B721 + pt_gun_tilt_deg: 20 + au_gun_tilt_deg: 0 + pt_power_W: 150 + au_power_W: 50 + pt_discharge_V: 432 + au_discharge_V: 311 +- batch_code: B722 + pt_gun_tilt_deg: 20 + au_gun_tilt_deg: 20 + pt_power_W: 100 + au_power_W: 100 + pt_discharge_V: 399 + au_discharge_V: 352 +- batch_code: B723 + pt_gun_tilt_deg: 20 + au_gun_tilt_deg: 20 + pt_power_W: 150 + au_power_W: 50 + pt_discharge_V: 430 + au_discharge_V: 311 +- batch_code: B724 + pt_gun_tilt_deg: 0 + au_gun_tilt_deg: 20 + pt_power_W: 50 + au_power_W: 150 + pt_discharge_V: 376 + au_discharge_V: 340 +hierarchy: + batches: 4 + wafers_per_batch: 3 + coupon_grid_per_wafer: + - 7 + - 7 + coupons_per_wafer: 49 +friction_assignment: + friction_coupons_total: 480 + reserve_coupons_total: 108 + runs: 4 + coupons_per_run: 120 + plates_per_run: 5 + probes_per_plate: 6 + coupons_per_probe_square: 4 + tracks_per_friction_coupon: 3 + counterfaces_per_holder: 3 + fresh_counterface_per_track: true + reserve_grid_positions: + - 1 + - 4 + - 7 + - 22 + - 25 + - 28 + - 43 + - 46 + - 49 +volumes: + coupons_total: 588 + tracks_total: 1440 + cycles_per_track: 1000 + cycle_rows_total: 1440000 + loop_every_n_cycles: 100 + loops_per_track: 10 + loop_points_per_loop: 200 + loop_points_total: 2880000 + xrf_grid: + - 20 + - 20 + xrf_points_per_coupon: 400 + xrf_points_total: 235200 + profilometry_grid: + - 10 + - 10 + profilometry_points_per_coupon: 100 + nanoindentation_indents_per_coupon: 25 + simtra_rows_per_batch: 1000 +runs: +- run_code: R1 + batch_code: B721 + environment: lab_air + date: '2026-04-06' + operator: A. Reyes +- run_code: R2 + batch_code: B722 + environment: lab_air + date: '2026-04-13' + operator: K. Patel +- run_code: R3 + batch_code: B723 + environment: dry_n2 + date: '2026-04-20' + operator: A. Reyes +- run_code: R4 + batch_code: B724 + environment: dry_n2 + date: '2026-04-27' + operator: M. Novak +schedule: + deposition_start_date: '2026-03-02' + batch_interval_days: 7 + characterization_lag_days: 3 + run_start_time: 09:00:00 + track_interval_s: 2100 +test_conditions: + normal_load_mN: 100 + stroke_mm: 1.0 + speed_mm_s: 1.0 + counterface: + material: ruby (Al2O3) + diameter_mm: 3.175 + rh_pct: + lab_air: 45 + dry_n2: 2 + temperature_C: 23 + temperature_tolerance_C: 1 +physical_models: + au_gradient: + description: linear Au wt% gradient along wafer x, batch-dependent center + batch_center_wtpct: + B721: 8 + B722: 25 + B723: 10 + B724: 60 + wafer_span_wtpct: 5 + xrf_noise_sd_wtpct: 0.3 + cof_run_in: + formula: cof(c) = cof_ss + (cof_0 - cof_ss) * exp(-c / tau) + N(0, sigma) + cof_0_range: + - 0.35 + - 0.5 + tau_cycles_range: + - 100 + - 400 + noise_sd: 0.01 + cof_steady_state: + formula: cof_ss = intercept + slope_per_au_wtpct * au_wtpct_mean, clamped to >= floor + lab_air: + intercept: 0.32 + slope_per_au_wtpct: -0.0015 + dry_n2: + intercept: 0.24 + slope_per_au_wtpct: -0.0012 + floor: 0.12 + hardness_GPa: + formula: H = intercept + slope_per_au_wtpct * au_wtpct_mean + N(0, noise_sd) + intercept: 8.5 + slope_per_au_wtpct: -0.05 + noise_sd: 0.25 + reduced_modulus_GPa: + formula: Er = intercept + slope_per_au_wtpct * au_wtpct_mean + N(0, noise_sd) + intercept: 190 + slope_per_au_wtpct: -0.6 + noise_sd: 4 + roughness_ra_nm: + distribution: lognormal + median_nm: 5 + sigma_log: 0.3 + rq_over_ra: 1.25 + thickness_um: + profile: radial parabolic across wafer plus noise + center_um: 1.05 + edge_um: 0.35 + noise_sd_um: 0.02 + range_um: + - 0.3 + - 1.1 + wear_archard: + formula: V = k * F * s; k = k0 * (1 + slope_per_au_wtpct * au_wtpct_mean), lognormal noise, clamped + to >= k_floor + k0_mm3_per_N_m: + lab_air: 1.5e-07 + dry_n2: 6.0e-08 + slope_per_au_wtpct: + lab_air: -0.008 + dry_n2: -0.01 + k_floor_mm3_per_N_m: 1.0e-09 + noise_sigma_log: 0.15 + friction_loop: + force_noise_sd_mN: 0.3 + nanoindentation: + max_load_mN: 10 + indent_grid: + - 5 + - 5 + grid_pitch_um: 20 + simtra: + angle_deg_range: + - 0 + - 90 + energy_eV_max: 50 + surface_binding_energy_eV: + pt: 5.84 + au: 3.81 + flux_per_W: 0.02 diff --git a/out/report/REPORT.md b/out/report/REPORT.md new file mode 100644 index 0000000..3c63643 --- /dev/null +++ b/out/report/REPORT.md @@ -0,0 +1,318 @@ +# Laboratory Data Storage Format Evaluation - Final Report + +Generated by task 11 (report.py) from the measured benchmark matrix (task 09) and the extrapolation model (task 10). Every number at scales beyond the 150 MB corpus is a PROJECTION and is labeled as such; scaling assumptions are listed in section 6. + +## 1. Executive summary + +**Recommendation: PostgreSQL as the system of record.** The measured matrix and the projections confirm the expected outcome: + +- **PostgreSQL** is the only format whose seven benchmark queries stay below the 1-hour practicality bound at every projected scale (600 GB, 1.2 TB, 6 TB); at 6 TB it needs one 16-core node with 551 GB RAM at an estimated $33,586 (bill of materials, prices as of 2026-07-11). Partitioned parallel scans and the track_summary materialized view are the scaling levers. +- **CSV** remains the canonical raw archive: byte-reproducible, instrument-native, and the best compressor (tar.gz ~2.8x); its full scans exceed 1 h beyond ~600 GB, so it is an archive, not a query layer. +- **SQLite** is an excellent single-user store up to ~600 GB (all queries OK), then its single-threaded scans degrade (Q5/Q7 IMPRACTICAL at 1.2 TB). +- **Hybrid JSON-LD** (metadata + sourceFile links, 1.8 MiB at 150 MB corpus) is the exchange format; the FULL variant re-reads the whole corpus per scan and becomes impractical past 600 GB. +- **RDF (materialized triplestore)** fails at scale: 11.7x storage blow-up, 5 of 7 queries FAIL at 6 TB, and the hot set needs 13.7 TB RAM across 14 nodes (~$527,732 projected). Semantic access should be a virtual layer (e.g. Ontop OBDA) over PostgreSQL instead. + +## 2. Methodology snapshot + +35 cells (Q1-Q7 x 5 formats); per cell 1 warm-up + 3 measured runs in isolated subprocesses sampled with psutil at 50 ms; randomized cell order; 30-min timeout; every run validated against canonical results (PostgreSQL cross-validated with SQLite, tolerance 1e-9). Medians are reported; raw runs and min/max live in out/bench/results_raw.csv. The cold-cache pass is skipped on this Windows host and marked as such. PostgreSQL runs on a remote host; its wall times include the LAN round-trip, and server-side metrics come from pg_stat_statements and Prometheus (docs/rules/bench-methodology.md section 6). + +## 3. Environment + +``` +benchmark_host_os: Windows-2025Server-10.0.26100-SP0 +benchmark_host_cpu: Intel64 Family 6 Model 141 Stepping 1, GenuineIntel +benchmark_host_cores_logical: 4 +benchmark_host_cores_physical: 4 +benchmark_host_ram_gib: 8.0 +python: 3.12.10 +benchmark_host_disks: VMware Virtual NVMe Disk / SSD / 100 GB +sqlite: 3.49.1 +psycopg: 3.3.4 +pyoxigraph: 0.5.9 +ijson: 3.5.1 +psutil: 7.2.2 +postgresql_server: PostgreSQL 16.14 (Ubuntu 16.14-0ubuntu0.24.04.1) on x86_64-pc-linux-gnu, compiled by gcc (Ubuntu 13.3.0-6ubuntu2~24.04.1) 13.3.0, 64-bit +postgresql_host: 192.168.10.73 (remote; 4 cores / 3.8 GiB, see .done/06.ok) +cold_cache_pass: skipped (Windows host, no page-cache drop - bench-methodology section 6) +pg_backend_sampling: not possible for a remote server; Prometheus windows + pg_stat_statements deltas instead +``` + +## 4. Storage footprint + +| format | measured | vs CSV | archival (compressed) | projected @600 GB | projected @1.2 TB | projected @6 TB | +|---|---|---|---|---|---|---| +| CSV | 149 MB | 1.00x | 53 MB | 600 GB | 1.20 TB | 6.00 TB | +| JSON-LD | 311 MB | 2.09x | 57 MB | 1.26 TB | 2.51 TB | 13 TB | +| SQLite | 123 MB | 0.83x | 55 MB | 498 GB | 996 GB | 4.98 TB | +| PostgreSQL | 397 MB | 2.67x | 53 MB | 1.60 TB | 3.21 TB | 16 TB | +| RDF triplestore | 1.74 GB | 11.70x | 126 MB | 7.02 TB | 14 TB | 70 TB | + +![C1 - disk footprint](charts/c1_disk_footprint.png) + +## 5. Measured performance (150 MB corpus) + +| format | query | wall time | peak RSS | CPU util | read | +|---|---|---|---|---|---| +| CSV | q1 | 0.06 s | 4 MB | 0% | 0.0 MB | +| CSV | q2 | 0.17 s | 20 MB | 16% | 2.5 MB | +| CSV | q3 | 1.15 s | 21 MB | 24% | 31.8 MB | +| CSV | q4 | 0.71 s | 21 MB | 22% | 16.8 MB | +| CSV | q5 | 1.09 s | 21 MB | 23% | 31.8 MB | +| CSV | q6 | 0.28 s | 20 MB | 17% | 1.2 MB | +| CSV | q7 | 1.27 s | 21 MB | 23% | 32.5 MB | +| JSON-LD | q1 | 0.11 s | 22 MB | 10% | 1.4 MB | +| JSON-LD | q2 | 1.48 s | 24 MB | 24% | 53.1 MB | +| JSON-LD | q3 | 9.94 s | 24 MB | 25% | 297.6 MB | +| JSON-LD | q4 | 9.92 s | 24 MB | 25% | 297.8 MB | +| JSON-LD | q5 | 9.96 s | 24 MB | 25% | 297.6 MB | +| JSON-LD | q6 | 9.79 s | 24 MB | 24% | 297.8 MB | +| JSON-LD | q7 | 10 s | 24 MB | 24% | 297.9 MB | +| SQLite | q1 | 0.06 s | 4 MB | 0% | 0.0 MB | +| SQLite | q2 | 0.06 s | 4 MB | 0% | 0.0 MB | +| SQLite | q3 | 0.11 s | 4 MB | 0% | 0.0 MB | +| SQLite | q4 | 0.06 s | 4 MB | 0% | 0.0 MB | +| SQLite | q5 | 0.77 s | 25 MB | 23% | 117.9 MB | +| SQLite | q6 | 0.06 s | 4 MB | 0% | 0.0 MB | +| SQLite | q7 | 0.71 s | 25 MB | 23% | 93.0 MB | +| PostgreSQL | q1 | 0.22 s | 42 MB | 18% | 4.1 MB | +| PostgreSQL | q2 | 0.22 s | 42 MB | 20% | 4.1 MB | +| PostgreSQL | q3 | 0.22 s | 42 MB | 18% | 4.1 MB | +| PostgreSQL | q4 | 0.22 s | 42 MB | 18% | 4.1 MB | +| PostgreSQL | q5 | 0.44 s | 42 MB | 10% | 4.1 MB | +| PostgreSQL | q6 | 0.22 s | 42 MB | 19% | 4.1 MB | +| PostgreSQL | q7 | 0.33 s | 42 MB | 12% | 4.1 MB | +| RDF triplestore | q1 | 0.11 s | 32 MB | 17% | 3.6 MB | +| RDF triplestore | q2 | 2.81 s | 85 MB | 24% | 375.6 MB | +| RDF triplestore | q3 | 46 s | 264 MB | 25% | 5445.1 MB | +| RDF triplestore | q4 | 20 s | 168 MB | 25% | 2694.6 MB | +| RDF triplestore | q5 | 60 s | 264 MB | 25% | 5390.5 MB | +| RDF triplestore | q6 | 0.17 s | 46 MB | 16% | 15.9 MB | +| RDF triplestore | q7 | 41 s | 264 MB | 25% | 5380.5 MB | + +Sub-interval note: cells faster than the 50 ms sampling cadence (fast SQLite/CSV point reads) under-report client RSS/CPU; wall times are exact. + +![C2 - RAM requirement](charts/c2_ram_requirement.png) + +Per-query charts: [Q1](charts/c3_q1_wall_time.png), [Q2](charts/c3_q2_wall_time.png), [Q3](charts/c3_q3_wall_time.png), [Q4](charts/c3_q4_wall_time.png), [Q5](charts/c3_q5_wall_time.png), [Q6](charts/c3_q6_wall_time.png), [Q7](charts/c3_q7_wall_time.png) + +## 6. Projections (600 GB / 1.2 TB / 6 TB) + +All values in this section are projected, never measured. Laws per access pattern (spec 10): flat / index-depth log / linear result set / linear x depth / full scan / parallel partitioned scan; constants calibrated on the measured point after subtracting the per-format harness floor. Flags: IMPRACTICAL > 1 h, FAIL > 24 h. + +Projected wall times at 600 GB: + +| format | query | projected wall time | flag | +|---|---|---|---| +| CSV | q1 | 0.06 s | OK | +| CSV | q2 | 7.1 min | OK | +| CSV | q3 | 73.5 min | IMPRACTICAL | +| CSV | q4 | 44.1 min | OK | +| CSV | q5 | 69.6 min | IMPRACTICAL | +| CSV | q6 | 14.5 min | OK | +| CSV | q7 | 81.6 min | IMPRACTICAL | +| JSON-LD | q1 | 0.11 s | OK | +| JSON-LD | q2 | 92.0 min | IMPRACTICAL | +| JSON-LD | q3 | 11.0 h | IMPRACTICAL | +| JSON-LD | q4 | 11.0 h | IMPRACTICAL | +| JSON-LD | q5 | 11.0 h | IMPRACTICAL | +| JSON-LD | q6 | 10.9 h | IMPRACTICAL | +| JSON-LD | q7 | 11.2 h | IMPRACTICAL | +| SQLite | q1 | 0.06 s | OK | +| SQLite | q2 | 1.27 s | OK | +| SQLite | q3 | 5.7 min | OK | +| SQLite | q4 | 0.06 s | OK | +| SQLite | q5 | 47.5 min | OK | +| SQLite | q6 | 5.18 s | OK | +| SQLite | q7 | 43.7 min | OK | +| PostgreSQL | q1 | 0.22 s | OK | +| PostgreSQL | q2 | 0.22 s | OK | +| PostgreSQL | q3 | 3.42 s | OK | +| PostgreSQL | q4 | 11 s | OK | +| PostgreSQL | q5 | 3.7 min | OK | +| PostgreSQL | q6 | 4.06 s | OK | +| PostgreSQL | q7 | 115 s | OK | +| RDF triplestore | q1 | 0.11 s | OK | +| RDF triplestore | q2 | 3.0 h | IMPRACTICAL | +| RDF triplestore | q3 | 2.1 d | FAIL | +| RDF triplestore | q4 | 22.1 h | IMPRACTICAL | +| RDF triplestore | q5 | 2.8 d | FAIL | +| RDF triplestore | q6 | 5.9 min | OK | +| RDF triplestore | q7 | 45.3 h | FAIL | + +Degradation by data size: [point read](charts/c4_degradation_point_read.png), [indexed / join](charts/c4_degradation_indexed.png), [full scan](charts/c4_degradation_full_scan.png) + +## 7. Hardware sizing and cost at 600 GB (projected) + +Bill-of-materials costing with street prices as of **2026-07-11** (sources below). Every configuration buys whole components: 64 GB DDR5 ECC RDIMM modules, 7.68 TB enterprise NVMe U.2 drives, and a priced 1U chassis; disk capacity includes 30% free-space headroom. + +| format | configuration | chassis | RAM | disk | extra nodes | total | +|---|---|---|---|---|---|---| +| CSV | 1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2 | $4,618 | $1,887 | $3,995 | $0 | $10,500 | +| JSON-LD | 1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2 | $4,618 | $1,887 | $3,995 | $0 | $10,500 | +| SQLite | 1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2 | $4,618 | $1,887 | $3,995 | $0 | $10,500 | +| PostgreSQL | 1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2 | $4,618 | $1,887 | $3,995 | $0 | $10,500 | +| RDF triplestore | 2 x 1U Supermicro AS-1015CS-TNR (16 cores); 22 x 64 GB DDR5 ECC RDIMM; 2 x 7.68 TB NVMe U.2 | $4,618 | $41,514 | $7,990 | $4,618 | $58,740 | + +Component prices (retrieved 2026-07-11): + +- 64 GB DDR5-5600 ECC RDIMM (A-Tech, 2x32GB kit): $1,887 - +- 7.68 TB NVMe U.2 1DWPD enterprise SSD (Dell 14-16G compatible, Cloud Ninjas): $3,995 - +- 1U AMD EPYC 9004/9005 chassis (Broadberry starting configurations): Supermicro AS-1015CS-TNR $4,618, Supermicro AS-1115CS-TNR $6,272 - +- Market context - server DRAM is in a documented 2025-2026 price surge: + +Full sizing at 1.2 TB and 6 TB: out/bench/hardware_sizing.csv. RDF exceeds the 1 TB single-node RAM ceiling already at 600 GB and reaches 14 nodes at 6 TB. + +![C6 - cost vs performance](charts/c6_cost_vs_performance.png) + +## 8. Weighted scoring + +Subscores are 10 x best/value per metric; weights: search x5, RAM economy x2, disk economy x1; max 80. Search uses the inverse geometric mean of Q1-Q7 (FAIL at 600 GB zeroes the search subscore). Left: measured scale; right: projected 600 GB. + +Measured scale (150 MB): + +| format | search (x5) | RAM (x2) | disk (x1) | total (max 80) | +|---|---|---|---|---| +| SQLite | 50.0 | 16.9 | 10.0 | 76.9 | +| CSV | 15.5 | 20.0 | 8.3 | 43.8 | +| PostgreSQL | 26.2 | 10.0 | 3.1 | 39.3 | +| JSON-LD | 1.7 | 17.2 | 4.0 | 22.9 | +| RDF triplestore | 1.3 | 1.6 | 0.7 | 3.6 | + +Projected at 600 GB: + +| format | search (x5) | RAM (x2) | disk (x1) | total (max 80) | +|---|---|---|---|---| +| PostgreSQL | 50.0 | 5.5 | 3.1 | 58.6 | +| SQLite | 21.8 | 15.2 | 10.0 | 47.0 | +| CSV | 0.6 | 20.0 | 8.3 | 28.9 | +| JSON-LD | 0.1 | 20.0 | 4.0 | 24.0 | +| RDF triplestore | 0.0 | 0.2 | 0.7 | 0.9 | + +![C5 - weighted score](charts/c5_weighted_score.png) + +## 9. Use-case mapping + +| Use case | Recommended format(s) | +|---|---| +| Interactive analysis (joins, aggregations) | PostgreSQL; SQLite acceptable single-user up to ~600 GB | +| Report generation (repeated summaries) | PostgreSQL (track_summary materialized view) | +| Search / filtering | PostgreSQL or SQLite (indexed); flat formats need full scans | +| Archiving | CSV tree + tar.gz (canonical raw) plus pg_dump of the system of record | +| Inter-lab exchange | Hybrid JSON-LD (metadata + sourceFile links to CSV) | +| Semantic / ontology queries | Virtual RDF layer over PostgreSQL (e.g. Ontop OBDA), not a materialized triplestore | + +## 10. Limitations + +- Single-node measurements on one Windows host + one remote PostgreSQL host; no cluster or cloud variance. +- The corpus is simulated (physically plausible models, fixed seed 20260711); real instrument data may have different value distributions, though volumes and shapes match the lab. +- Extrapolation uses analytic laws calibrated on a single 150 MB point; no intermediate-scale validation runs were performed. +- Client wall times include a per-format harness floor (interpreter start, imports, connection); the floor is assumed constant across scales. The cheapest cell of each format therefore projects flat. +- The cold-cache matrix pass is skipped (no page-cache drop on Windows); all measured runs are warm-cache. +- The pandas CSV variant was not measured; its RAM is O(n) and it is infeasible above roughly 10 GB (spec 10). +- RDF numbers reflect the compact modeling and a client that derives run-in / steady-state from raw cycles; SPARQL-side aggregation engines could shift (not remove) the scan penalty. +- Hardware prices are spot street prices retrieved 2026-07-11 during a documented server-DRAM price surge; re-cost by editing out/config/hw_prices.yaml and re-running tasks 10-11. + +## Appendix A - process flow diagrams (task 02) + +### D1 - Coupon assembly + +```mermaid +flowchart LR + A([Ti-6Al-4V Base 10x10x3 mm]) --> B[Cleaning] + B --> B1([Rinsed in deionized water]) + B1 --> B2([Sonicated in cleaning solution]) + B --> C[Smearing Adhesive] + C --> C1([Cr adhesive coating]) + C --> D[Vapor Deposition System
Kurt J. Lesker PVD 200] + D --> D1([PtAu sputter coating, gradient]) + D --> E([Test Coupon]) +``` + +### D2 - Characterization and batch assembly + +```mermaid +flowchart LR + TC([Test Coupon]) --> S[SIMTRA simulation
composition / atom energies] + S --> OP[Optical profilometry
film thickness 0.3-1.1 um] + OP --> TW[Assemble Test Wafer
x49 coupons] + TW --> TB[Assemble Test Batch
x3 wafers] + TB --> B721[B721
Pt 150 W / Au 50 W] + TB --> B722[B722
Pt 100 W / Au 100 W] + TB --> B723[B723
Pt 150 W / Au 50 W] + TB --> B724[B724
Pt 50 W / Au 150 W] +``` + +### D3 - Testing tree + +```mermaid +flowchart TD + TC([Test Coupon]) --> F[Friction
RAPID 6-probe tribometer] + F --> FA([Lab Air -> COF dataset mu_normal]) + F --> FN([Dry N2 -> COF dataset mu_dry_nit]) + TC --> NI[Nanoindentation
Bruker TI980] + NI --> NIR([Hardness, reduced modulus
25 indents per coupon]) + TC --> AF[AFM] + AF --> AFR([Topography -> Ra / Rq]) + TC --> XR[micro-XRF
Bruker M4 Tornado] + XR --> XRR([Pt/Au wt% map, 20x20 grid]) +``` + +### D4 - Tribometer session sequence + +```mermaid +sequenceDiagram + actor Op as Operator + participant Prep as Sample Prep + participant Plan as Excel Test Plan + participant SW as Control Software + participant TR as RAPID Tribometer + + Op->>Prep: Request test + Op->>Prep: Get samples + Op->>Prep: Load samples onto plates + Op->>Prep: Load ball holders (3 counterfaces per holder) + Op->>Plan: Create test plan
(Sample Plate, Plate Location, Sample ID, Save Location,
Folder Name, X/Y/Z Offset, Load, Iterations) + Op->>TR: Transfer plates to tribometer + Op->>TR: Set up platter + Op->>SW: Activate software + Op->>SW: Load plan + SW->>TR: Activate tribometer + TR-->>SW: Run reciprocating tests (execution loop, see D5) + Op->>SW: Software stop + SW-->>Op: Save avg files + Op->>TR: Remove plates and platters + Op->>SW: Close software + Op->>TR: Pull equipment out +``` + +### D5 - Execution loop + +```mermaid +flowchart TD + L[Load all counterfaces] --> P{5 plates} + P -->|for each plate| PR{6 probes in parallel} + PR -->|for each probe| C{4 coupons on coupon square} + C -->|for each coupon| T[Draw track] + T --> RC[Rotate counterface - fresh ball per track] + RC -->|3 tracks per coupon| T + T --> DONE([120 coupons x 3 tracks = 360 tracks per run]) +``` + +### D6 - Data hierarchy + +```mermaid +erDiagram + DEPOSITION_RUN ||--|| SIMTRA_PROFILE : produces + DEPOSITION_RUN ||--|| BATCH : creates + BATCH ||--|{ WAFER : contains + WAFER ||--|{ COUPON : contains + COUPON ||--|| XRF_MAP : has + COUPON ||--|| NANOINDENTATION : has + COUPON ||--|| AFM : has + COUPON ||--|| PROFILOMETRY : has + COUPON ||--o{ TRACK : "friction coupons only" + TRACK ||--|{ CYCLE : has + TRACK ||--|{ LOOP_POINT : has + TRACK ||--|| WEAR : has +``` + diff --git a/out/report/charts/c1_disk_footprint.png b/out/report/charts/c1_disk_footprint.png new file mode 100644 index 0000000..4f19570 Binary files /dev/null and b/out/report/charts/c1_disk_footprint.png differ diff --git a/out/report/charts/c2_ram_requirement.png b/out/report/charts/c2_ram_requirement.png new file mode 100644 index 0000000..da55867 Binary files /dev/null and b/out/report/charts/c2_ram_requirement.png differ diff --git a/out/report/charts/c3_q1_wall_time.png b/out/report/charts/c3_q1_wall_time.png new file mode 100644 index 0000000..15acae3 Binary files /dev/null and b/out/report/charts/c3_q1_wall_time.png differ diff --git a/out/report/charts/c3_q2_wall_time.png b/out/report/charts/c3_q2_wall_time.png new file mode 100644 index 0000000..63ba56e Binary files /dev/null and b/out/report/charts/c3_q2_wall_time.png differ diff --git a/out/report/charts/c3_q3_wall_time.png b/out/report/charts/c3_q3_wall_time.png new file mode 100644 index 0000000..9dafd74 Binary files /dev/null and b/out/report/charts/c3_q3_wall_time.png differ diff --git a/out/report/charts/c3_q4_wall_time.png b/out/report/charts/c3_q4_wall_time.png new file mode 100644 index 0000000..af8d080 Binary files /dev/null and b/out/report/charts/c3_q4_wall_time.png differ diff --git a/out/report/charts/c3_q5_wall_time.png b/out/report/charts/c3_q5_wall_time.png new file mode 100644 index 0000000..94c9b64 Binary files /dev/null and b/out/report/charts/c3_q5_wall_time.png differ diff --git a/out/report/charts/c3_q6_wall_time.png b/out/report/charts/c3_q6_wall_time.png new file mode 100644 index 0000000..b99b0db Binary files /dev/null and b/out/report/charts/c3_q6_wall_time.png differ diff --git a/out/report/charts/c3_q7_wall_time.png b/out/report/charts/c3_q7_wall_time.png new file mode 100644 index 0000000..6b3aedb Binary files /dev/null and b/out/report/charts/c3_q7_wall_time.png differ diff --git a/out/report/charts/c4_degradation_full_scan.png b/out/report/charts/c4_degradation_full_scan.png new file mode 100644 index 0000000..1bd23c3 Binary files /dev/null and b/out/report/charts/c4_degradation_full_scan.png differ diff --git a/out/report/charts/c4_degradation_indexed.png b/out/report/charts/c4_degradation_indexed.png new file mode 100644 index 0000000..c35db67 Binary files /dev/null and b/out/report/charts/c4_degradation_indexed.png differ diff --git a/out/report/charts/c4_degradation_point_read.png b/out/report/charts/c4_degradation_point_read.png new file mode 100644 index 0000000..be6ef1f Binary files /dev/null and b/out/report/charts/c4_degradation_point_read.png differ diff --git a/out/report/charts/c5_weighted_score.png b/out/report/charts/c5_weighted_score.png new file mode 100644 index 0000000..0f22901 Binary files /dev/null and b/out/report/charts/c5_weighted_score.png differ diff --git a/out/report/charts/c6_cost_vs_performance.png b/out/report/charts/c6_cost_vs_performance.png new file mode 100644 index 0000000..38ce542 Binary files /dev/null and b/out/report/charts/c6_cost_vs_performance.png differ diff --git a/out/report/diagrams/D1.mermaid b/out/report/diagrams/D1.mermaid new file mode 100644 index 0000000..f441683 --- /dev/null +++ b/out/report/diagrams/D1.mermaid @@ -0,0 +1,9 @@ +flowchart LR + A([Ti-6Al-4V Base 10x10x3 mm]) --> B[Cleaning] + B --> B1([Rinsed in deionized water]) + B1 --> B2([Sonicated in cleaning solution]) + B --> C[Smearing Adhesive] + C --> C1([Cr adhesive coating]) + C --> D[Vapor Deposition System
Kurt J. Lesker PVD 200] + D --> D1([PtAu sputter coating, gradient]) + D --> E([Test Coupon]) diff --git a/out/report/diagrams/D2.mermaid b/out/report/diagrams/D2.mermaid new file mode 100644 index 0000000..5270634 --- /dev/null +++ b/out/report/diagrams/D2.mermaid @@ -0,0 +1,9 @@ +flowchart LR + TC([Test Coupon]) --> S[SIMTRA simulation
composition / atom energies] + S --> OP[Optical profilometry
film thickness 0.3-1.1 um] + OP --> TW[Assemble Test Wafer
x49 coupons] + TW --> TB[Assemble Test Batch
x3 wafers] + TB --> B721[B721
Pt 150 W / Au 50 W] + TB --> B722[B722
Pt 100 W / Au 100 W] + TB --> B723[B723
Pt 150 W / Au 50 W] + TB --> B724[B724
Pt 50 W / Au 150 W] diff --git a/out/report/diagrams/D3.mermaid b/out/report/diagrams/D3.mermaid new file mode 100644 index 0000000..fc57805 --- /dev/null +++ b/out/report/diagrams/D3.mermaid @@ -0,0 +1,10 @@ +flowchart TD + TC([Test Coupon]) --> F[Friction
RAPID 6-probe tribometer] + F --> FA([Lab Air -> COF dataset mu_normal]) + F --> FN([Dry N2 -> COF dataset mu_dry_nit]) + TC --> NI[Nanoindentation
Bruker TI980] + NI --> NIR([Hardness, reduced modulus
25 indents per coupon]) + TC --> AF[AFM] + AF --> AFR([Topography -> Ra / Rq]) + TC --> XR[micro-XRF
Bruker M4 Tornado] + XR --> XRR([Pt/Au wt% map, 20x20 grid]) diff --git a/out/report/diagrams/D4.mermaid b/out/report/diagrams/D4.mermaid new file mode 100644 index 0000000..5e6a4a5 --- /dev/null +++ b/out/report/diagrams/D4.mermaid @@ -0,0 +1,23 @@ +sequenceDiagram + actor Op as Operator + participant Prep as Sample Prep + participant Plan as Excel Test Plan + participant SW as Control Software + participant TR as RAPID Tribometer + + Op->>Prep: Request test + Op->>Prep: Get samples + Op->>Prep: Load samples onto plates + Op->>Prep: Load ball holders (3 counterfaces per holder) + Op->>Plan: Create test plan
(Sample Plate, Plate Location, Sample ID, Save Location,
Folder Name, X/Y/Z Offset, Load, Iterations) + Op->>TR: Transfer plates to tribometer + Op->>TR: Set up platter + Op->>SW: Activate software + Op->>SW: Load plan + SW->>TR: Activate tribometer + TR-->>SW: Run reciprocating tests (execution loop, see D5) + Op->>SW: Software stop + SW-->>Op: Save avg files + Op->>TR: Remove plates and platters + Op->>SW: Close software + Op->>TR: Pull equipment out diff --git a/out/report/diagrams/D5.mermaid b/out/report/diagrams/D5.mermaid new file mode 100644 index 0000000..ef69767 --- /dev/null +++ b/out/report/diagrams/D5.mermaid @@ -0,0 +1,8 @@ +flowchart TD + L[Load all counterfaces] --> P{5 plates} + P -->|for each plate| PR{6 probes in parallel} + PR -->|for each probe| C{4 coupons on coupon square} + C -->|for each coupon| T[Draw track] + T --> RC[Rotate counterface - fresh ball per track] + RC -->|3 tracks per coupon| T + T --> DONE([120 coupons x 3 tracks = 360 tracks per run]) diff --git a/out/report/diagrams/D6.mermaid b/out/report/diagrams/D6.mermaid new file mode 100644 index 0000000..59cd246 --- /dev/null +++ b/out/report/diagrams/D6.mermaid @@ -0,0 +1,13 @@ +erDiagram + DEPOSITION_RUN ||--|| SIMTRA_PROFILE : produces + DEPOSITION_RUN ||--|| BATCH : creates + BATCH ||--|{ WAFER : contains + WAFER ||--|{ COUPON : contains + COUPON ||--|| XRF_MAP : has + COUPON ||--|| NANOINDENTATION : has + COUPON ||--|| AFM : has + COUPON ||--|| PROFILOMETRY : has + COUPON ||--o{ TRACK : "friction coupons only" + TRACK ||--|{ CYCLE : has + TRACK ||--|{ LOOP_POINT : has + TRACK ||--|| WEAR : has diff --git a/out/report/process_flow.md b/out/report/process_flow.md new file mode 100644 index 0000000..c882bad --- /dev/null +++ b/out/report/process_flow.md @@ -0,0 +1,127 @@ +# Process Flow - Simulated Tribology Laboratory + +Generated by make_process_flow.py (task 02) from out/config/lab_config.yaml. +Standalone diagram sources: ./diagrams/D1..D6.mermaid. + +## D1 - Coupon Assembly + +Substrate preparation, adhesion layer, and gradient Pt-Au sputter deposition. + +```mermaid +flowchart LR + A([Ti-6Al-4V Base 10x10x3 mm]) --> B[Cleaning] + B --> B1([Rinsed in deionized water]) + B1 --> B2([Sonicated in cleaning solution]) + B --> C[Smearing Adhesive] + C --> C1([Cr adhesive coating]) + C --> D[Vapor Deposition System
Kurt J. Lesker PVD 200] + D --> D1([PtAu sputter coating, gradient]) + D --> E([Test Coupon]) +``` + +## D2 - Characterization and Batch Assembly + +Per-coupon simulation and thickness characterization, then assembly into wafers (x49 coupons) and batches (x3 wafers), deposited per the parameter table below. + +```mermaid +flowchart LR + TC([Test Coupon]) --> S[SIMTRA simulation
composition / atom energies] + S --> OP[Optical profilometry
film thickness 0.3-1.1 um] + OP --> TW[Assemble Test Wafer
x49 coupons] + TW --> TB[Assemble Test Batch
x3 wafers] + TB --> B721[B721
Pt 150 W / Au 50 W] + TB --> B722[B722
Pt 100 W / Au 100 W] + TB --> B723[B723
Pt 150 W / Au 50 W] + TB --> B724[B724
Pt 50 W / Au 150 W] +``` + +Deposition matrix: + +| Batch | Pt:Au gun tilt | Pt power W | Au power W | Pt discharge V | Au discharge V | +|---|---|---|---|---|---| +| B721 | 20:0 deg | 150 | 50 | 432 | 311 | +| B722 | 20:20 deg | 100 | 100 | 399 | 352 | +| B723 | 20:20 deg | 150 | 50 | 430 | 311 | +| B724 | 0:20 deg | 50 | 150 | 376 | 340 | + +## D3 - Testing Tree + +The four characterization/testing paths every coupon can take. + +```mermaid +flowchart TD + TC([Test Coupon]) --> F[Friction
RAPID 6-probe tribometer] + F --> FA([Lab Air -> COF dataset mu_normal]) + F --> FN([Dry N2 -> COF dataset mu_dry_nit]) + TC --> NI[Nanoindentation
Bruker TI980] + NI --> NIR([Hardness, reduced modulus
25 indents per coupon]) + TC --> AF[AFM] + AF --> AFR([Topography -> Ra / Rq]) + TC --> XR[micro-XRF
Bruker M4 Tornado] + XR --> XRR([Pt/Au wt% map, 20x20 grid]) +``` + +## D4 - Tribometer Session Sequence + +Operator workflow for one RAPID tribometer session, from test request to teardown. + +```mermaid +sequenceDiagram + actor Op as Operator + participant Prep as Sample Prep + participant Plan as Excel Test Plan + participant SW as Control Software + participant TR as RAPID Tribometer + + Op->>Prep: Request test + Op->>Prep: Get samples + Op->>Prep: Load samples onto plates + Op->>Prep: Load ball holders (3 counterfaces per holder) + Op->>Plan: Create test plan
(Sample Plate, Plate Location, Sample ID, Save Location,
Folder Name, X/Y/Z Offset, Load, Iterations) + Op->>TR: Transfer plates to tribometer + Op->>TR: Set up platter + Op->>SW: Activate software + Op->>SW: Load plan + SW->>TR: Activate tribometer + TR-->>SW: Run reciprocating tests (execution loop, see D5) + Op->>SW: Software stop + SW-->>Op: Save avg files + Op->>TR: Remove plates and platters + Op->>SW: Close software + Op->>TR: Pull equipment out +``` + +## D5 - Execution Loop + +Nested iteration executed by the tribometer within one run. + +```mermaid +flowchart TD + L[Load all counterfaces] --> P{5 plates} + P -->|for each plate| PR{6 probes in parallel} + PR -->|for each probe| C{4 coupons on coupon square} + C -->|for each coupon| T[Draw track] + T --> RC[Rotate counterface - fresh ball per track] + RC -->|3 tracks per coupon| T + T --> DONE([120 coupons x 3 tracks = 360 tracks per run]) +``` + +## D6 - Data Hierarchy + +Entity containment and per-entity measurement datasets (ERD-style). + +```mermaid +erDiagram + DEPOSITION_RUN ||--|| SIMTRA_PROFILE : produces + DEPOSITION_RUN ||--|| BATCH : creates + BATCH ||--|{ WAFER : contains + WAFER ||--|{ COUPON : contains + COUPON ||--|| XRF_MAP : has + COUPON ||--|| NANOINDENTATION : has + COUPON ||--|| AFM : has + COUPON ||--|| PROFILOMETRY : has + COUPON ||--o{ TRACK : "friction coupons only" + TRACK ||--|{ CYCLE : has + TRACK ||--|{ LOOP_POINT : has + TRACK ||--|| WEAR : has +``` diff --git a/out/report/tables/t1_storage_footprint.csv b/out/report/tables/t1_storage_footprint.csv new file mode 100644 index 0000000..ede7628 --- /dev/null +++ b/out/report/tables/t1_storage_footprint.csv @@ -0,0 +1,6 @@ +format,measured,vs CSV,archival (compressed),projected @600 GB,projected @1.2 TB,projected @6 TB +CSV,149 MB,1.00x,53 MB,600 GB,1.20 TB,6.00 TB +JSON-LD,311 MB,2.09x,57 MB,1.26 TB,2.51 TB,13 TB +SQLite,123 MB,0.83x,55 MB,498 GB,996 GB,4.98 TB +PostgreSQL,397 MB,2.67x,53 MB,1.60 TB,3.21 TB,16 TB +RDF triplestore,1.74 GB,11.70x,126 MB,7.02 TB,14 TB,70 TB diff --git a/out/report/tables/t2_measured_medians.csv b/out/report/tables/t2_measured_medians.csv new file mode 100644 index 0000000..6ecd316 --- /dev/null +++ b/out/report/tables/t2_measured_medians.csv @@ -0,0 +1,36 @@ +format,query,wall time,peak RSS,CPU util,read +CSV,q1,0.06 s,4 MB,0%,0.0 MB +CSV,q2,0.17 s,20 MB,16%,2.5 MB +CSV,q3,1.15 s,21 MB,24%,31.8 MB +CSV,q4,0.71 s,21 MB,22%,16.8 MB +CSV,q5,1.09 s,21 MB,23%,31.8 MB +CSV,q6,0.28 s,20 MB,17%,1.2 MB +CSV,q7,1.27 s,21 MB,23%,32.5 MB +JSON-LD,q1,0.11 s,22 MB,10%,1.4 MB +JSON-LD,q2,1.48 s,24 MB,24%,53.1 MB +JSON-LD,q3,9.94 s,24 MB,25%,297.6 MB +JSON-LD,q4,9.92 s,24 MB,25%,297.8 MB +JSON-LD,q5,9.96 s,24 MB,25%,297.6 MB +JSON-LD,q6,9.79 s,24 MB,24%,297.8 MB +JSON-LD,q7,10 s,24 MB,24%,297.9 MB +SQLite,q1,0.06 s,4 MB,0%,0.0 MB +SQLite,q2,0.06 s,4 MB,0%,0.0 MB +SQLite,q3,0.11 s,4 MB,0%,0.0 MB +SQLite,q4,0.06 s,4 MB,0%,0.0 MB +SQLite,q5,0.77 s,25 MB,23%,117.9 MB +SQLite,q6,0.06 s,4 MB,0%,0.0 MB +SQLite,q7,0.71 s,25 MB,23%,93.0 MB +PostgreSQL,q1,0.22 s,42 MB,18%,4.1 MB +PostgreSQL,q2,0.22 s,42 MB,20%,4.1 MB +PostgreSQL,q3,0.22 s,42 MB,18%,4.1 MB +PostgreSQL,q4,0.22 s,42 MB,18%,4.1 MB +PostgreSQL,q5,0.44 s,42 MB,10%,4.1 MB +PostgreSQL,q6,0.22 s,42 MB,19%,4.1 MB +PostgreSQL,q7,0.33 s,42 MB,12%,4.1 MB +RDF triplestore,q1,0.11 s,32 MB,17%,3.6 MB +RDF triplestore,q2,2.81 s,85 MB,24%,375.6 MB +RDF triplestore,q3,46 s,264 MB,25%,5445.1 MB +RDF triplestore,q4,20 s,168 MB,25%,2694.6 MB +RDF triplestore,q5,60 s,264 MB,25%,5390.5 MB +RDF triplestore,q6,0.17 s,46 MB,16%,15.9 MB +RDF triplestore,q7,41 s,264 MB,25%,5380.5 MB diff --git a/out/report/tables/t3_projected_600gb.csv b/out/report/tables/t3_projected_600gb.csv new file mode 100644 index 0000000..dbccdae --- /dev/null +++ b/out/report/tables/t3_projected_600gb.csv @@ -0,0 +1,36 @@ +format,query,projected wall time,flag +CSV,q1,0.06 s,OK +CSV,q2,7.1 min,OK +CSV,q3,73.5 min,IMPRACTICAL +CSV,q4,44.1 min,OK +CSV,q5,69.6 min,IMPRACTICAL +CSV,q6,14.5 min,OK +CSV,q7,81.6 min,IMPRACTICAL +JSON-LD,q1,0.11 s,OK +JSON-LD,q2,92.0 min,IMPRACTICAL +JSON-LD,q3,11.0 h,IMPRACTICAL +JSON-LD,q4,11.0 h,IMPRACTICAL +JSON-LD,q5,11.0 h,IMPRACTICAL +JSON-LD,q6,10.9 h,IMPRACTICAL +JSON-LD,q7,11.2 h,IMPRACTICAL +SQLite,q1,0.06 s,OK +SQLite,q2,1.27 s,OK +SQLite,q3,5.7 min,OK +SQLite,q4,0.06 s,OK +SQLite,q5,47.5 min,OK +SQLite,q6,5.18 s,OK +SQLite,q7,43.7 min,OK +PostgreSQL,q1,0.22 s,OK +PostgreSQL,q2,0.22 s,OK +PostgreSQL,q3,3.42 s,OK +PostgreSQL,q4,11 s,OK +PostgreSQL,q5,3.7 min,OK +PostgreSQL,q6,4.06 s,OK +PostgreSQL,q7,115 s,OK +RDF triplestore,q1,0.11 s,OK +RDF triplestore,q2,3.0 h,IMPRACTICAL +RDF triplestore,q3,2.1 d,FAIL +RDF triplestore,q4,22.1 h,IMPRACTICAL +RDF triplestore,q5,2.8 d,FAIL +RDF triplestore,q6,5.9 min,OK +RDF triplestore,q7,45.3 h,FAIL diff --git a/out/report/tables/t4_hardware_600gb.csv b/out/report/tables/t4_hardware_600gb.csv new file mode 100644 index 0000000..11acd22 --- /dev/null +++ b/out/report/tables/t4_hardware_600gb.csv @@ -0,0 +1,6 @@ +format,configuration,chassis,RAM,disk,extra nodes,total +CSV,1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2,"$4,618","$1,887","$3,995",$0,"$10,500" +JSON-LD,1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2,"$4,618","$1,887","$3,995",$0,"$10,500" +SQLite,1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2,"$4,618","$1,887","$3,995",$0,"$10,500" +PostgreSQL,1 x 1U Supermicro AS-1015CS-TNR (16 cores); 1 x 64 GB DDR5 ECC RDIMM; 1 x 7.68 TB NVMe U.2,"$4,618","$1,887","$3,995",$0,"$10,500" +RDF triplestore,2 x 1U Supermicro AS-1015CS-TNR (16 cores); 22 x 64 GB DDR5 ECC RDIMM; 2 x 7.68 TB NVMe U.2,"$4,618","$41,514","$7,990","$4,618","$58,740" diff --git a/out/report/tables/t5_weighted_scores.csv b/out/report/tables/t5_weighted_scores.csv new file mode 100644 index 0000000..bfd1c3c --- /dev/null +++ b/out/report/tables/t5_weighted_scores.csv @@ -0,0 +1,11 @@ +scale,format,search_pts,ram_pts,disk_pts,total +measured,CSV,15.5,20.0,8.3,43.8 +measured,JSON-LD,1.7,17.2,4.0,22.9 +measured,SQLite,50.0,16.9,10.0,76.9 +measured,PostgreSQL,26.2,10.0,3.1,39.3 +measured,RDF triplestore,1.3,1.6,0.7,3.6 +projected_600gb,CSV,0.6,20.0,8.3,28.9 +projected_600gb,JSON-LD,0.1,20.0,4.0,24.0 +projected_600gb,SQLite,21.8,15.2,10.0,47.0 +projected_600gb,PostgreSQL,50.0,5.5,3.1,58.6 +projected_600gb,RDF triplestore,0.0,0.2,0.7,0.9