{ "boundary_caveat": "This is a bounded fixed-concurrency study on one native Arm64 runner. It is not an open-loop production-capacity benchmark, a cost or energy claim, or a replacement for the frozen canonical v1.1 recommendation. The KleidiAI log marker establishes only that llama.cpp reported a CPU_KLEIDIAI model buffer; it is not kernel-level acceleration proof.", "canonical_outputs_modified": false, "cells": [ { "candidate_id": "q8-generic", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1884.9235650000082, "generated_tokens_per_second": 35.134759195622934, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5489806124316083, "server_peak_rss_mib": 3437.59375, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p1", "source_load_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "ttft_ms_p95": 223.21656899998743 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1881.7339209999773, "generated_tokens_per_second": 35.06622390485378, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5479097485133403, "server_peak_rss_mib": 3437.59765625, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p1", "source_load_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "ttft_ms_p95": 226.40649200002372 } ], "server_parallel": 1, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 1883.3287429999928, "e2e_relative_spread_percent": 0.16936204111397554, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 35.10049155023836, "quality_gate_met": true, "requests_per_second_median": 0.5484451804724744, "server_peak_rss_mib_max": 3437.59765625, "throughput_relative_spread_percent": 0.1952545042597553, "ttft_ms_p95_median": 224.81153050000557, "ttft_relative_spread_percent": 1.4189321130199808 } }, { "candidate_id": "q8-generic", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3668.967000000066, "generated_tokens_per_second": 35.47060058516804, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5542281341432507, "server_peak_rss_mib": 3437.59375, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q8-generic/p1", "source_load_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "ttft_ms_p95": 2056.6426259999844 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3687.7824379998856, "generated_tokens_per_second": 35.04666039190969, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5476040686235889, "server_peak_rss_mib": 3437.59765625, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q8-generic/p1", "source_load_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "ttft_ms_p95": 2040.5987489998552 } ], "server_parallel": 1, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3678.374718999976, "e2e_relative_spread_percent": 0.5115149879274623, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 35.25863048853887, "quality_gate_met": true, "requests_per_second_median": 0.5509161013834198, "server_peak_rss_mib_max": 3437.59765625, "throughput_relative_spread_percent": 1.2023728300966166, "ttft_ms_p95_median": 2048.6206874999198, "ttft_relative_spread_percent": 0.7831550807830959 } }, { "candidate_id": "q8-generic", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 7254.258521999986, "generated_tokens_per_second": 35.52994853934689, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5551554459272952, "server_peak_rss_mib": 3437.59375, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q8-generic/p1", "source_load_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "ttft_ms_p95": 5617.4895230000175 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 7441.183771000169, "generated_tokens_per_second": 34.75048650506307, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5429763516416105, "server_peak_rss_mib": 3437.59765625, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q8-generic/p1", "source_load_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "ttft_ms_p95": 5752.069523000046 } ], "server_parallel": 1, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 7347.721146500078, "e2e_relative_spread_percent": 2.5439894257448836, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "forward:e2e_latency_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum", "reverse:e2e_latency_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 35.14021752220498, "quality_gate_met": true, "requests_per_second_median": 0.5490658987844528, "server_peak_rss_mib_max": 3437.59765625, "throughput_relative_spread_percent": 2.2181480060312033, "ttft_ms_p95_median": 5684.779523000032, "ttft_relative_spread_percent": 2.3673741339577252 } }, { "candidate_id": "q8-generic", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1854.2111919999797, "generated_tokens_per_second": 35.54798524427483, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5554372694417942, "server_peak_rss_mib": 3437.55078125, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p2", "source_load_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "ttft_ms_p95": 222.6910530000623 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1873.1312170000365, "generated_tokens_per_second": 35.16462890593168, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5494473266551825, "server_peak_rss_mib": 3437.55078125, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p2", "source_load_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "ttft_ms_p95": 231.5461990001495 } ], "server_parallel": 2, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 1863.6712045000081, "e2e_relative_spread_percent": 1.0152018743634914, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 35.35630707510325, "quality_gate_met": true, "requests_per_second_median": 0.5524422980484883, "server_peak_rss_mib_max": 3437.55078125, "throughput_relative_spread_percent": 1.0842657790272874, "ttft_ms_p95_median": 227.1186260001059, "ttft_relative_spread_percent": 3.8989078773676007 } }, { "candidate_id": "q8-generic", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3329.0332560000024, "generated_tokens_per_second": 38.8899080673751, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.607654813552736, "server_peak_rss_mib": 3437.55078125, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p2", "source_load_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "ttft_ms_p95": 391.7180969999663 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3365.6050560000494, "generated_tokens_per_second": 38.81548786800706, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.6064919979376103, "server_peak_rss_mib": 3437.55078125, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p2", "source_load_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "ttft_ms_p95": 387.9238280001118 } ], "server_parallel": 2, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3347.319156000026, "e2e_relative_spread_percent": 1.0925698535346573, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 38.852697967691086, "quality_gate_met": true, "requests_per_second_median": 0.6070734057451732, "server_peak_rss_mib_max": 3437.55078125, "throughput_relative_spread_percent": 0.1915444827793631, "ttft_ms_p95_median": 389.82096250003906, "ttft_relative_spread_percent": 0.9733363171443317 } }, { "candidate_id": "q8-generic", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 6684.801609000033, "generated_tokens_per_second": 38.665443722985536, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.604147558171649, "server_peak_rss_mib": 3437.55078125, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q8-generic/p2", "source_load_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "ttft_ms_p95": 3696.6847560000815 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 6654.875219999894, "generated_tokens_per_second": 38.628533640139615, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.6035708381271815, "server_peak_rss_mib": 3437.55078125, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q8-generic/p2", "source_load_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "ttft_ms_p95": 3704.2261950000466 } ], "server_parallel": 2, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 6669.838414499964, "e2e_relative_spread_percent": 0.44868236890236596, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "forward:e2e_latency_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum", "reverse:e2e_latency_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 38.646988681562576, "quality_gate_met": true, "requests_per_second_median": 0.6038591981494152, "server_peak_rss_mib_max": 3437.55078125, "throughput_relative_spread_percent": 0.09550571494728076, "ttft_ms_p95_median": 3700.455475500064, "ttft_relative_spread_percent": 0.20379758788871655 } }, { "candidate_id": "q8-generic", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1858.8786120000123, "generated_tokens_per_second": 35.3130448362052, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5517663255657063, "server_peak_rss_mib": 3446.19140625, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p4", "source_load_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "ttft_ms_p95": 220.54406899997048 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1854.306553000015, "generated_tokens_per_second": 35.52597658797478, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.555093384187106, "server_peak_rss_mib": 3448.0703125, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p4", "source_load_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "ttft_ms_p95": 221.72692699996333 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 1856.5925825000136, "e2e_relative_spread_percent": 0.24626075979689632, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 35.41951071208999, "quality_gate_met": true, "requests_per_second_median": 0.5534298548764061, "server_peak_rss_mib_max": 3448.0703125, "throughput_relative_spread_percent": 0.6011707883274049, "ttft_ms_p95_median": 221.1354979999669, "ttft_relative_spread_percent": 0.534901908870834 } }, { "candidate_id": "q8-generic", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3534.7170180000376, "generated_tokens_per_second": 37.69445819948809, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5889759093670014, "server_peak_rss_mib": 3446.19140625, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p4", "source_load_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "ttft_ms_p95": 391.03381600000375 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3655.1638060000187, "generated_tokens_per_second": 36.49275434021204, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5701992865658131, "server_peak_rss_mib": 3448.0703125, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p4", "source_load_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "ttft_ms_p95": 440.80272599990167 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3594.940412000028, "e2e_relative_spread_percent": 3.3504529754631203, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 37.09360626985006, "quality_gate_met": true, "requests_per_second_median": 0.5795875979664072, "server_peak_rss_mib_max": 3448.0703125, "throughput_relative_spread_percent": 3.2396522746638534, "ttft_ms_p95_median": 415.9182709999527, "ttft_relative_spread_percent": 11.96603118209817 } }, { "candidate_id": "q8-generic", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3090.0780179999856, "generated_tokens_per_second": 84.384245678028, "pass_id": "forward", "request_count": 8, "requests_per_second": 1.3185038387191874, "server_peak_rss_mib": 3446.19140625, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q8-generic/p4", "source_load_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "ttft_ms_p95": 776.5359169999329 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3023.502796999992, "generated_tokens_per_second": 85.49299534933145, "pass_id": "reverse", "request_count": 8, "requests_per_second": 1.335828052333304, "server_peak_rss_mib": 3448.0703125, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q8-generic/p4", "source_load_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "ttft_ms_p95": 736.6408529999262 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3056.7904074999888, "e2e_relative_spread_percent": 2.177945234211938, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 84.93862051367972, "quality_gate_met": true, "requests_per_second_median": 1.3271659455262457, "server_peak_rss_mib_max": 3448.0703125, "throughput_relative_spread_percent": 1.3053539892667412, "ttft_ms_p95_median": 756.5883849999295, "ttft_relative_spread_percent": 5.273020943879592 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 2216.8644500000028, "generated_tokens_per_second": 32.45226642100907, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5070666628282667, "server_peak_rss_mib": 1966.4765625, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p1", "source_load_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "ttft_ms_p95": 168.96748500005287 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1964.0805829999408, "generated_tokens_per_second": 33.183635754278356, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5184943086605993, "server_peak_rss_mib": 1966.47265625, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p1", "source_load_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "ttft_ms_p95": 171.24502100000427 } ], "server_parallel": 1, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 2090.472516499972, "e2e_relative_spread_percent": 12.09218801035911, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 32.81795108764371, "quality_gate_met": true, "requests_per_second_median": 0.512780485744433, "server_peak_rss_mib_max": 1966.4765625, "throughput_relative_spread_percent": 2.22856488303029, "ttft_ms_p95_median": 170.10625300002857, "ttft_relative_spread_percent": 1.3388902287742557 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3896.099208999999, "generated_tokens_per_second": 33.26389569820159, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5197483702843998, "server_peak_rss_mib": 1966.4765625, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q4-kleidiai-tuned/p1", "source_load_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "ttft_ms_p95": 2107.5858809999772 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3881.8361439998625, "generated_tokens_per_second": 33.223827608350966, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5191223063804838, "server_peak_rss_mib": 1966.47265625, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q4-kleidiai-tuned/p1", "source_load_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "ttft_ms_p95": 2094.8557839999467 } ], "server_parallel": 1, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3888.967676499931, "e2e_relative_spread_percent": 0.3667570981966421, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 33.24386165327628, "quality_gate_met": true, "requests_per_second_median": 0.5194353383324418, "server_peak_rss_mib_max": 1966.4765625, "throughput_relative_spread_percent": 0.12052778425238127, "ttft_ms_p95_median": 2101.220832499962, "ttft_relative_spread_percent": 0.6058428891971693 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 7758.620410000049, "generated_tokens_per_second": 33.081674744586984, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5169011678841716, "server_peak_rss_mib": 1966.4765625, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q4-kleidiai-tuned/p1", "source_load_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "ttft_ms_p95": 5969.818140000029 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 7849.5314490000965, "generated_tokens_per_second": 32.862249628087135, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5134726504388615, "server_peak_rss_mib": 1966.47265625, "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q4-kleidiai-tuned/p1", "source_load_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "ttft_ms_p95": 6049.835923000046 } ], "server_parallel": 1, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 7804.075929500073, "e2e_relative_spread_percent": 1.1649174075356743, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "forward:e2e_latency_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum", "reverse:e2e_latency_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 32.97196218633706, "quality_gate_met": true, "requests_per_second_median": 0.5151869091615165, "server_peak_rss_mib_max": 1966.4765625, "throughput_relative_spread_percent": 0.6654900162137594, "ttft_ms_p95_median": 6009.827031500037, "ttft_relative_spread_percent": 1.3314490180933607 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1950.3699480000023, "generated_tokens_per_second": 33.31412800964699, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5205332501507343, "server_peak_rss_mib": 1966.4296875, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p2", "source_load_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "ttft_ms_p95": 168.94154700003128 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1978.485245999991, "generated_tokens_per_second": 32.900713571849174, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5140736495601433, "server_peak_rss_mib": 1966.43359375, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p2", "source_load_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "ttft_ms_p95": 172.64410099983252 } ], "server_parallel": 2, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 1964.4275969999967, "e2e_relative_spread_percent": 1.4312208830157687, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 33.10742079074808, "quality_gate_met": true, "requests_per_second_median": 0.5173034498554387, "server_peak_rss_mib_max": 1966.43359375, "throughput_relative_spread_percent": 1.2487062656156778, "ttft_ms_p95_median": 170.7928239999319, "ttft_relative_spread_percent": 2.1678627433449518 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 2951.646697000001, "generated_tokens_per_second": 43.83258865350848, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.68488419771107, "server_peak_rss_mib": 1966.4296875, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p2", "source_load_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "ttft_ms_p95": 314.5024420000482 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 2982.4296659999163, "generated_tokens_per_second": 43.62433363520482, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.6816302130500753, "server_peak_rss_mib": 1966.43359375, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p2", "source_load_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "ttft_ms_p95": 314.7659329999897 } ], "server_parallel": 2, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 2967.0381814999587, "e2e_relative_spread_percent": 1.0374982429229556, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 43.72846114435665, "quality_gate_met": true, "requests_per_second_median": 0.6832572053805727, "server_peak_rss_mib_max": 1966.43359375, "throughput_relative_spread_percent": 0.4762459342352971, "ttft_ms_p95_median": 314.6341875000189, "ttft_relative_spread_percent": 0.08374519057674876 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 6017.29259199999, "generated_tokens_per_second": 43.51438989640389, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.6799123421313108, "server_peak_rss_mib": 1966.4296875, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "forward/q4-kleidiai-tuned/p2", "source_load_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "ttft_ms_p95": 3231.956867000008 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 6030.341808999992, "generated_tokens_per_second": 43.55757587147319, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.6805871229917686, "server_peak_rss_mib": 1966.43359375, "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "source_load_label": "reverse/q4-kleidiai-tuned/p2", "source_load_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "ttft_ms_p95": 3231.0797979998824 } ], "server_parallel": 2, "summary": { "capacity_gate_met": false, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 6023.817200499991, "e2e_relative_spread_percent": 0.21662704171898484, "every_pass_slo_met": false, "failure_reasons": [ "one_or_more_passes_failed_load_slo", "forward:ttft_ms_p95_above_maximum", "reverse:ttft_ms_p95_above_maximum" ], "generated_tokens_per_second_median": 43.535982883938544, "quality_gate_met": true, "requests_per_second_median": 0.6802497325615398, "server_peak_rss_mib_max": 1966.43359375, "throughput_relative_spread_percent": 0.09919604935629389, "ttft_ms_p95_median": 3231.5183324999452, "ttft_relative_spread_percent": 0.02714108075157097 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 1, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1960.1255799999535, "generated_tokens_per_second": 33.32025517439394, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.5206289870999053, "server_peak_rss_mib": 2031.296875, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p4", "source_load_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "ttft_ms_p95": 168.4933159999673 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 1987.104448999844, "generated_tokens_per_second": 32.981993359812954, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5153436462470774, "server_peak_rss_mib": 2030.69140625, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p4", "source_load_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "ttft_ms_p95": 171.85684399987622 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 1973.6150144998987, "e2e_relative_spread_percent": 1.3669772879553568, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 33.15112426710345, "quality_gate_met": true, "requests_per_second_median": 0.5179863166734914, "server_peak_rss_mib_max": 2031.296875, "throughput_relative_spread_percent": 1.0203630255660825, "ttft_ms_p95_median": 170.17507999992176, "ttft_relative_spread_percent": 1.9765103092124048 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 2, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3827.4439890001304, "generated_tokens_per_second": 40.47207855761459, "pass_id": "forward", "request_count": 8, "requests_per_second": 0.6323762274627279, "server_peak_rss_mib": 2031.296875, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p4", "source_load_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "ttft_ms_p95": 339.10964200003946 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 3933.9661210001395, "generated_tokens_per_second": 34.9013418466546, "pass_id": "reverse", "request_count": 8, "requests_per_second": 0.5453334663539782, "server_peak_rss_mib": 2030.69140625, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p4", "source_load_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "ttft_ms_p95": 310.71384200004104 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 3880.705055000135, "e2e_relative_spread_percent": 2.744916980041026, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 37.68671020213459, "quality_gate_met": true, "requests_per_second_median": 0.588854846908353, "server_peak_rss_mib_max": 2031.296875, "throughput_relative_spread_percent": 14.781700713808805, "ttft_ms_p95_median": 324.91174200004025, "ttft_relative_spread_percent": 8.739542567838283 } }, { "candidate_id": "q4-kleidiai-tuned", "client_concurrency": 4, "pass_metrics": [ { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 2841.4814269999624, "generated_tokens_per_second": 90.97481662809376, "pass_id": "forward", "request_count": 8, "requests_per_second": 1.421481509813965, "server_peak_rss_mib": 2031.296875, "slo_failures": [], "slo_met": true, "source_load_label": "forward/q4-kleidiai-tuned/p4", "source_load_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "ttft_ms_p95": 664.8192449999897 }, { "completed_requests": 8, "completion_rate": 1.0, "e2e_latency_ms_p95": 2863.816679000138, "generated_tokens_per_second": 90.3600914874451, "pass_id": "reverse", "request_count": 8, "requests_per_second": 1.4118764294913297, "server_peak_rss_mib": 2030.69140625, "slo_failures": [], "slo_met": true, "source_load_label": "reverse/q4-kleidiai-tuned/p4", "source_load_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "ttft_ms_p95": 615.0487229999726 } ], "server_parallel": 4, "summary": { "capacity_gate_met": true, "completion_rate_min": 1.0, "e2e_latency_ms_p95_median": 2852.64905300005, "e2e_relative_spread_percent": 0.7829652924423366, "every_pass_slo_met": true, "failure_reasons": [], "generated_tokens_per_second_median": 90.66745405776943, "quality_gate_met": true, "requests_per_second_median": 1.4166789696526474, "server_peak_rss_mib_max": 2031.296875, "throughput_relative_spread_percent": 0.6779997817706256, "ttft_ms_p95_median": 639.9339839999811, "ttft_relative_spread_percent": 7.777446306089371 } } ], "classification": "supplementary-capacity", "input_fingerprints": { "capacity_plan_content_sha256": "1600cb6ceef228c6e124ce09c2c58058b479dd46f8fcccde783c9072702958e9", "capacity_plan_sha256": "85cf5ecad8d3e9c8cc83f2b7fe554b913013360d6a81322305edee32e06b98db", "load_artifacts": { "forward/q4-kleidiai-tuned/p1": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "forward/q4-kleidiai-tuned/p2": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "forward/q4-kleidiai-tuned/p4": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "forward/q8-generic/p1": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "forward/q8-generic/p2": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "forward/q8-generic/p4": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "reverse/q4-kleidiai-tuned/p1": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "reverse/q4-kleidiai-tuned/p2": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "reverse/q4-kleidiai-tuned/p4": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "reverse/q8-generic/p1": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "reverse/q8-generic/p2": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "reverse/q8-generic/p4": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf" }, "load_plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "manifest_content_sha256": "76e34e4a7abb599a298a7ece79c2a032872bd8c9db08a1ce4055a6fdea727da9", "manifest_sha256": "76e34e4a7abb599a298a7ece79c2a032872bd8c9db08a1ce4055a6fdea727da9", "quality_artifacts": { "q4-kleidiai-tuned/p1": "11f4d029b55759c6d92a10175c5ecf3b87f24ca54a2d299a4e5555b827712c45", "q4-kleidiai-tuned/p2": "6422ad9f4b512ea99fa8a5e734953f6d828fe1c09db33bd7602130d5973c74a8", "q4-kleidiai-tuned/p4": "01215f03699eca7b365e16d1d7c8c8afab3d43dda76cc445cb34b4b0fa533e62", "q8-generic/p1": "911751fe1af5022efd27520b2b8fb6f599156784d32e1f40765e6adefe8de23c", "q8-generic/p2": "372e61aa64b12a72ff7a4d8a9c9da57a61fdd1a673c8dd8cea54e5e5387e5596", "q8-generic/p4": "e12f521247d9fcac0b5c64e8b947fbc271e80b0740c9ed233a54f474502c3aee" }, "rss_artifacts": { "forward/q4-kleidiai-tuned/p1": "b846b5f62edeb02ef5fd632d21975714b20981e1780d5178be89767b0576717f", "forward/q4-kleidiai-tuned/p2": "0e9341ba713b06dc347cbcf72e5f96c872a45afe9dbe2525b1e7f6c427c04389", "forward/q4-kleidiai-tuned/p4": "73c118c10defa9305962be19ca6e5ec781902a211fe8dea86c8d9ca4cea45f93", "forward/q8-generic/p1": "3e0ddd7a8867fde99f37705070329ed6949d6a9ee88f38b8dbace9376a27af7a", "forward/q8-generic/p2": "bf0d05c1ca07c18a24eae784ea304f2ed27ed047dd7265e38b0abccf2c07fadb", "forward/q8-generic/p4": "24a54dcabe2e2af8e0fd5611d247ba463848e9deb6367b27d1e34d4e8b1d439a", "reverse/q4-kleidiai-tuned/p1": "439242e383330e146b0277c9b16782e3406390476a573800074120dd42ecd3dc", "reverse/q4-kleidiai-tuned/p2": "71a5bea46810674630de19cf8b6a6a5aeea5e12a9ecd3fbacbc5e1a351e2e791", "reverse/q4-kleidiai-tuned/p4": "7df3f46af65ed7baec97a1ddd131fb0eebc29ec28f92aa9cf9b4b2bf11004709", "reverse/q8-generic/p1": "df4b229b90bece7f25b2281cfebd79f78cfd4ba7285b6af436ab79ab2f514859", "reverse/q8-generic/p2": "a419a2ee0d170f91130af6d14f4e5b9ad1fbe0fb36cee8b99831dc3c2c2ffc7c", "reverse/q8-generic/p4": "475b4622d3d3bc939c9101637a4ae8b08ebdd56553c8855d0aac4867b9d2b6f9" }, "server_logs": { "forward/q4-kleidiai-tuned/p1": "2cea71d76969107d17d141c453e51974e772de698c9d158d863ed578e61ad213", "forward/q4-kleidiai-tuned/p2": "70d029c04d5f9d0d4d0fa6367814948c64278828120f995c25d97a465b20a71c", "forward/q4-kleidiai-tuned/p4": "fd005b7df820669acc3f5b6e2962d380d50a65b19ebe114f4410450639001d35", "forward/q8-generic/p1": "78a509c69d633406dd49dc08d8aa0700e4b04793312af7e6093cd4885998f927", "forward/q8-generic/p2": "cd6c830640c5e0844da6fbf513916ebb92193457a69fb6edc343736e9c1dba4f", "forward/q8-generic/p4": "cad02aa75e5c1bf65be57f4f8d805c997a8bcca9aacb42d4b571616523569c2b", "reverse/q4-kleidiai-tuned/p1": "d12ab4b8bd2c45b360d2415987ef470f059df57841582d9370e208a51f03dfcd", "reverse/q4-kleidiai-tuned/p2": "bdce5f8d10724d84b761feeee4f7349385857cce42abb8698d332e9d97b91043", "reverse/q4-kleidiai-tuned/p4": "ecb40574776faf6daa5e00d35de3a6749a9676ade4db2b94bfb10c19831e2c89", "reverse/q8-generic/p1": "93062c048b15393500400fcd0136b0eca771f2595d5c9c9f0e30bbaefb14ea40", "reverse/q8-generic/p2": "0944c80b188a69060f916d3c186e70a0e7bd341223603eeb95a436721a8e642d", "reverse/q8-generic/p4": "e28375be04fcbd7f25f8cd604f0ef8a8c699ca567bb7ea4c8680e60568ca517e" } }, "load_contract": { "load_plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 } }, "plan": { "candidates": [ { "canonical_server_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_server_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "id": "q8-generic", "kleidiai_expected": false, "label": "Q8 generic reference", "role": "canonical-reference" }, { "canonical_server_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_server_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "id": "q4-kleidiai-tuned", "kleidiai_expected": true, "label": "Q4 with KleidiAI and 512-token micro-batch", "role": "resource-alternative" } ], "capacity_gate": { "max_e2e_relative_spread_percent": 20.0, "max_server_peak_rss_mib": 4096.0, "max_throughput_relative_spread_percent": 15.0, "require_every_pass_slo": true }, "classification": "supplementary-capacity", "client_concurrency_levels": [ 1, 2, 4 ], "load_slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "passes": [ { "candidate_order": [ "q8-generic", "q4-kleidiai-tuned" ], "client_concurrency_order": [ 1, 2, 4 ], "id": "forward", "server_parallel_order": [ 1, 2, 4 ] }, { "candidate_order": [ "q4-kleidiai-tuned", "q8-generic" ], "client_concurrency_order": [ 4, 2, 1 ], "id": "reverse", "server_parallel_order": [ 4, 2, 1 ] } ], "per_slot_context_tokens": 2048, "pins": { "canonical_evidence_lock_sha256": "9a00187cb4619daec3596139c97de49127841ceb3c2c7edd85092df2474c578d", "canonical_release_sha256": "b5586878ccd214667911390f417db0417111ac2c31d163a2f5f55c4469aefeb2", "evaluation_suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "load_plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb" }, "quality_gate": { "minimum_retention_vs_reference": 0.95, "minimum_score": 0.8, "reference_candidate_id": "q8-generic", "require_identical_outcomes_across_parallel": true }, "schema_version": "1.1", "scope": "A bounded fixed-concurrency study on one native Arm64 runner. It compares two predeclared deployment candidates and does not change the frozen canonical v1.1 recommendation.", "selection": { "direction": "max", "objective": "generated_tokens_per_second_median", "objective_tolerance_percent": 1.0, "tie_breakers": [ "e2e_latency_ms_p95_median", "ttft_ms_p95_median", "server_peak_rss_mib_max", "server_parallel", "client_concurrency" ] }, "server_parallel_levels": [ 1, 2, 4 ] }, "provenance": { "candidates": { "q4-kleidiai-tuned": { "build": { "cmake_cache_sha256": "25f0a4a8966883735a1bd615d2bf42cdb4ae820f300d34d92e4373b6798dd64e", "compile_log_sha256": "70d06a2e8c7849bcc450bb97a08e64393816a3468a4da64ad8e6e32e378e1e16", "configure_log_sha256": "67aaa8de410b855a33d374ac5cca6a423e0ece962b00245f4b54b9747c5b5b6c", "kleidiai_enabled": true, "label": "kleidiai", "server_binary_sha256": "2b55365cfbee30b17df661caa025a0e7dd38063dc34a4a0064e10d9d63032db2" }, "model": { "family": "Qwen2.5-1.5B-Instruct", "filename": "qwen2.5-1.5b-instruct-q4_0.gguf", "repository": "Qwen/Qwen2.5-1.5B-Instruct-GGUF", "revision": "91cad51170dc346986eccefdc2dd33a9da36ead9", "sha256": "dcd819ff094852c38faba6873d8ff0c9d51eadb2844539e52042ae5d647bbfdb", "size_bytes": 1066227232 } }, "q8-generic": { "build": { "cmake_cache_sha256": "c2bfba63616e30d73dd5d7f9300f4b6878a4c714f5a4ad8595a7194640d5921c", "compile_log_sha256": "f132389129405fbaec71320f0ad62ed3f270c6c46499decddd7623f37428668c", "configure_log_sha256": "2f43c256d89f4e41021e1fe12b72293e10e13e226658d86c4b5c9f31bfaab8be", "kleidiai_enabled": false, "label": "generic", "server_binary_sha256": "b58a8cbec83c3589032fed948d5f062ede6c89086f3a8ff86e0a614f37dcf4b3" }, "model": { "family": "Qwen2.5-1.5B-Instruct", "filename": "qwen2.5-1.5b-instruct-q8_0.gguf", "repository": "Qwen/Qwen2.5-1.5B-Instruct-GGUF", "revision": "91cad51170dc346986eccefdc2dd33a9da36ead9", "sha256": "d7efb072e7724d25048a4fda0a3e10b04bdef5d06b1403a1c93bd9f1240a63c8", "size_bytes": 1894532128 } } }, "canonical_commands": { "q4-kleidiai-tuned": { "argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487" }, "q8-generic": { "argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f" } }, "canonical_evidence": { "lock_sha256": "9a00187cb4619daec3596139c97de49127841ceb3c2c7edd85092df2474c578d", "release_sha256": "b5586878ccd214667911390f417db0417111ac2c31d163a2f5f55c4469aefeb2", "release_tag": "v1.1.0", "run_id": "30055662526" }, "classification": "supplementary-capacity", "optimization_library": { "name": "KleidiAI", "repository": "https://github.com/ARM-software/kleidiai", "size_bytes": 2466038, "source_archive_sha256": "9348b969e042d8890a54b01a463dbe71f5a4c074b5329e9c26a85ef3b68aa19b", "version": "v1.24.0" }, "runner": { "architecture": "arm64", "cpu": "Neoverse-N2", "cpu_count": 4, "os": "Ubuntu 24.04" }, "runtime": { "name": "llama.cpp", "repository": "https://github.com/ggml-org/llama.cpp", "revision": "67b9b0e7f6ce45d929a4411907d3c48ec719e81c" }, "schema_version": "1.1", "source": { "generated_at_utc": "2026-07-25T05:05:40Z", "repository": "agrovr/ParetoPilot", "revision": "db9ccaf37e3c7e807832652e237de813675ed807", "run_attempt": 1, "run_id": "30144901854", "workflow": ".github/workflows/capacity-study-arm64.yml" }, "synthetic": false, "toolchain": { "cmake_version_sha256": "f6e81c2559f3ce00c5fdad01d7b226c301f5d6b07ee04f65db6dbfdc1cc5970b", "gcc_version_sha256": "5698ba2bce15103316282b418bf772fd5ce99fbe60fd5ba2987d69fa39f61e58", "gxx_version_sha256": "b240ade48d7a4743cb2e7940990dd5a8dbc15120543c7eadd9541435edd2efd2", "ninja_version_sha256": "810ab97f4b3fccb9adf1db7959945b479635931ccc4a0243c34a41b739eae25a" } }, "quality_checks": [ { "candidate_id": "q8-generic", "evaluation_sha256": "f45adb5517dd28e1c4520eb3da44774e38ae7afd5532fc875479125700405003", "failure_reasons": [], "gate_met": true, "input_sha256": "911751fe1af5022efd27520b2b8fb6f599156784d32e1f40765e6adefe8de23c", "outcomes_match_candidate_reference": true, "outcomes_sha256": "ba911d125fad27f6025c2fcf341230c920edb2e1b9a28bcdabe7dc9ffd4da21d", "passed": 21, "retention_vs_reference": 1.0, "score": 0.875, "server_parallel": 1, "source_content_sha256": "911751fe1af5022efd27520b2b8fb6f599156784d32e1f40765e6adefe8de23c", "source_label": "q8-generic/p1", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 }, { "candidate_id": "q8-generic", "evaluation_sha256": "05c8aa9349890b232019b6be1780d8ea1efbff455d1c5c7a1ffb14d6b3a4be27", "failure_reasons": [], "gate_met": true, "input_sha256": "372e61aa64b12a72ff7a4d8a9c9da57a61fdd1a673c8dd8cea54e5e5387e5596", "outcomes_match_candidate_reference": true, "outcomes_sha256": "ba911d125fad27f6025c2fcf341230c920edb2e1b9a28bcdabe7dc9ffd4da21d", "passed": 21, "retention_vs_reference": 1.0, "score": 0.875, "server_parallel": 2, "source_content_sha256": "372e61aa64b12a72ff7a4d8a9c9da57a61fdd1a673c8dd8cea54e5e5387e5596", "source_label": "q8-generic/p2", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 }, { "candidate_id": "q8-generic", "evaluation_sha256": "ac6e974334e2222dc27fbd17a60fc8ed35bfed96c3c98ce41a8e1e5148ee1a3a", "failure_reasons": [], "gate_met": true, "input_sha256": "e12f521247d9fcac0b5c64e8b947fbc271e80b0740c9ed233a54f474502c3aee", "outcomes_match_candidate_reference": true, "outcomes_sha256": "ba911d125fad27f6025c2fcf341230c920edb2e1b9a28bcdabe7dc9ffd4da21d", "passed": 21, "retention_vs_reference": 1.0, "score": 0.875, "server_parallel": 4, "source_content_sha256": "e12f521247d9fcac0b5c64e8b947fbc271e80b0740c9ed233a54f474502c3aee", "source_label": "q8-generic/p4", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 }, { "candidate_id": "q4-kleidiai-tuned", "evaluation_sha256": "8d98444c54c8eed866e532dff6f68c9e68494800448831a62f66b29451999db0", "failure_reasons": [], "gate_met": true, "input_sha256": "11f4d029b55759c6d92a10175c5ecf3b87f24ca54a2d299a4e5555b827712c45", "outcomes_match_candidate_reference": true, "outcomes_sha256": "104f1d9c50c5a7e9036fcad26e0d4bcd786536b1c7132434618ae2727b6027b6", "passed": 20, "retention_vs_reference": 0.9523809523809524, "score": 0.8333333333333334, "server_parallel": 1, "source_content_sha256": "11f4d029b55759c6d92a10175c5ecf3b87f24ca54a2d299a4e5555b827712c45", "source_label": "q4-kleidiai-tuned/p1", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 }, { "candidate_id": "q4-kleidiai-tuned", "evaluation_sha256": "dd0a48ef8646b10ae4bdac03644c008b7aa4f7a8b4b05a518118c6097baa6e31", "failure_reasons": [], "gate_met": true, "input_sha256": "6422ad9f4b512ea99fa8a5e734953f6d828fe1c09db33bd7602130d5973c74a8", "outcomes_match_candidate_reference": true, "outcomes_sha256": "104f1d9c50c5a7e9036fcad26e0d4bcd786536b1c7132434618ae2727b6027b6", "passed": 20, "retention_vs_reference": 0.9523809523809524, "score": 0.8333333333333334, "server_parallel": 2, "source_content_sha256": "6422ad9f4b512ea99fa8a5e734953f6d828fe1c09db33bd7602130d5973c74a8", "source_label": "q4-kleidiai-tuned/p2", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 }, { "candidate_id": "q4-kleidiai-tuned", "evaluation_sha256": "043880d9e6f96691aec26c408cf56b0a0bbb6382700ebf33e1cc27ce924f5942", "failure_reasons": [], "gate_met": true, "input_sha256": "01215f03699eca7b365e16d1d7c8c8afab3d43dda76cc445cb34b4b0fa533e62", "outcomes_match_candidate_reference": true, "outcomes_sha256": "104f1d9c50c5a7e9036fcad26e0d4bcd786536b1c7132434618ae2727b6027b6", "passed": 20, "retention_vs_reference": 0.9523809523809524, "score": 0.8333333333333334, "server_parallel": 4, "source_content_sha256": "01215f03699eca7b365e16d1d7c8c8afab3d43dda76cc445cb34b4b0fa533e62", "source_label": "q4-kleidiai-tuned/p4", "suite_id": "paretopilot-qwen-behavior-v2", "suite_sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "total": 24 } ], "schema_version": "1.1", "selections": [ { "candidate_id": "q8-generic", "comparison_to_reference_percent": { "e2e_latency_ms_p95_median": 62.307850865737066, "generated_tokens_per_second_median": 141.98698298030794, "server_peak_rss_mib_max": 0.3046504360671572, "ttft_ms_p95_median": 236.5434074120637 }, "eligible_cell_count": 6, "numeric_best_generated_tokens_per_second_median": 84.93862051367972, "objective_tolerance_percent": 1.0, "reference_cell": { "client_concurrency": 1, "server_parallel": 1 }, "selected_cell": { "client_concurrency": 4, "generated_tokens_per_second_median": 84.93862051367972, "server_parallel": 4 }, "selection_basis": "Find the best median generated tokens/s among cells where both counterbalanced passes meet the load SLO, quality, RSS, and spread gates; treat cells within 1.0% of that best as equivalent, then apply the fixed lower-is-better tie-breakers.", "within_tolerance_cell_count": 1 }, { "candidate_id": "q4-kleidiai-tuned", "comparison_to_reference_percent": { "e2e_latency_ms_p95_median": 36.45953393236532, "generated_tokens_per_second_median": 176.27396303819415, "server_peak_rss_mib_max": 3.2962667206973135, "ttft_ms_p95_median": 276.1966257641767 }, "eligible_cell_count": 6, "numeric_best_generated_tokens_per_second_median": 90.66745405776943, "objective_tolerance_percent": 1.0, "reference_cell": { "client_concurrency": 1, "server_parallel": 1 }, "selected_cell": { "client_concurrency": 4, "generated_tokens_per_second_median": 90.66745405776943, "server_parallel": 4 }, "selection_basis": "Find the best median generated tokens/s among cells where both counterbalanced passes meet the load SLO, quality, RSS, and spread gates; treat cells within 1.0% of that best as equivalent, then apply the fixed lower-is-better tie-breakers.", "within_tolerance_cell_count": 1 } ], "server_configurations": [ { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19001" ], "load_argv_sha256": "0c0120483605d7f0dba419fa3034bafbf56b02d8b067f42feb1f1f2409df4c9c", "load_command_content_sha256": "c0cc25bc891115c2543d7ea13c0dc325a3d9815f33677a2fe82ce1642de74543", "load_command_sha256": "c0cc25bc891115c2543d7ea13c0dc325a3d9815f33677a2fe82ce1642de74543", "load_evaluation_content_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "load_evaluation_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19001", "rss_sha256": "3e0ddd7a8867fde99f37705070329ed6949d6a9ee88f38b8dbace9376a27af7a", "server_log_sha256": "78a509c69d633406dd49dc08d8aa0700e4b04793312af7e6093cd4885998f927", "server_parallel": 1, "server_peak_rss_mib": 3437.59375, "source_load_label": "forward/q8-generic/p1", "total_context_tokens": 2048 }, { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19002" ], "load_argv_sha256": "6f968f014fad213f4777e861e1438ee169eae505be7908faa445a2cc8b014bcc", "load_command_content_sha256": "cb195d55c879076548aa41afcbab18d10542c298a398dbae45e88e5e16793be1", "load_command_sha256": "cb195d55c879076548aa41afcbab18d10542c298a398dbae45e88e5e16793be1", "load_evaluation_content_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "load_evaluation_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19002", "rss_sha256": "bf0d05c1ca07c18a24eae784ea304f2ed27ed047dd7265e38b0abccf2c07fadb", "server_log_sha256": "cd6c830640c5e0844da6fbf513916ebb92193457a69fb6edc343736e9c1dba4f", "server_parallel": 2, "server_peak_rss_mib": 3437.55078125, "source_load_label": "forward/q8-generic/p2", "total_context_tokens": 4096 }, { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19004" ], "load_argv_sha256": "4e0a995a5742ef0bd90f5d71a4adecbdf7b40e16334f880d8b850a22a92d96da", "load_command_content_sha256": "3c6322934a8af4fb8da9fed8aae394fceb8e75bc2893eafb9729e4d1f830514d", "load_command_sha256": "3c6322934a8af4fb8da9fed8aae394fceb8e75bc2893eafb9729e4d1f830514d", "load_evaluation_content_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "load_evaluation_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19004", "rss_sha256": "24a54dcabe2e2af8e0fd5611d247ba463848e9deb6367b27d1e34d4e8b1d439a", "server_log_sha256": "cad02aa75e5c1bf65be57f4f8d805c997a8bcca9aacb42d4b571616523569c2b", "server_parallel": 4, "server_peak_rss_mib": 3446.19140625, "source_load_label": "forward/q8-generic/p4", "total_context_tokens": 8192 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19101" ], "load_argv_sha256": "19910d317cc1c88c6dc942aa0caad0708e98df9dd615ad04110f55500bf43a39", "load_command_content_sha256": "668def70f24a6530047427d9886a9e36c368306004a9d01b14e33805d215e1fb", "load_command_sha256": "668def70f24a6530047427d9886a9e36c368306004a9d01b14e33805d215e1fb", "load_evaluation_content_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "load_evaluation_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19101", "rss_sha256": "b846b5f62edeb02ef5fd632d21975714b20981e1780d5178be89767b0576717f", "server_log_sha256": "2cea71d76969107d17d141c453e51974e772de698c9d158d863ed578e61ad213", "server_parallel": 1, "server_peak_rss_mib": 1966.4765625, "source_load_label": "forward/q4-kleidiai-tuned/p1", "total_context_tokens": 2048 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19102" ], "load_argv_sha256": "d7e7076181251fae7a27a204fbdbea01b3051ef67c9b5fc1143d9c8ccf782893", "load_command_content_sha256": "0bb618e5ee0329ed932047625123aadd0623b6b9988cc893ed5e1769b2970ec1", "load_command_sha256": "0bb618e5ee0329ed932047625123aadd0623b6b9988cc893ed5e1769b2970ec1", "load_evaluation_content_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "load_evaluation_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19102", "rss_sha256": "0e9341ba713b06dc347cbcf72e5f96c872a45afe9dbe2525b1e7f6c427c04389", "server_log_sha256": "70d029c04d5f9d0d4d0fa6367814948c64278828120f995c25d97a465b20a71c", "server_parallel": 2, "server_peak_rss_mib": 1966.4296875, "source_load_label": "forward/q4-kleidiai-tuned/p2", "total_context_tokens": 4096 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19104" ], "load_argv_sha256": "d3adce9571eaad636067459ffa4330bfa8da2981f0fa30834899d621779b9a4c", "load_command_content_sha256": "2a1711f9ad0aee544bb43bc68e4c3367de2e6ed7831942cc8b92c18cb67ce2b5", "load_command_sha256": "2a1711f9ad0aee544bb43bc68e4c3367de2e6ed7831942cc8b92c18cb67ce2b5", "load_evaluation_content_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "load_evaluation_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "pass_id": "forward", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19104", "rss_sha256": "73c118c10defa9305962be19ca6e5ec781902a211fe8dea86c8d9ca4cea45f93", "server_log_sha256": "fd005b7df820669acc3f5b6e2962d380d50a65b19ebe114f4410450639001d35", "server_parallel": 4, "server_peak_rss_mib": 2031.296875, "source_load_label": "forward/q4-kleidiai-tuned/p4", "total_context_tokens": 8192 }, { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19301" ], "load_argv_sha256": "adc670ccf289a83252d1f8ebff91ef87a611c362bb6c224cd37af400cdcee746", "load_command_content_sha256": "9676ea730b51a9abda25d0effd0d04d2a3fca56d1d6d3e2e7df4ef061a345384", "load_command_sha256": "9676ea730b51a9abda25d0effd0d04d2a3fca56d1d6d3e2e7df4ef061a345384", "load_evaluation_content_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "load_evaluation_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19301", "rss_sha256": "df4b229b90bece7f25b2281cfebd79f78cfd4ba7285b6af436ab79ab2f514859", "server_log_sha256": "93062c048b15393500400fcd0136b0eca771f2595d5c9c9f0e30bbaefb14ea40", "server_parallel": 1, "server_peak_rss_mib": 3437.59765625, "source_load_label": "reverse/q8-generic/p1", "total_context_tokens": 2048 }, { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19302" ], "load_argv_sha256": "38bbd8a2c5b7b7d430595ce2fe0af08c6b4c28cd5dee3c2c1f85e4af466b82c1", "load_command_content_sha256": "27f04d125ee1111c43d19478526ef0ce87c832825f234ad4aa917ee79d643eee", "load_command_sha256": "27f04d125ee1111c43d19478526ef0ce87c832825f234ad4aa917ee79d643eee", "load_evaluation_content_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "load_evaluation_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19302", "rss_sha256": "a419a2ee0d170f91130af6d14f4e5b9ad1fbe0fb36cee8b99831dc3c2c2ffc7c", "server_log_sha256": "0944c80b188a69060f916d3c186e70a0e7bd341223603eeb95a436721a8e642d", "server_parallel": 2, "server_peak_rss_mib": 3437.55078125, "source_load_label": "reverse/q8-generic/p2", "total_context_tokens": 4096 }, { "candidate_id": "q8-generic", "canonical_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18081" ], "canonical_argv_sha256": "9f36fcb5b8b9f4db687bacab6e738185cc0146711fe75b3d06f4718d573bc77d", "canonical_command_sha256": "8b5040317a38b52930ab87494bb4d02aaccf696ebe7d0826df33816a3045d74f", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 0, "load_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19304" ], "load_argv_sha256": "e8d41962bbb2c18129c8e544a95fd9b343d4d3938ad277a896aad06f7383046b", "load_command_content_sha256": "e92a0a0b4ae9a9f688f3ecf990f7e93c0e542db86289794c7e97b9ba0b1bcc36", "load_command_sha256": "e92a0a0b4ae9a9f688f3ecf990f7e93c0e542db86289794c7e97b9ba0b1bcc36", "load_evaluation_content_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "load_evaluation_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19304", "rss_sha256": "475b4622d3d3bc939c9101637a4ae8b08ebdd56553c8855d0aac4867b9d2b6f9", "server_log_sha256": "e28375be04fcbd7f25f8cd604f0ef8a8c699ca567bb7ea4c8680e60568ca517e", "server_parallel": 4, "server_peak_rss_mib": 3448.0703125, "source_load_label": "reverse/q8-generic/p4", "total_context_tokens": 8192 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19201" ], "load_argv_sha256": "1dc50b6d5ad1033dccc0f187bd9aaaa4c27855a53594de8870b1426f8d5b76ba", "load_command_content_sha256": "08b73cb1bdc3d83c9f7dc7fd81538d9ce7ad73f54f12eed62308247a163f1bda", "load_command_sha256": "08b73cb1bdc3d83c9f7dc7fd81538d9ce7ad73f54f12eed62308247a163f1bda", "load_evaluation_content_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "load_evaluation_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19201", "rss_sha256": "439242e383330e146b0277c9b16782e3406390476a573800074120dd42ecd3dc", "server_log_sha256": "d12ab4b8bd2c45b360d2415987ef470f059df57841582d9370e208a51f03dfcd", "server_parallel": 1, "server_peak_rss_mib": 1966.47265625, "source_load_label": "reverse/q4-kleidiai-tuned/p1", "total_context_tokens": 2048 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19202" ], "load_argv_sha256": "9154cfcf30fc045ebb71319637940b9544dfaa52ac8f3f5fd2d8656f0f8af336", "load_command_content_sha256": "56b259eda7b1fe23da792cfed3097528d77b0ce849638c8aa19091ee022fd959", "load_command_sha256": "56b259eda7b1fe23da792cfed3097528d77b0ce849638c8aa19091ee022fd959", "load_evaluation_content_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "load_evaluation_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19202", "rss_sha256": "71a5bea46810674630de19cf8b6a6a5aeea5e12a9ecd3fbacbc5e1a351e2e791", "server_log_sha256": "bdce5f8d10724d84b761feeee4f7349385857cce42abb8698d332e9d97b91043", "server_parallel": 2, "server_peak_rss_mib": 1966.43359375, "source_load_label": "reverse/q4-kleidiai-tuned/p2", "total_context_tokens": 4096 }, { "candidate_id": "q4-kleidiai-tuned", "canonical_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "18084" ], "canonical_argv_sha256": "3fe8b661b4bba61c25525270f956795f3c044076addca347ced64c84a24c8740", "canonical_command_sha256": "4e09666b2e6542694e0a96f3e20f27697a7f9202155f7b9c80658fd51ccc3487", "differing_options": [ "--ctx-size", "--parallel", "--port" ], "kleidiai_model_buffer_marker_count": 1, "load_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19204" ], "load_argv_sha256": "cb8d3eb1ca738625ba4622b66b60a8b0b9a4d3b7dc3d612a1968efbc510e9640", "load_command_content_sha256": "77e23cb063072b4bb2a25c5b1b56e3243192c937dd10233624ae3704ba65b51a", "load_command_sha256": "77e23cb063072b4bb2a25c5b1b56e3243192c937dd10233624ae3704ba65b51a", "load_evaluation_content_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "load_evaluation_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "pass_id": "reverse", "per_slot_context_tokens": 2048, "request_base_url": "http://127.0.0.1:19204", "rss_sha256": "7df3f46af65ed7baec97a1ddd131fb0eebc29ec28f92aa9cf9b4b2bf11004709", "server_log_sha256": "ecb40574776faf6daa5e00d35de3a6749a9676ade4db2b94bfb10c19831e2c89", "server_parallel": 4, "server_peak_rss_mib": 2030.69140625, "source_load_label": "reverse/q4-kleidiai-tuned/p4", "total_context_tokens": 8192 } ], "source_artifacts": { "load_evaluations": { "forward/q4-kleidiai-tuned/p1": { "content_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19101", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 1, "canonical_port": 19101, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19101" ], "canonical_server_command_sha256": "668def70f24a6530047427d9886a9e36c368306004a9d01b14e33805d215e1fb", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 1, "load_port": 19101, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19101" ], "load_server_command_sha256": "668def70f24a6530047427d9886a9e36c368306004a9d01b14e33805d215e1fb" } }, "highest_slo_concurrency": 1, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1942.3130985000512, "e2e_latency_ms_p95": 2216.8644500000028, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 32.45226642100907, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5070666628282667, "samples": [ { "completed": true, "e2e_latency_ms": 2216.8644500000028, "error": null, "finished_at_seconds": 865.927525581, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 863.710661131, "ttft_ms": 144.56345799999326 }, { "completed": true, "e2e_latency_ms": 1937.4596709999423, "error": null, "finished_at_seconds": 867.865102368, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 865.927642697, "ttft_ms": 168.69994900002894 }, { "completed": true, "e2e_latency_ms": 1954.380044000004, "error": null, "finished_at_seconds": 869.819589399, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 867.865209355, "ttft_ms": 101.46955599998364 }, { "completed": true, "e2e_latency_ms": 1997.510770999952, "error": null, "finished_at_seconds": 871.817207221, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 869.81969645, "ttft_ms": 143.26558999994177 }, { "completed": true, "e2e_latency_ms": 1941.3045140000804, "error": null, "finished_at_seconds": 873.758619498, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 871.817314984, "ttft_ms": 168.96748500005287 }, { "completed": true, "e2e_latency_ms": 1874.089720000029, "error": null, "finished_at_seconds": 875.632816661, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 873.758726941, "ttft_ms": 100.72546800006421 }, { "completed": true, "e2e_latency_ms": 1911.3218190000225, "error": null, "finished_at_seconds": 877.544248164, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 875.632926345, "ttft_ms": 143.5552609999604 }, { "completed": true, "e2e_latency_ms": 1943.321683000022, "error": null, "finished_at_seconds": 879.487679394, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 877.544357711, "ttft_ms": 168.28632800002197 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 144.05935949997684, "ttft_ms_p95": 168.96748500005287, "wall_time_seconds": 15.777018262999945 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3831.8362499999807, "e2e_latency_ms_p95": 3896.099208999999, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.26389569820159, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5197483702843998, "samples": [ { "completed": true, "e2e_latency_ms": 1926.1912120000488, "error": null, "finished_at_seconds": 889.153841143, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 887.227649931, "ttft_ms": 143.1267720000733 }, { "completed": true, "e2e_latency_ms": 3868.5852439999735, "error": null, "finished_at_seconds": 891.096493982, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 887.227908738, "ttft_ms": 2093.925626999976 }, { "completed": true, "e2e_latency_ms": 3815.307118000078, "error": null, "finished_at_seconds": 892.969246695, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 889.153939577, "ttft_ms": 2043.5796660000278 }, { "completed": true, "e2e_latency_ms": 3802.355730000045, "error": null, "finished_at_seconds": 894.898956355, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 891.096600625, "ttft_ms": 2015.9120720001056 }, { "completed": true, "e2e_latency_ms": 3896.099208999999, "error": null, "finished_at_seconds": 896.865450555, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 892.969351346, "ttft_ms": 2098.362272999907 }, { "completed": true, "e2e_latency_ms": 3846.6523529999677, "error": null, "finished_at_seconds": 898.745749392, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 894.899097039, "ttft_ms": 2067.172938999988 }, { "completed": true, "e2e_latency_ms": 3817.020146999994, "error": null, "finished_at_seconds": 900.682572321, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 896.865552174, "ttft_ms": 2024.1367110000965 }, { "completed": true, "e2e_latency_ms": 3873.85424699994, "error": null, "finished_at_seconds": 902.619713586, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 898.745859339, "ttft_ms": 2107.5858809999772 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 2055.376302500008, "ttft_ms_p95": 2107.5858809999772, "wall_time_seconds": 15.392063655000015 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 7732.8995579999855, "e2e_latency_ms_p95": 7758.620410000049, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.081674744586984, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5169011678841716, "samples": [ { "completed": true, "e2e_latency_ms": 1955.2690279999752, "error": null, "finished_at_seconds": 912.310520539, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 910.355251511, "ttft_ms": 162.86699499994484 }, { "completed": true, "e2e_latency_ms": 7741.946858000006, "error": null, "finished_at_seconds": 918.097419351, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 910.355472493, "ttft_ms": 5948.793192000039 }, { "completed": true, "e2e_latency_ms": 3843.0761649999567, "error": null, "finished_at_seconds": 914.198789641, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 910.355713476, "ttft_ms": 2054.752891000021 }, { "completed": true, "e2e_latency_ms": 5780.051281999931, "error": null, "finished_at_seconds": 916.136069534, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 910.356018252, "ttft_ms": 3985.04274000004 }, { "completed": true, "e2e_latency_ms": 7758.620410000049, "error": null, "finished_at_seconds": 920.069229608, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 912.310609198, "ttft_ms": 5955.2314940000315 }, { "completed": true, "e2e_latency_ms": 7737.503165000021, "error": null, "finished_at_seconds": 921.936379409, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 914.198876244, "ttft_ms": 5969.818140000029 }, { "completed": true, "e2e_latency_ms": 7731.22549499999, "error": null, "finished_at_seconds": 923.86738988, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 916.136164385, "ttft_ms": 5942.588583999964 }, { "completed": true, "e2e_latency_ms": 7734.573620999981, "error": null, "finished_at_seconds": 925.832097951, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 918.09752433, "ttft_ms": 5937.754154999993 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 5940.171369499978, "ttft_ms_p95": 5969.818140000029, "wall_time_seconds": 15.476846440000031 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "368260c1a115cf02d0f623c57ab91cb405ce9ced459573471c107bcbb807b2e5" }, "forward/q4-kleidiai-tuned/p2": { "content_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19102", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 2, "canonical_port": 19102, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19102" ], "canonical_server_command_sha256": "0bb618e5ee0329ed932047625123aadd0623b6b9988cc893ed5e1769b2970ec1", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 2, "load_port": 19102, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19102" ], "load_server_command_sha256": "0bb618e5ee0329ed932047625123aadd0623b6b9988cc893ed5e1769b2970ec1" } }, "highest_slo_concurrency": 2, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1922.7700220000088, "e2e_latency_ms_p95": 1950.3699480000023, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.31412800964699, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5205332501507343, "samples": [ { "completed": true, "e2e_latency_ms": 1927.2841399999834, "error": null, "finished_at_seconds": 943.65089391, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 941.72360977, "ttft_ms": 144.34505599990644 }, { "completed": true, "e2e_latency_ms": 1950.3699480000023, "error": null, "finished_at_seconds": 945.601410502, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 943.651040554, "ttft_ms": 168.94154700003128 }, { "completed": true, "e2e_latency_ms": 1887.2978129999183, "error": null, "finished_at_seconds": 947.488814966, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 945.601517153, "ttft_ms": 102.11654099998668 }, { "completed": true, "e2e_latency_ms": 1918.2559040000342, "error": null, "finished_at_seconds": 949.407176555, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 947.488920651, "ttft_ms": 142.99291900010758 }, { "completed": true, "e2e_latency_ms": 1950.2773089999437, "error": null, "finished_at_seconds": 951.357558715, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 949.407281406, "ttft_ms": 168.8555409999708 }, { "completed": true, "e2e_latency_ms": 1885.0282719999996, "error": null, "finished_at_seconds": 953.242694462, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 951.35766619, "ttft_ms": 100.81771399995887 }, { "completed": true, "e2e_latency_ms": 1910.8280059999743, "error": null, "finished_at_seconds": 955.153629735, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 953.242801729, "ttft_ms": 143.32873800003654 }, { "completed": true, "e2e_latency_ms": 1938.724331000003, "error": null, "finished_at_seconds": 957.092464685, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 955.153740354, "ttft_ms": 168.42407199999343 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 143.8368969999715, "ttft_ms_p95": 168.94154700003128, "wall_time_seconds": 15.368854914999929 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 2933.9892075000193, "e2e_latency_ms_p95": 2951.646697000001, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 43.83258865350848, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.68488419771107, "samples": [ { "completed": true, "e2e_latency_ms": 2937.7478440000004, "error": null, "finished_at_seconds": 965.809292628, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 962.871544784, "ttft_ms": 312.8580710000506 }, { "completed": true, "e2e_latency_ms": 2937.124633000053, "error": null, "finished_at_seconds": 965.808921841, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 962.871797208, "ttft_ms": 312.336791000007 }, { "completed": true, "e2e_latency_ms": 2860.817675000021, "error": null, "finished_at_seconds": 968.669853191, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 965.809035516, "ttft_ms": 243.87059499997576 }, { "completed": true, "e2e_latency_ms": 2860.777913999982, "error": null, "finished_at_seconds": 968.670136912, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 965.809358998, "ttft_ms": 243.84974600002352 }, { "completed": true, "e2e_latency_ms": 2951.646697000001, "error": null, "finished_at_seconds": 971.621619596, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 968.669972899, "ttft_ms": 286.9450660000439 }, { "completed": true, "e2e_latency_ms": 2951.0560559999703, "error": null, "finished_at_seconds": 971.621260755, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 968.670204699, "ttft_ms": 286.4284490000273 }, { "completed": true, "e2e_latency_ms": 2930.8537819999856, "error": null, "finished_at_seconds": 974.552204341, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 971.621350559, "ttft_ms": 314.5024420000482 }, { "completed": true, "e2e_latency_ms": 2930.6636100000105, "error": null, "finished_at_seconds": 974.552351593, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 971.621687983, "ttft_ms": 314.21002100000806 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 299.64092850002544, "ttft_ms_p95": 314.5024420000482, "wall_time_seconds": 11.680806809000046 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 5860.738167999954, "e2e_latency_ms_p95": 6017.29259199999, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 43.51438989640389, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6799123421313108, "samples": [ { "completed": true, "e2e_latency_ms": 3036.2260950000746, "error": null, "finished_at_seconds": 983.421529347, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 980.385303252, "ttft_ms": 272.08174300005794 }, { "completed": true, "e2e_latency_ms": 6017.29259199999, "error": null, "finished_at_seconds": 986.402877733, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 980.385585141, "ttft_ms": 3231.956867000008 }, { "completed": true, "e2e_latency_ms": 2853.0901459999995, "error": null, "finished_at_seconds": 983.238892638, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 980.385802492, "ttft_ms": 100.85139499994966 }, { "completed": true, "e2e_latency_ms": 5813.756872999988, "error": null, "finished_at_seconds": 986.199967635, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 980.386210762, "ttft_ms": 3035.882635000007 }, { "completed": true, "e2e_latency_ms": 5913.241252000034, "error": null, "finished_at_seconds": 989.152225413, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 983.238984161, "ttft_ms": 3163.710632000061 }, { "completed": true, "e2e_latency_ms": 5907.719462999921, "error": null, "finished_at_seconds": 989.329350517, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 983.421631054, "ttft_ms": 3109.88918999999 }, { "completed": true, "e2e_latency_ms": 5922.405872000013, "error": null, "finished_at_seconds": 992.122486565, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 986.200080693, "ttft_ms": 3129.2265900000302 }, { "completed": true, "e2e_latency_ms": 5748.537303000035, "error": null, "finished_at_seconds": 992.151525902, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 986.402988599, "ttft_ms": 3122.063078999986 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 3115.976134499988, "ttft_ms_p95": 3231.956867000008, "wall_time_seconds": 11.766222650000032 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "9d0a4e032bc8a9ccac02af188f3ed91f79f06705b75e940b2c6b60bc4566ced7" }, "forward/q4-kleidiai-tuned/p4": { "content_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19104", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 4, "canonical_port": 19104, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19104" ], "canonical_server_command_sha256": "2a1711f9ad0aee544bb43bc68e4c3367de2e6ed7831942cc8b92c18cb67ce2b5", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 4, "load_port": 19104, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19104" ], "load_server_command_sha256": "2a1711f9ad0aee544bb43bc68e4c3367de2e6ed7831942cc8b92c18cb67ce2b5" } }, "highest_slo_concurrency": 4, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1924.7720149999736, "e2e_latency_ms_p95": 1960.1255799999535, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.32025517439394, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5206289870999053, "samples": [ { "completed": true, "e2e_latency_ms": 1925.6731269999818, "error": null, "finished_at_seconds": 1010.357532776, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1008.431859649, "ttft_ms": 142.78833399998803 }, { "completed": true, "e2e_latency_ms": 1939.3417449999788, "error": null, "finished_at_seconds": 1012.296989788, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1010.357648043, "ttft_ms": 167.28653999996368 }, { "completed": true, "e2e_latency_ms": 1880.1884279999967, "error": null, "finished_at_seconds": 1014.177302244, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1012.297113816, "ttft_ms": 100.64840500001537 }, { "completed": true, "e2e_latency_ms": 1923.8709029999654, "error": null, "finished_at_seconds": 1016.101280158, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1014.177409255, "ttft_ms": 144.13765099993725 }, { "completed": true, "e2e_latency_ms": 1943.282302000057, "error": null, "finished_at_seconds": 1018.044666623, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1016.101384321, "ttft_ms": 167.97324900005606 }, { "completed": true, "e2e_latency_ms": 1875.746481999954, "error": null, "finished_at_seconds": 1019.92051698, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1018.044770498, "ttft_ms": 101.48924899999656 }, { "completed": true, "e2e_latency_ms": 1917.0293280000124, "error": null, "finished_at_seconds": 1021.837653287, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1019.920623959, "ttft_ms": 142.21356699999887 }, { "completed": true, "e2e_latency_ms": 1960.1255799999535, "error": null, "finished_at_seconds": 1023.79788843, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1021.83776285, "ttft_ms": 168.4933159999673 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 143.46299249996264, "ttft_ms_p95": 168.4933159999673, "wall_time_seconds": 15.366028780999955 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 2943.971048499975, "e2e_latency_ms_p95": 3827.4439890001304, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 40.47207855761459, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6323762274627279, "samples": [ { "completed": true, "e2e_latency_ms": 2944.263120999949, "error": null, "finished_at_seconds": 1033.536043648, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1030.591780527, "ttft_ms": 314.82071300001735 }, { "completed": true, "e2e_latency_ms": 2943.678976000001, "error": null, "finished_at_seconds": 1033.535748759, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1030.592069783, "ttft_ms": 314.41168600008496 }, { "completed": true, "e2e_latency_ms": 3827.4439890001304, "error": null, "finished_at_seconds": 1037.363280855, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1033.535836866, "ttft_ms": 244.20149100001254 }, { "completed": true, "e2e_latency_ms": 3827.422582000054, "error": null, "finished_at_seconds": 1037.363539296, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1033.536116714, "ttft_ms": 244.0084930001376 }, { "completed": true, "e2e_latency_ms": 2908.2613369998853, "error": null, "finished_at_seconds": 1040.271645652, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1037.363384315, "ttft_ms": 284.76097799989475 }, { "completed": true, "e2e_latency_ms": 2907.819570999891, "error": null, "finished_at_seconds": 1040.271427941, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1037.36360837, "ttft_ms": 284.43068799992943 }, { "completed": true, "e2e_latency_ms": 2940.715100000034, "error": null, "finished_at_seconds": 1043.212236084, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1040.271520984, "ttft_ms": 143.4024220000083 }, { "completed": true, "e2e_latency_ms": 2970.758012000033, "error": null, "finished_at_seconds": 1043.242477458, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1040.271719446, "ttft_ms": 339.10964200003946 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 284.5958329999121, "ttft_ms_p95": 339.10964200003946, "wall_time_seconds": 12.650696930999857 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 2813.476547999926, "e2e_latency_ms_p95": 2841.4814269999624, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 90.97481662809376, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 1.421481509813965, "samples": [ { "completed": true, "e2e_latency_ms": 2757.7455020000343, "error": null, "finished_at_seconds": 1048.717104182, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1045.95935868, "ttft_ms": 434.3279600000187 }, { "completed": true, "e2e_latency_ms": 2758.2378919998973, "error": null, "finished_at_seconds": 1048.717819971, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1045.959582079, "ttft_ms": 434.4860839999001 }, { "completed": true, "e2e_latency_ms": 2757.842759999903, "error": null, "finished_at_seconds": 1048.717690887, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1045.959848127, "ttft_ms": 437.23899700012225 }, { "completed": true, "e2e_latency_ms": 2787.9073049998624, "error": null, "finished_at_seconds": 1048.74816466, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1045.960257355, "ttft_ms": 623.6396679998961 }, { "completed": true, "e2e_latency_ms": 2841.4814269999624, "error": null, "finished_at_seconds": 1051.5586915, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1048.717210073, "ttft_ms": 476.3175929999761 }, { "completed": true, "e2e_latency_ms": 2840.705059999891, "error": null, "finished_at_seconds": 1051.558453037, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1048.717747977, "ttft_ms": 475.58398000001034 }, { "completed": true, "e2e_latency_ms": 2840.5266710001342, "error": null, "finished_at_seconds": 1051.558387171, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1048.7178605, "ttft_ms": 475.32212400005847 }, { "completed": true, "e2e_latency_ms": 2839.0457909999895, "error": null, "finished_at_seconds": 1051.587289782, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1048.748243991, "ttft_ms": 664.8192449999897 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 475.4530520000344, "ttft_ms_p95": 664.8192449999897, "wall_time_seconds": 5.627931102000048 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "303876a463004ffd1eb43baf798c0e83c1a62f5c8a2e5010c519ad597406bbce" }, "forward/q8-generic/p1": { "content_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19001", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 1, "canonical_port": 19001, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19001" ], "canonical_server_command_sha256": "c0cc25bc891115c2543d7ea13c0dc325a3d9815f33677a2fe82ce1642de74543", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 1, "load_port": 19001, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19001" ], "load_server_command_sha256": "c0cc25bc891115c2543d7ea13c0dc325a3d9815f33677a2fe82ce1642de74543" } }, "highest_slo_concurrency": 1, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1817.0775390000244, "e2e_latency_ms_p95": 1884.9235650000082, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.134759195622934, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5489806124316083, "samples": [ { "completed": true, "e2e_latency_ms": 1884.9235650000082, "error": null, "finished_at_seconds": 662.742996789, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 660.858073224, "ttft_ms": 165.51505600000382 }, { "completed": true, "e2e_latency_ms": 1844.6773580000126, "error": null, "finished_at_seconds": 664.587808599, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 662.743131241, "ttft_ms": 221.13820899994607 }, { "completed": true, "e2e_latency_ms": 1776.2657250000302, "error": null, "finished_at_seconds": 666.364183895, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 664.58791817, "ttft_ms": 128.75927400000364 }, { "completed": true, "e2e_latency_ms": 1789.4777200000362, "error": null, "finished_at_seconds": 668.15376801, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 666.36429029, "ttft_ms": 165.84538500001145 }, { "completed": true, "e2e_latency_ms": 1851.448892999997, "error": null, "finished_at_seconds": 670.00533185, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 668.153882957, "ttft_ms": 220.81145800007107 }, { "completed": true, "e2e_latency_ms": 1783.0509400000665, "error": null, "finished_at_seconds": 671.788492257, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 670.005441317, "ttft_ms": 124.33138800008692 }, { "completed": true, "e2e_latency_ms": 1786.3312360000236, "error": null, "finished_at_seconds": 673.5749398, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 671.788608564, "ttft_ms": 165.4231100000061 }, { "completed": true, "e2e_latency_ms": 1855.465578999997, "error": null, "finished_at_seconds": 675.430536839, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 673.57507126, "ttft_ms": 223.21656899998743 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 165.68022050000764, "ttft_ms_p95": 223.21656899998743, "wall_time_seconds": 14.572463614999947 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3572.244969499991, "e2e_latency_ms_p95": 3668.967000000066, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.47060058516804, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5542281341432507, "samples": [ { "completed": true, "e2e_latency_ms": 3658.605499999908, "error": null, "finished_at_seconds": 686.305246639, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 682.646641139, "ttft_ms": 2009.1443219999974 }, { "completed": true, "e2e_latency_ms": 1842.7648979999276, "error": null, "finished_at_seconds": 684.489666741, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 682.646901843, "ttft_ms": 219.69096999998783 }, { "completed": true, "e2e_latency_ms": 3561.4905349999617, "error": null, "finished_at_seconds": 688.051249959, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 684.489759424, "ttft_ms": 1940.4304470000397 }, { "completed": true, "e2e_latency_ms": 3581.7418189999444, "error": null, "finished_at_seconds": 689.887094309, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 686.30535249, "ttft_ms": 1913.284894999947 }, { "completed": true, "e2e_latency_ms": 3668.967000000066, "error": null, "finished_at_seconds": 691.720311569, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 688.051344569, "ttft_ms": 2056.6426259999844 }, { "completed": true, "e2e_latency_ms": 3562.7481200000375, "error": null, "finished_at_seconds": 693.449947808, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 689.887199688, "ttft_ms": 1957.3650410000027 }, { "completed": true, "e2e_latency_ms": 3524.066990000051, "error": null, "finished_at_seconds": 695.244485538, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 691.720418548, "ttft_ms": 1895.9971580000001 }, { "completed": true, "e2e_latency_ms": 3631.0622259999263, "error": null, "finished_at_seconds": 697.081130309, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 693.450068083, "ttft_ms": 2013.2017789999281 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 1948.8977440000212, "ttft_ms_p95": 2056.6426259999844, "wall_time_seconds": 14.434489169999893 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 7150.125154500018, "e2e_latency_ms_p95": 7254.258521999986, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.52994853934689, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5551554459272952, "samples": [ { "completed": true, "e2e_latency_ms": 1768.7644110000065, "error": null, "finished_at_seconds": 706.026961108, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 704.258196697, "ttft_ms": 166.25205899993034 }, { "completed": true, "e2e_latency_ms": 5420.625770000015, "error": null, "finished_at_seconds": 709.679094555, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 704.258468785, "ttft_ms": 3787.7770860000055 }, { "completed": true, "e2e_latency_ms": 7163.617042000055, "error": null, "finished_at_seconds": 711.42232873, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 704.258711688, "ttft_ms": 5544.479814000056 }, { "completed": true, "e2e_latency_ms": 3565.085682000017, "error": null, "finished_at_seconds": 707.824039552, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 704.25895387, "ttft_ms": 1935.5078080000112 }, { "completed": true, "e2e_latency_ms": 7254.258521999986, "error": null, "finished_at_seconds": 713.281334337, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 706.027075815, "ttft_ms": 5617.4895230000175 }, { "completed": true, "e2e_latency_ms": 7202.386719999936, "error": null, "finished_at_seconds": 715.026518283, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 707.824131563, "ttft_ms": 5581.364098999984 }, { "completed": true, "e2e_latency_ms": 7136.633266999979, "error": null, "finished_at_seconds": 716.815818872, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 709.679185605, "ttft_ms": 5514.868338999918 }, { "completed": true, "e2e_latency_ms": 7246.13488, "error": null, "finished_at_seconds": 718.668575013, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 711.422440133, "ttft_ms": 5613.021936999985 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 5529.674076499987, "ttft_ms_p95": 5617.4895230000175, "wall_time_seconds": 14.410378315999992 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "6579320b2df1c1007bfc09847e3b92671353d0e8cbab2418d3b3250148e218fc" }, "forward/q8-generic/p2": { "content_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19002", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 2, "canonical_port": 19002, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19002" ], "canonical_server_command_sha256": "cb195d55c879076548aa41afcbab18d10542c298a398dbae45e88e5e16793be1", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 2, "load_port": 19002, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19002" ], "load_server_command_sha256": "cb195d55c879076548aa41afcbab18d10542c298a398dbae45e88e5e16793be1" } }, "highest_slo_concurrency": 2, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1792.4390155000083, "e2e_latency_ms_p95": 1854.2111919999797, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.54798524427483, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5554372694417942, "samples": [ { "completed": true, "e2e_latency_ms": 1799.6011520000366, "error": null, "finished_at_seconds": 735.950291114, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 734.150689962, "ttft_ms": 167.9450670000051 }, { "completed": true, "e2e_latency_ms": 1854.2111919999797, "error": null, "finished_at_seconds": 737.804618541, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 735.950407349, "ttft_ms": 220.57710800004315 }, { "completed": true, "e2e_latency_ms": 1743.1727939999746, "error": null, "finished_at_seconds": 739.547897058, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 737.804724264, "ttft_ms": 125.03677199993035 }, { "completed": true, "e2e_latency_ms": 1782.1569610000552, "error": null, "finished_at_seconds": 741.330185463, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 739.548028502, "ttft_ms": 166.5812090000145 }, { "completed": true, "e2e_latency_ms": 1850.1653649999525, "error": null, "finished_at_seconds": 743.180458735, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 741.33029337, "ttft_ms": 221.34429299990188 }, { "completed": true, "e2e_latency_ms": 1743.7808180000047, "error": null, "finished_at_seconds": 744.924342515, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 743.180561697, "ttft_ms": 124.22410199997103 }, { "completed": true, "e2e_latency_ms": 1785.27687899998, "error": null, "finished_at_seconds": 746.709727837, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 744.924450958, "ttft_ms": 165.5832779999855 }, { "completed": true, "e2e_latency_ms": 1843.9187590000756, "error": null, "finished_at_seconds": 748.553756591, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 746.709837832, "ttft_ms": 222.6910530000623 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 167.2631380000098, "ttft_ms_p95": 222.6910530000623, "wall_time_seconds": 14.403066629000023 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3288.6674119999384, "e2e_latency_ms_p95": 3329.0332560000024, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 38.8899080673751, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.607654813552736, "samples": [ { "completed": true, "e2e_latency_ms": 3317.861975000028, "error": null, "finished_at_seconds": 758.472899946, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 755.155037971, "ttft_ms": 388.90586200000143 }, { "completed": true, "e2e_latency_ms": 3317.2856469999488, "error": null, "finished_at_seconds": 758.472583745, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 755.155298098, "ttft_ms": 388.5543479999569 }, { "completed": true, "e2e_latency_ms": 3259.154586999898, "error": null, "finished_at_seconds": 761.731833967, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 758.47267938, "ttft_ms": 290.14125999992757 }, { "completed": true, "e2e_latency_ms": 3258.4227910000436, "error": null, "finished_at_seconds": 761.731387579, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 758.472964788, "ttft_ms": 289.9561179999637 }, { "completed": true, "e2e_latency_ms": 3260.049176999928, "error": null, "finished_at_seconds": 764.991526134, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 761.731476957, "ttft_ms": 345.3710159999446 }, { "completed": true, "e2e_latency_ms": 3259.3073799999956, "error": null, "finished_at_seconds": 764.991207005, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 761.731899625, "ttft_ms": 344.780046999972 }, { "completed": true, "e2e_latency_ms": 3329.0332560000024, "error": null, "finished_at_seconds": 768.320335576, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 764.99130232, "ttft_ms": 391.7180969999663 }, { "completed": true, "e2e_latency_ms": 3328.8138020000133, "error": null, "finished_at_seconds": 768.320407226, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 764.991593424, "ttft_ms": 391.5092749999758 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 366.96268199995075, "ttft_ms_p95": 391.7180969999663, "wall_time_seconds": 13.165369255000087 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 6590.93548650003, "e2e_latency_ms_p95": 6684.801609000033, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 38.665443722985536, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.604147558171649, "samples": [ { "completed": true, "e2e_latency_ms": 3224.4116140000187, "error": null, "finished_at_seconds": 778.083784265, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 774.859372651, "ttft_ms": 290.5478210000183 }, { "completed": true, "e2e_latency_ms": 6556.4199110000345, "error": null, "finished_at_seconds": 781.416034985, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 774.859615074, "ttft_ms": 3613.7769619999744 }, { "completed": true, "e2e_latency_ms": 3227.2162020000223, "error": null, "finished_at_seconds": 778.087100364, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 774.859884162, "ttft_ms": 289.9118110000245 }, { "completed": true, "e2e_latency_ms": 6556.157462999977, "error": null, "finished_at_seconds": 781.416399259, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 774.860241796, "ttft_ms": 3613.249003000078 }, { "completed": true, "e2e_latency_ms": 6628.541714999983, "error": null, "finished_at_seconds": 784.712424895, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 778.08388318, "ttft_ms": 3678.413920999901 }, { "completed": true, "e2e_latency_ms": 6625.451062000025, "error": null, "finished_at_seconds": 784.712632332, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 778.08718127, "ttft_ms": 3675.013669000009 }, { "completed": true, "e2e_latency_ms": 6684.801609000033, "error": null, "finished_at_seconds": 788.100947765, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 781.416146156, "ttft_ms": 3696.6847560000815 }, { "completed": true, "e2e_latency_ms": 6684.7126380000645, "error": null, "finished_at_seconds": 788.101170771, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 781.416458133, "ttft_ms": 3696.4502690000245 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 3644.3953154999917, "ttft_ms_p95": 3696.6847560000815, "wall_time_seconds": 13.241798120000112 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "b95c1d47a41e8b04af198ca9cc5284f9fd911f2b1ed437d731df3336bcf883c7" }, "forward/q8-generic/p4": { "content_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19004", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 4, "canonical_port": 19004, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19004" ], "canonical_server_command_sha256": "3c6322934a8af4fb8da9fed8aae394fceb8e75bc2893eafb9729e4d1f830514d", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 4, "load_port": 19004, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19004" ], "load_server_command_sha256": "3c6322934a8af4fb8da9fed8aae394fceb8e75bc2893eafb9729e4d1f830514d" } }, "highest_slo_concurrency": 4, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1813.337024500015, "e2e_latency_ms_p95": 1858.8786120000123, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.3130448362052, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5517663255657063, "samples": [ { "completed": true, "e2e_latency_ms": 1801.1070759999939, "error": null, "finished_at_seconds": 805.485867007, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 803.684759931, "ttft_ms": 165.74043599996457 }, { "completed": true, "e2e_latency_ms": 1856.9588880000083, "error": null, "finished_at_seconds": 807.342943987, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 805.485985099, "ttft_ms": 220.43871999994735 }, { "completed": true, "e2e_latency_ms": 1788.836133000018, "error": null, "finished_at_seconds": 809.131911476, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 807.343075343, "ttft_ms": 124.24855099993692 }, { "completed": true, "e2e_latency_ms": 1790.6289510000306, "error": null, "finished_at_seconds": 810.922672999, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 809.132044048, "ttft_ms": 164.88964400002715 }, { "completed": true, "e2e_latency_ms": 1846.4468409999881, "error": null, "finished_at_seconds": 812.769226763, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 810.922779922, "ttft_ms": 219.97946199996932 }, { "completed": true, "e2e_latency_ms": 1729.6356339999193, "error": null, "finished_at_seconds": 814.49896924, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 812.769333606, "ttft_ms": 124.14542900000924 }, { "completed": true, "e2e_latency_ms": 1825.5669730000363, "error": null, "finished_at_seconds": 816.324664417, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 814.499097444, "ttft_ms": 175.44937600007415 }, { "completed": true, "e2e_latency_ms": 1858.8786120000123, "error": null, "finished_at_seconds": 818.183651272, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 816.32477266, "ttft_ms": 220.54406899997048 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 170.59490600001936, "ttft_ms_p95": 220.54406899997048, "wall_time_seconds": 14.49889134099999 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3373.2164354999554, "e2e_latency_ms_p95": 3534.7170180000376, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 37.69445819948809, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5889759093670014, "samples": [ { "completed": true, "e2e_latency_ms": 3365.3682560000107, "error": null, "finished_at_seconds": 828.44144457, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 825.076076314, "ttft_ms": 391.03381600000375 }, { "completed": true, "e2e_latency_ms": 3364.7364700000253, "error": null, "finished_at_seconds": 828.44108868, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 825.07635221, "ttft_ms": 390.64111699997284 }, { "completed": true, "e2e_latency_ms": 3534.7170180000376, "error": null, "finished_at_seconds": 831.975900821, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 828.441183803, "ttft_ms": 289.8365590000367 }, { "completed": true, "e2e_latency_ms": 3534.1733230000045, "error": null, "finished_at_seconds": 831.975685223, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 828.4415119, "ttft_ms": 289.4157629999654 }, { "completed": true, "e2e_latency_ms": 3381.6066219999357, "error": null, "finished_at_seconds": 835.357382352, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 831.97577573, "ttft_ms": 342.85653699998875 }, { "completed": true, "e2e_latency_ms": 3381.0646149999, "error": null, "finished_at_seconds": 835.357036166, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 831.975971551, "ttft_ms": 342.56565699990915 }, { "completed": true, "e2e_latency_ms": 3301.7258000000993, "error": null, "finished_at_seconds": 838.658860953, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 835.357135153, "ttft_ms": 387.93740800008436 }, { "completed": true, "e2e_latency_ms": 3301.523266999993, "error": null, "finished_at_seconds": 838.658974821, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 835.357451554, "ttft_ms": 387.4933790000341 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 365.17495800001143, "ttft_ms_p95": 391.03381600000375, "wall_time_seconds": 13.582898506999982 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 3007.988222499989, "e2e_latency_ms_p95": 3090.0780179999856, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 84.384245678028, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 1.3185038387191874, "samples": [ { "completed": true, "e2e_latency_ms": 2976.0522299999366, "error": null, "finished_at_seconds": 844.622631385, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 841.646579155, "ttft_ms": 671.098575999963 }, { "completed": true, "e2e_latency_ms": 2977.125329000046, "error": null, "finished_at_seconds": 844.623929931, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 841.646804602, "ttft_ms": 671.1574500000097 }, { "completed": true, "e2e_latency_ms": 2975.8850740000753, "error": null, "finished_at_seconds": 844.622980285, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 841.647095211, "ttft_ms": 670.6998750000821 }, { "completed": true, "e2e_latency_ms": 2976.3143750000154, "error": null, "finished_at_seconds": 844.623662778, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 841.647348403, "ttft_ms": 670.9066019999455 }, { "completed": true, "e2e_latency_ms": 3039.5453439999756, "error": null, "finished_at_seconds": 847.662276853, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 844.622731509, "ttft_ms": 346.3997249999693 }, { "completed": true, "e2e_latency_ms": 3038.8511159999325, "error": null, "finished_at_seconds": 847.661904115, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 844.623052999, "ttft_ms": 345.98621599991475 }, { "completed": true, "e2e_latency_ms": 3090.0780179999856, "error": null, "finished_at_seconds": 847.713793454, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 844.623715436, "ttft_ms": 776.3692160000346 }, { "completed": true, "e2e_latency_ms": 3090.064329000029, "error": null, "finished_at_seconds": 847.714062438, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 844.623998109, "ttft_ms": 776.5359169999329 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 671.0025889999542, "ttft_ms_p95": 776.5359169999329, "wall_time_seconds": 6.067483283000001 } ], "schema_version": "1.0", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "cd3592ab909287cc27d61dc709008fa769c9c51b11b5d6a87b4cb44e2f732796" }, "reverse/q4-kleidiai-tuned/p1": { "content_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19201", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 1, "canonical_port": 19201, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19201" ], "canonical_server_command_sha256": "08b73cb1bdc3d83c9f7dc7fd81538d9ce7ad73f54f12eed62308247a163f1bda", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 1, "load_port": 19201, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19201" ], "load_server_command_sha256": "08b73cb1bdc3d83c9f7dc7fd81538d9ce7ad73f54f12eed62308247a163f1bda" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 1, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1935.3954115000533, "e2e_latency_ms_p95": 1964.0805829999408, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.183635754278356, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5184943086605993, "samples": [ { "completed": true, "e2e_latency_ms": 1936.5994540000884, "error": null, "finished_at_seconds": 1249.36391557, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1247.427316116, "ttft_ms": 143.41856900000494 }, { "completed": true, "e2e_latency_ms": 1964.0805829999408, "error": null, "finished_at_seconds": 1251.328125493, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1249.36404491, "ttft_ms": 171.24502100000427 }, { "completed": true, "e2e_latency_ms": 1882.248824000044, "error": null, "finished_at_seconds": 1253.210478016, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1251.328229192, "ttft_ms": 101.05466300001353 }, { "completed": true, "e2e_latency_ms": 1934.1913690000183, "error": null, "finished_at_seconds": 1255.144778132, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1253.210586763, "ttft_ms": 143.4980669998822 }, { "completed": true, "e2e_latency_ms": 1959.424399999989, "error": null, "finished_at_seconds": 1257.104313783, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1255.144889383, "ttft_ms": 168.73901000008118 }, { "completed": true, "e2e_latency_ms": 1874.6500980000746, "error": null, "finished_at_seconds": 1258.979075212, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1257.104425114, "ttft_ms": 100.90678699998534 }, { "completed": true, "e2e_latency_ms": 1923.7113599999702, "error": null, "finished_at_seconds": 1260.902890231, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1258.979178871, "ttft_ms": 143.68147000004683 }, { "completed": true, "e2e_latency_ms": 1953.6108469999363, "error": null, "finished_at_seconds": 1262.856607945, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1260.902997098, "ttft_ms": 169.44998799999667 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 143.58976849996452, "ttft_ms_p95": 171.24502100000427, "wall_time_seconds": 15.429291829000022 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3845.3899770000817, "e2e_latency_ms_p95": 3881.8361439998625, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 33.223827608350966, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5191223063804838, "samples": [ { "completed": true, "e2e_latency_ms": 1928.0881470001532, "error": null, "finished_at_seconds": 1226.178592939, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1224.250504792, "ttft_ms": 143.3231160001469 }, { "completed": true, "e2e_latency_ms": 3881.8361439998625, "error": null, "finished_at_seconds": 1228.132615392, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1224.250779248, "ttft_ms": 2094.8557839999467 }, { "completed": true, "e2e_latency_ms": 3843.838352000148, "error": null, "finished_at_seconds": 1230.022540694, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1226.178702342, "ttft_ms": 2054.3465509999805 }, { "completed": true, "e2e_latency_ms": 3814.2397389999587, "error": null, "finished_at_seconds": 1231.946954038, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1228.132714299, "ttft_ms": 2032.5660199998765 }, { "completed": true, "e2e_latency_ms": 3877.296810000189, "error": null, "finished_at_seconds": 1233.899953603, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1230.022656793, "ttft_ms": 2092.990079000174 }, { "completed": true, "e2e_latency_ms": 3846.9416020000153, "error": null, "finished_at_seconds": 1235.794028179, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1231.947086577, "ttft_ms": 2053.2752379999692 }, { "completed": true, "e2e_latency_ms": 3815.275681000003, "error": null, "finished_at_seconds": 1237.715365688, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1233.900090007, "ttft_ms": 2036.1369620000005 }, { "completed": true, "e2e_latency_ms": 3867.0015010000043, "error": null, "finished_at_seconds": 1239.661131347, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1235.794129846, "ttft_ms": 2089.2393019998963 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 2053.810894499975, "ttft_ms_p95": 2094.8557839999467, "wall_time_seconds": 15.410626555000135 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 7769.548415500026, "e2e_latency_ms_p95": 7849.5314490000965, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 32.862249628087135, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5134726504388615, "samples": [ { "completed": true, "e2e_latency_ms": 3829.4982419999997, "error": null, "finished_at_seconds": 1204.798269438, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1200.968771196, "ttft_ms": 2042.2257840000384 }, { "completed": true, "e2e_latency_ms": 5802.931575000002, "error": null, "finished_at_seconds": 1206.771968883, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1200.969037308, "ttft_ms": 3998.840254000015 }, { "completed": true, "e2e_latency_ms": 1896.8548110001393, "error": null, "finished_at_seconds": 1202.866147054, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1200.969292243, "ttft_ms": 109.80160799999794 }, { "completed": true, "e2e_latency_ms": 7729.695940000056, "error": null, "finished_at_seconds": 1208.699317649, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1200.969621709, "ttft_ms": 5945.6153749999885 }, { "completed": true, "e2e_latency_ms": 7833.193381000001, "error": null, "finished_at_seconds": 1210.699436214, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1202.866242833, "ttft_ms": 6002.015609000182 }, { "completed": true, "e2e_latency_ms": 7835.916873000087, "error": null, "finished_at_seconds": 1212.634294962, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1204.798378089, "ttft_ms": 6001.0993870000675 }, { "completed": true, "e2e_latency_ms": 7809.400890999996, "error": null, "finished_at_seconds": 1214.581503386, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1206.772102495, "ttft_ms": 6004.807088000007 }, { "completed": true, "e2e_latency_ms": 7849.5314490000965, "error": null, "finished_at_seconds": 1216.548958365, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1208.699426916, "ttft_ms": 6049.835923000046 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 5973.357381000028, "ttft_ms_p95": 6049.835923000046, "wall_time_seconds": 15.580187169000055 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "b0938cdf135fbcd04840b801b8a4d01bcede8b2e2143dcac6f41653d916a4aff" }, "reverse/q4-kleidiai-tuned/p2": { "content_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19202", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 2, "canonical_port": 19202, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19202" ], "canonical_server_command_sha256": "56b259eda7b1fe23da792cfed3097528d77b0ce849638c8aa19091ee022fd959", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 2, "load_port": 19202, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19202" ], "load_server_command_sha256": "56b259eda7b1fe23da792cfed3097528d77b0ce849638c8aa19091ee022fd959" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 2, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1949.0506720001122, "e2e_latency_ms_p95": 1978.485245999991, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 32.900713571849174, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5140736495601433, "samples": [ { "completed": true, "e2e_latency_ms": 1952.378010000075, "error": null, "finished_at_seconds": 1171.31506562, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1169.36268761, "ttft_ms": 144.5654569999988 }, { "completed": true, "e2e_latency_ms": 1962.4616959999912, "error": null, "finished_at_seconds": 1173.277631311, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1171.315169615, "ttft_ms": 168.4364159998495 }, { "completed": true, "e2e_latency_ms": 1889.467624999952, "error": null, "finished_at_seconds": 1175.167205411, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1173.277737786, "ttft_ms": 101.78666599995267 }, { "completed": true, "e2e_latency_ms": 1945.7233340001494, "error": null, "finished_at_seconds": 1177.11303502, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1175.167311686, "ttft_ms": 143.14714800002548 }, { "completed": true, "e2e_latency_ms": 1978.485245999991, "error": null, "finished_at_seconds": 1179.091629517, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1177.113144271, "ttft_ms": 167.6983150000524 }, { "completed": true, "e2e_latency_ms": 1910.780000999921, "error": null, "finished_at_seconds": 1181.002520881, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1179.09174088, "ttft_ms": 101.38827999981004 }, { "completed": true, "e2e_latency_ms": 1944.583878000003, "error": null, "finished_at_seconds": 1182.947213419, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1181.002629541, "ttft_ms": 143.31174599988117 }, { "completed": true, "e2e_latency_ms": 1977.3373089999495, "error": null, "finished_at_seconds": 1184.924660115, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1182.947322806, "ttft_ms": 172.64410099983252 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 143.93860149993998, "ttft_ms_p95": 172.64410099983252, "wall_time_seconds": 15.561972505000085 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 2940.558489999944, "e2e_latency_ms_p95": 2982.4296659999163, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 43.62433363520482, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6816302130500753, "samples": [ { "completed": true, "e2e_latency_ms": 2982.3984409999866, "error": null, "finished_at_seconds": 1152.811969935, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1149.829571494, "ttft_ms": 314.7659329999897 }, { "completed": true, "e2e_latency_ms": 2982.4296659999163, "error": null, "finished_at_seconds": 1152.812213254, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1149.829783588, "ttft_ms": 314.4472039998618 }, { "completed": true, "e2e_latency_ms": 2872.769083000094, "error": null, "finished_at_seconds": 1155.684857149, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1152.812088066, "ttft_ms": 245.28825400011556 }, { "completed": true, "e2e_latency_ms": 2873.018400999854, "error": null, "finished_at_seconds": 1155.685296057, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1152.812277656, "ttft_ms": 245.1834029998281 }, { "completed": true, "e2e_latency_ms": 2932.4118320000707, "error": null, "finished_at_seconds": 1158.617366847, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1155.684955015, "ttft_ms": 286.3331979999657 }, { "completed": true, "e2e_latency_ms": 2931.788941999912, "error": null, "finished_at_seconds": 1158.617153465, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1155.685364523, "ttft_ms": 285.6237290000081 }, { "completed": true, "e2e_latency_ms": 2948.7164049999137, "error": null, "finished_at_seconds": 1161.565964912, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1158.617248507, "ttft_ms": 314.67277299998386 }, { "completed": true, "e2e_latency_ms": 2948.705147999817, "error": null, "finished_at_seconds": 1161.566140453, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1158.617435305, "ttft_ms": 314.62450699996225 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 300.39020099991376, "ttft_ms_p95": 314.7659329999897, "wall_time_seconds": 11.736568958999896 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 5861.868001500056, "e2e_latency_ms_p95": 6030.341808999992, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 43.55757587147319, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6805871229917686, "samples": [ { "completed": true, "e2e_latency_ms": 5833.761908000042, "error": null, "finished_at_seconds": 1138.058040722, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1132.224278814, "ttft_ms": 3055.24554800013 }, { "completed": true, "e2e_latency_ms": 3055.502291000039, "error": null, "finished_at_seconds": 1135.280050561, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1132.22454827, "ttft_ms": 306.3658419998774 }, { "completed": true, "e2e_latency_ms": 2884.1469599999527, "error": null, "finished_at_seconds": 1135.108937836, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1132.224790876, "ttft_ms": 104.240511999933 }, { "completed": true, "e2e_latency_ms": 6030.341808999992, "error": null, "finished_at_seconds": 1138.255564082, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1132.225222273, "ttft_ms": 3231.0797979998824 }, { "completed": true, "e2e_latency_ms": 5889.9740950000705, "error": null, "finished_at_seconds": 1140.999195243, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1135.109221148, "ttft_ms": 3145.8580160001475 }, { "completed": true, "e2e_latency_ms": 5891.256715000054, "error": null, "finished_at_seconds": 1141.171404278, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1135.280147563, "ttft_ms": 3109.4186490001903 }, { "completed": true, "e2e_latency_ms": 5891.953150000063, "error": null, "finished_at_seconds": 1143.950098131, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1138.058144981, "ttft_ms": 3112.6612069999737 }, { "completed": true, "e2e_latency_ms": 5723.170048999918, "error": null, "finished_at_seconds": 1143.978835622, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1138.255665573, "ttft_ms": 3116.9329100000596 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 3111.039928000082, "ttft_ms_p95": 3231.0797979998824, "wall_time_seconds": 11.754556807999961 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "0f4c0b656427902bdddb8f4ecbd436e99050319803d13e353f3498549b8ef282" }, "reverse/q4-kleidiai-tuned/p4": { "content_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19204", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "canonical_parallel": 4, "canonical_port": 19204, "canonical_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19204" ], "canonical_server_command_sha256": "77e23cb063072b4bb2a25c5b1b56e3243192c937dd10233624ae3704ba65b51a", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "load_parallel": 4, "load_port": 19204, "load_runtime_path": "./.candidate-build/kleidiai/bin/llama-server", "load_server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19204" ], "load_server_command_sha256": "77e23cb063072b4bb2a25c5b1b56e3243192c937dd10233624ae3704ba65b51a" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 4, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1944.1725359999964, "e2e_latency_ms_p95": 1987.104448999844, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 32.981993359812954, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5153436462470774, "samples": [ { "completed": true, "e2e_latency_ms": 1940.514394000047, "error": null, "finished_at_seconds": 1104.527282358, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1102.586767964, "ttft_ms": 142.79193199990914 }, { "completed": true, "e2e_latency_ms": 1964.8174179999387, "error": null, "finished_at_seconds": 1106.492211396, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1104.527393978, "ttft_ms": 167.43826100014303 }, { "completed": true, "e2e_latency_ms": 1897.98266899993, "error": null, "finished_at_seconds": 1108.390302644, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1106.492319975, "ttft_ms": 101.79524199998014 }, { "completed": true, "e2e_latency_ms": 1947.8306779999457, "error": null, "finished_at_seconds": 1110.338239813, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1108.390409135, "ttft_ms": 143.31117200003973 }, { "completed": true, "e2e_latency_ms": 1962.0510099998683, "error": null, "finished_at_seconds": 1112.300397258, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1110.338346248, "ttft_ms": 169.6911359999831 }, { "completed": true, "e2e_latency_ms": 1885.4787899999792, "error": null, "finished_at_seconds": 1114.185984179, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1112.300505389, "ttft_ms": 101.13320599998588 }, { "completed": true, "e2e_latency_ms": 1937.058013999831, "error": null, "finished_at_seconds": 1116.123180677, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1114.186122663, "ttft_ms": 143.86829499994747 }, { "completed": true, "e2e_latency_ms": 1987.104448999844, "error": null, "finished_at_seconds": 1118.110390033, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1116.123285584, "ttft_ms": 171.85684399987622 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 143.5897334999936, "ttft_ms_p95": 171.85684399987622, "wall_time_seconds": 15.523622068999885 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3921.6093740000133, "e2e_latency_ms_p95": 3933.9661210001395, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 34.9013418466546, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5453334663539782, "samples": [ { "completed": true, "e2e_latency_ms": 3923.167913000043, "error": null, "finished_at_seconds": 1084.111402659, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1080.188234746, "ttft_ms": 310.71384200004104 }, { "completed": true, "e2e_latency_ms": 3922.471021000092, "error": null, "finished_at_seconds": 1084.110981583, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1080.188510562, "ttft_ms": 310.3484949999711 }, { "completed": true, "e2e_latency_ms": 2892.4082089999956, "error": null, "finished_at_seconds": 1087.003515548, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1084.111107339, "ttft_ms": 246.5345719999732 }, { "completed": true, "e2e_latency_ms": 2891.827791999958, "error": null, "finished_at_seconds": 1087.003299045, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1084.111471253, "ttft_ms": 246.0575019999851 }, { "completed": true, "e2e_latency_ms": 3920.69413299987, "error": null, "finished_at_seconds": 1090.924086861, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1087.003392728, "ttft_ms": 269.7439689998191 }, { "completed": true, "e2e_latency_ms": 3920.7477269999345, "error": null, "finished_at_seconds": 1090.924335325, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1087.003587598, "ttft_ms": 269.6673180000744 }, { "completed": true, "e2e_latency_ms": 3933.9661210001395, "error": null, "finished_at_seconds": 1094.858157818, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1090.924191697, "ttft_ms": 310.36084900006244 }, { "completed": true, "e2e_latency_ms": 3933.5033390000262, "error": null, "finished_at_seconds": 1094.857906746, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1090.924403407, "ttft_ms": 310.0630250000904 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 289.90349699995477, "ttft_ms_p95": 310.71384200004104, "wall_time_seconds": 14.669923072000074 }, { "candidate_id": "q4-kleidiai-tuned", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 2820.5771580001056, "e2e_latency_ms_p95": 2863.816679000138, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 90.3600914874451, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 1.4118764294913297, "samples": [ { "completed": true, "e2e_latency_ms": 2751.1710349999703, "error": null, "finished_at_seconds": 1070.426146737, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1067.674975702, "ttft_ms": 147.26570099992387 }, { "completed": true, "e2e_latency_ms": 2801.9922779999433, "error": null, "finished_at_seconds": 1070.47726562, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1067.675273342, "ttft_ms": 591.1751779999577 }, { "completed": true, "e2e_latency_ms": 2802.7686830000675, "error": null, "finished_at_seconds": 1070.478329018, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1067.675560335, "ttft_ms": 591.2807160000284 }, { "completed": true, "e2e_latency_ms": 2802.2696620000715, "error": null, "finished_at_seconds": 1070.478145109, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1067.675875447, "ttft_ms": 590.7521579999866 }, { "completed": true, "e2e_latency_ms": 2838.3856330001436, "error": null, "finished_at_seconds": 1073.264641893, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1070.42625626, "ttft_ms": 219.79017300009218 }, { "completed": true, "e2e_latency_ms": 2863.816679000138, "error": null, "finished_at_seconds": 1073.341193958, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1070.477377279, "ttft_ms": 615.0487229999726 }, { "completed": true, "e2e_latency_ms": 2862.815641999987, "error": null, "finished_at_seconds": 1073.341028945, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1070.478213303, "ttft_ms": 613.9883490000102 }, { "completed": true, "e2e_latency_ms": 2862.4985540000125, "error": null, "finished_at_seconds": 1073.340931511, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1070.478432957, "ttft_ms": 613.4726630000387 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 591.227946999993, "ttft_ms_p95": 615.0487229999726, "wall_time_seconds": 5.666218256000093 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "16bd7d7414d08463e8196f44c6a623b1e198fe9ad552284a4d4a6cbd032f9903" }, "reverse/q8-generic/p1": { "content_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19301", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 1, "canonical_port": 19301, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19301" ], "canonical_server_command_sha256": "9676ea730b51a9abda25d0effd0d04d2a3fca56d1d6d3e2e7df4ef061a345384", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 1, "load_port": 19301, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19301" ], "load_server_command_sha256": "9676ea730b51a9abda25d0effd0d04d2a3fca56d1d6d3e2e7df4ef061a345384" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 1, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1815.675844999987, "e2e_latency_ms_p95": 1881.7339209999773, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.06622390485378, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5479097485133403, "samples": [ { "completed": true, "e2e_latency_ms": 1806.5500379998412, "error": null, "finished_at_seconds": 1453.694221865, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1451.887671827, "ttft_ms": 165.63123599985374 }, { "completed": true, "e2e_latency_ms": 1869.4023670000206, "error": null, "finished_at_seconds": 1455.563730707, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1453.69432834, "ttft_ms": 221.80836199981968 }, { "completed": true, "e2e_latency_ms": 1764.5947769999566, "error": null, "finished_at_seconds": 1457.328436008, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1455.563841231, "ttft_ms": 125.20989199992982 }, { "completed": true, "e2e_latency_ms": 1807.256561000031, "error": null, "finished_at_seconds": 1459.135798724, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1457.328542163, "ttft_ms": 165.29293799999323 }, { "completed": true, "e2e_latency_ms": 1877.8589529999863, "error": null, "finished_at_seconds": 1461.01376592, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1459.135906967, "ttft_ms": 222.36860800012437 }, { "completed": true, "e2e_latency_ms": 1768.69459500017, "error": null, "finished_at_seconds": 1462.782568798, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1461.013874203, "ttft_ms": 124.80727100000877 }, { "completed": true, "e2e_latency_ms": 1824.095128999943, "error": null, "finished_at_seconds": 1464.606774555, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1462.782679426, "ttft_ms": 167.96294200003103 }, { "completed": true, "e2e_latency_ms": 1881.7339209999773, "error": null, "finished_at_seconds": 1466.488616639, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1464.606882718, "ttft_ms": 226.40649200002372 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 166.7970889999424, "ttft_ms_p95": 226.40649200002372, "wall_time_seconds": 14.600944811999852 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3648.3754194999847, "e2e_latency_ms_p95": 3687.7824379998856, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.04666039190969, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5476040686235889, "samples": [ { "completed": true, "e2e_latency_ms": 1814.7351500001605, "error": null, "finished_at_seconds": 1431.841260757, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1430.026525607, "ttft_ms": 169.373152000162 }, { "completed": true, "e2e_latency_ms": 3683.064965000085, "error": null, "finished_at_seconds": 1433.709799762, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1430.026734797, "ttft_ms": 2034.1108309999072 }, { "completed": true, "e2e_latency_ms": 3643.1995039999947, "error": null, "finished_at_seconds": 1435.484566376, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1431.841366872, "ttft_ms": 1994.0209500000492 }, { "completed": true, "e2e_latency_ms": 3589.8167950001607, "error": null, "finished_at_seconds": 1437.299728808, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1433.709912013, "ttft_ms": 1940.8430299999964 }, { "completed": true, "e2e_latency_ms": 3687.7824379998856, "error": null, "finished_at_seconds": 1439.172452953, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1435.484670515, "ttft_ms": 2040.5987489998552 }, { "completed": true, "e2e_latency_ms": 3653.5513349999746, "error": null, "finished_at_seconds": 1440.953397307, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1437.299845972, "ttft_ms": 1997.4130160001096 }, { "completed": true, "e2e_latency_ms": 3598.329012000022, "error": null, "finished_at_seconds": 1442.770879376, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1439.172550364, "ttft_ms": 1948.074015999964 }, { "completed": true, "e2e_latency_ms": 3682.1143269999084, "error": null, "finished_at_seconds": 1444.635620861, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1440.953506534, "ttft_ms": 2037.7132549999715 } ], "slo_failures": [ "ttft_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 1995.7169830000794, "ttft_ms_p95": 2040.5987489998552, "wall_time_seconds": 14.609095254000067 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 7335.751101000028, "e2e_latency_ms_p95": 7441.183771000169, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 34.75048650506307, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5429763516416105, "samples": [ { "completed": true, "e2e_latency_ms": 7344.256806999965, "error": null, "finished_at_seconds": 1415.293119152, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1407.948862345, "ttft_ms": 5693.363883000075 }, { "completed": true, "e2e_latency_ms": 5526.1891139998625, "error": null, "finished_at_seconds": 1413.475282792, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1407.949093678, "ttft_ms": 3860.6838139999127 }, { "completed": true, "e2e_latency_ms": 1811.7477939999844, "error": null, "finished_at_seconds": 1409.761143551, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1407.949395757, "ttft_ms": 128.8550019999093 }, { "completed": true, "e2e_latency_ms": 3638.4512849999737, "error": null, "finished_at_seconds": 1411.588121161, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1407.949669876, "ttft_ms": 1979.2135870000038 }, { "completed": true, "e2e_latency_ms": 7441.183771000169, "error": null, "finished_at_seconds": 1417.202429285, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1409.761245514, "ttft_ms": 5752.069523000046 }, { "completed": true, "e2e_latency_ms": 7387.895656999945, "error": null, "finished_at_seconds": 1418.976113949, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1411.588218292, "ttft_ms": 5738.65243399996 }, { "completed": true, "e2e_latency_ms": 7327.245395000091, "error": null, "finished_at_seconds": 1420.802627862, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1413.475382467, "ttft_ms": 5666.380020000133 }, { "completed": true, "e2e_latency_ms": 7389.240642000004, "error": null, "finished_at_seconds": 1422.682469022, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1415.29322838, "ttft_ms": 5731.178444999841 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 5679.871951500104, "ttft_ms_p95": 5752.069523000046, "wall_time_seconds": 14.733606677000125 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "6331a036c9bfdb18c1cc8604e3296a138ee29d9a750e35e4ca0542ed81d56b3e" }, "reverse/q8-generic/p2": { "content_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19302", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 2, "canonical_port": 19302, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19302" ], "canonical_server_command_sha256": "27f04d125ee1111c43d19478526ef0ce87c832825f234ad4aa917ee79d643eee", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 2, "load_port": 19302, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19302" ], "load_server_command_sha256": "27f04d125ee1111c43d19478526ef0ce87c832825f234ad4aa917ee79d643eee" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 2, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1830.196205499874, "e2e_latency_ms_p95": 1873.1312170000365, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.16462890593168, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5494473266551825, "samples": [ { "completed": true, "e2e_latency_ms": 1811.3272609998603, "error": null, "finished_at_seconds": 1379.54292434, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1377.731597079, "ttft_ms": 167.69964900004197 }, { "completed": true, "e2e_latency_ms": 1856.535569000016, "error": null, "finished_at_seconds": 1381.399591761, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1379.543056192, "ttft_ms": 222.4489770001128 }, { "completed": true, "e2e_latency_ms": 1755.4261430000224, "error": null, "finished_at_seconds": 1383.155126747, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1381.399700604, "ttft_ms": 125.43569200011007 }, { "completed": true, "e2e_latency_ms": 1797.7491959998133, "error": null, "finished_at_seconds": 1384.952980698, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1383.155231502, "ttft_ms": 167.09145400000125 }, { "completed": true, "e2e_latency_ms": 1861.8762880000759, "error": null, "finished_at_seconds": 1386.814987054, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1384.953110766, "ttft_ms": 221.78007799993793 }, { "completed": true, "e2e_latency_ms": 1754.1344020000906, "error": null, "finished_at_seconds": 1388.5692613, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1386.815126898, "ttft_ms": 127.18080299987378 }, { "completed": true, "e2e_latency_ms": 1849.0651499998876, "error": null, "finished_at_seconds": 1390.418438125, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1388.569372975, "ttft_ms": 167.98827799993887 }, { "completed": true, "e2e_latency_ms": 1873.1312170000365, "error": null, "finished_at_seconds": 1392.291682481, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1390.418551264, "ttft_ms": 231.5461990001495 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 167.84396349999042, "ttft_ms_p95": 231.5461990001495, "wall_time_seconds": 14.560085401999913 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3301.9177405000164, "e2e_latency_ms_p95": 3365.6050560000494, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 38.81548786800706, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6064919979376103, "samples": [ { "completed": true, "e2e_latency_ms": 3318.7592319998203, "error": null, "finished_at_seconds": 1360.62330336, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1357.304544128, "ttft_ms": 386.9170049999866 }, { "completed": true, "e2e_latency_ms": 3318.326083000102, "error": null, "finished_at_seconds": 1360.623081529, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1357.304755446, "ttft_ms": 386.5881880001325 }, { "completed": true, "e2e_latency_ms": 3220.9719850000056, "error": null, "finished_at_seconds": 1363.844150141, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1360.623178156, "ttft_ms": 288.6133649999465 }, { "completed": true, "e2e_latency_ms": 3221.0343790000024, "error": null, "finished_at_seconds": 1363.844410477, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1360.623376098, "ttft_ms": 288.5745239998414 }, { "completed": true, "e2e_latency_ms": 3285.509397999931, "error": null, "finished_at_seconds": 1367.129760086, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1363.844250688, "ttft_ms": 346.13292099993487 }, { "completed": true, "e2e_latency_ms": 3284.9081179999757, "error": null, "finished_at_seconds": 1367.129380884, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1363.844472766, "ttft_ms": 346.01720599994223 }, { "completed": true, "e2e_latency_ms": 3365.6050560000494, "error": null, "finished_at_seconds": 1370.495077902, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1367.129472846, "ttft_ms": 387.9238280001118 }, { "completed": true, "e2e_latency_ms": 3365.3303199998845, "error": null, "finished_at_seconds": 1370.495155096, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1367.129824776, "ttft_ms": 387.67351699993924 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 366.3605545000337, "ttft_ms_p95": 387.9238280001118, "wall_time_seconds": 13.190610967999874 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 6626.796836499921, "e2e_latency_ms_p95": 6654.875219999894, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 38.628533640139615, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.6035708381271815, "samples": [ { "completed": true, "e2e_latency_ms": 6599.3941149999955, "error": null, "finished_at_seconds": 1344.079233421, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1337.479839306, "ttft_ms": 3627.9245919999994 }, { "completed": true, "e2e_latency_ms": 6599.443924999832, "error": null, "finished_at_seconds": 1344.079524246, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1337.480080321, "ttft_ms": 3627.545828999928 }, { "completed": true, "e2e_latency_ms": 3240.792843999998, "error": null, "finished_at_seconds": 1340.721177501, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1337.480384657, "ttft_ms": 292.72692499989716 }, { "completed": true, "e2e_latency_ms": 3241.0839320000377, "error": null, "finished_at_seconds": 1340.721730109, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1337.480646177, "ttft_ms": 293.1556650000857 }, { "completed": true, "e2e_latency_ms": 6654.253759000085, "error": null, "finished_at_seconds": 1347.375530495, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1340.721276736, "ttft_ms": 3704.2261950000466 }, { "completed": true, "e2e_latency_ms": 6654.149748000009, "error": null, "finished_at_seconds": 1347.375948443, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1340.721798695, "ttft_ms": 3703.7925189999896 }, { "completed": true, "e2e_latency_ms": 6654.875219999894, "error": null, "finished_at_seconds": 1350.734214092, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1344.079338872, "ttft_ms": 3683.583575999819 }, { "completed": true, "e2e_latency_ms": 6654.69275800001, "error": null, "finished_at_seconds": 1350.734290142, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1344.079597384, "ttft_ms": 3683.432980000134 } ], "slo_failures": [ "ttft_ms_p95_above_maximum", "e2e_latency_ms_p95_above_maximum" ], "slo_met": false, "ttft_ms_p50": 3655.678786000067, "ttft_ms_p95": 3704.2261950000466, "wall_time_seconds": 13.254450835999933 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "f4a599eb6eb7f06c39ff0deb40cb949888a22ea63e859538a955a7177725b189" }, "reverse/q8-generic/p4": { "content_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf", "evaluation": { "candidate_id": "q8-generic", "evidence_binding": { "plan_sha256": "b10f61b7093f58ad6d142deafd7d6befe600d1d54a433b59c101131dd8c2d9bb", "request_base_url": "http://127.0.0.1:19304", "server_configuration": { "allowed_binding_options": [ "--host", "--port" ], "argv_equivalent_except_binding": true, "canonical_host": "127.0.0.1", "canonical_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "canonical_parallel": 4, "canonical_port": 19304, "canonical_runtime_path": "./.candidate-build/generic/bin/llama-server", "canonical_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19304" ], "canonical_server_command_sha256": "e92a0a0b4ae9a9f688f3ecf990f7e93c0e542db86289794c7e97b9ba0b1bcc36", "differing_binding_options": [], "load_host": "127.0.0.1", "load_model_path": "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "load_parallel": 4, "load_port": 19304, "load_runtime_path": "./.candidate-build/generic/bin/llama-server", "load_server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19304" ], "load_server_command_sha256": "e92a0a0b4ae9a9f688f3ecf990f7e93c0e542db86289794c7e97b9ba0b1bcc36" } }, "execution_order": [ 4, 2, 1 ], "highest_slo_concurrency": 4, "methodology": { "client_scheduler": "bounded thread pool", "concurrency_levels": [ 1, 2, 4 ], "measured_requests_per_level": 8, "output_tokens": 64, "prompt_schedule": "round-robin", "prompts": [ { "index": 1, "sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "text": "An engineering team is choosing an Arm64 inference configuration. Explain in two concise sentences why measured latency and memory evidence should be reviewed together." }, { "index": 2, "sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "text": "Summarize this deployment note in two concise sentences: the service must preserve response quality, keep tail latency within its declared limit, and leave enough memory for other workloads." }, { "index": 3, "sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "text": "Describe in two concise sentences how a reproducible benchmark helps a student developer make an honest deployment decision." } ], "warmup_requests_per_level": 4 }, "rows": [ { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 1, "e2e_latency_ms_p50": 1799.9500524999803, "e2e_latency_ms_p95": 1854.306553000015, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 35.52597658797478, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.555093384187106, "samples": [ { "completed": true, "e2e_latency_ms": 1793.8257640000757, "error": null, "finished_at_seconds": 1309.980336562, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1308.186510798, "ttft_ms": 164.98762900005204 }, { "completed": true, "e2e_latency_ms": 1854.306553000015, "error": null, "finished_at_seconds": 1311.834749614, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1309.980443061, "ttft_ms": 221.72692699996333 }, { "completed": true, "e2e_latency_ms": 1748.1440500000645, "error": null, "finished_at_seconds": 1313.582995603, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1311.834851553, "ttft_ms": 124.1721349999807 }, { "completed": true, "e2e_latency_ms": 1793.478769000103, "error": null, "finished_at_seconds": 1315.376598151, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1313.583119382, "ttft_ms": 165.85359100008645 }, { "completed": true, "e2e_latency_ms": 1837.2645919998831, "error": null, "finished_at_seconds": 1317.213972434, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1315.376707842, "ttft_ms": 220.98911599982785 }, { "completed": true, "e2e_latency_ms": 1739.2485000000306, "error": null, "finished_at_seconds": 1318.953351914, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1317.214103414, "ttft_ms": 124.10780899995189 }, { "completed": true, "e2e_latency_ms": 1806.074340999885, "error": null, "finished_at_seconds": 1320.75953005, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1318.953455709, "ttft_ms": 165.80425500001184 }, { "completed": true, "e2e_latency_ms": 1838.8647450001372, "error": null, "finished_at_seconds": 1322.598500254, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1320.759635509, "ttft_ms": 219.9909830001161 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 165.82892300004914, "ttft_ms_p95": 221.72692699996333, "wall_time_seconds": 14.411989456000128 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 2, "e2e_latency_ms_p50": 3492.994119000059, "e2e_latency_ms_p95": 3655.1638060000187, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 36.49275434021204, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 0.5701992865658131, "samples": [ { "completed": true, "e2e_latency_ms": 3408.061306000036, "error": null, "finished_at_seconds": 1290.358836952, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1286.950775646, "ttft_ms": 440.80272599990167 }, { "completed": true, "e2e_latency_ms": 3378.182686999935, "error": null, "finished_at_seconds": 1290.329215316, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1286.951032629, "ttft_ms": 220.7953469999211 }, { "completed": true, "e2e_latency_ms": 3577.926932000082, "error": null, "finished_at_seconds": 1293.907236067, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1290.329309135, "ttft_ms": 155.23122800004785 }, { "completed": true, "e2e_latency_ms": 3582.412735999924, "error": null, "finished_at_seconds": 1293.941338722, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1290.358925986, "ttft_ms": 317.6387559999512 }, { "completed": true, "e2e_latency_ms": 3655.1638060000187, "error": null, "finished_at_seconds": 1297.562506412, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1293.907342606, "ttft_ms": 259.52032600002894 }, { "completed": true, "e2e_latency_ms": 3648.2900920000247, "error": null, "finished_at_seconds": 1297.589730321, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1293.941440229, "ttft_ms": 374.3121270001666 }, { "completed": true, "e2e_latency_ms": 3391.7489469999964, "error": null, "finished_at_seconds": 1300.954364778, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1297.562615831, "ttft_ms": 193.4863270000733 }, { "completed": true, "e2e_latency_ms": 3391.1333549999654, "error": null, "finished_at_seconds": 1300.980958055, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1297.5898247, "ttft_ms": 413.8632560000133 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 288.57954099999006, "ttft_ms_p95": 440.80272599990167, "wall_time_seconds": 14.030182408999963 }, { "candidate_id": "q8-generic", "completed_requests": 8, "completion_rate": 1.0, "concurrency": 4, "e2e_latency_ms_p50": 2993.9723695000566, "e2e_latency_ms_p95": 3023.502796999992, "error_rate": 0.0, "failed_requests": 0, "generated_tokens_per_second": 85.49299534933145, "peak_rss_mib": null, "request_count": 8, "requests_per_second": 1.335828052333304, "samples": [ { "completed": true, "e2e_latency_ms": 2965.6667660001403, "error": null, "finished_at_seconds": 1277.058606169, "generated_tokens": 64, "index": 1, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1274.092939403, "ttft_ms": 664.8510699999406 }, { "completed": true, "e2e_latency_ms": 2964.693851999982, "error": null, "finished_at_seconds": 1277.057937592, "generated_tokens": 64, "index": 2, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1274.09324374, "ttft_ms": 664.2328210000414 }, { "completed": true, "e2e_latency_ms": 2965.2648180001506, "error": null, "finished_at_seconds": 1277.058764173, "generated_tokens": 64, "index": 3, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1274.093499355, "ttft_ms": 664.5392279999669 }, { "completed": true, "e2e_latency_ms": 2965.1704139998856, "error": null, "finished_at_seconds": 1277.058975746, "generated_tokens": 64, "index": 4, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1274.093805332, "ttft_ms": 663.7897040000098 }, { "completed": true, "e2e_latency_ms": 3023.502796999992, "error": null, "finished_at_seconds": 1280.08155028, "generated_tokens": 64, "index": 5, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1277.058047483, "ttft_ms": 736.6408529999262 }, { "completed": true, "e2e_latency_ms": 3023.054649999949, "error": null, "finished_at_seconds": 1280.081734133, "generated_tokens": 64, "index": 6, "prompt_index": 3, "prompt_sha256": "a48be792aff96c3b81cc8be1bc60faa0a072f1f106cf956261339b33d0d18d04", "started_at_seconds": 1277.058679483, "ttft_ms": 736.2344749999465 }, { "completed": true, "e2e_latency_ms": 3022.666599000104, "error": null, "finished_at_seconds": 1280.081493654, "generated_tokens": 64, "index": 7, "prompt_index": 1, "prompt_sha256": "549face5a90122a765c1cbefe79cff73ed227a759e36590082aefbe44dcb5291", "started_at_seconds": 1277.058827055, "ttft_ms": 735.701052999957 }, { "completed": true, "e2e_latency_ms": 3022.277972999973, "error": null, "finished_at_seconds": 1280.081335377, "generated_tokens": 64, "index": 8, "prompt_index": 2, "prompt_sha256": "5fd318ce3ceb729df9ee61f9e38c59fcd89cb885b01f4f0c8025053abb67571c", "started_at_seconds": 1277.059057404, "ttft_ms": 735.3181890000542 } ], "slo_failures": [], "slo_met": true, "ttft_ms_p50": 700.0846294999974, "ttft_ms_p95": 736.6408529999262, "wall_time_seconds": 5.988794730000109 } ], "schema_version": "1.1", "slo": { "max_e2e_latency_ms_p95": 6500.0, "max_ttft_ms_p95": 2000.0, "min_completion_rate": 1.0 }, "synthetic": false }, "input_sha256": "3b2a7c6d393bb7d33bb054c69d811dc41726dac61cfa89cc6780157b23e12dcf" } }, "quality_wrappers": { "q4-kleidiai-tuned/p1": { "content_sha256": "11f4d029b55759c6d92a10175c5ecf3b87f24ca54a2d299a4e5555b827712c45", "input_sha256": "11f4d029b55759c6d92a10175c5ecf3b87f24ca54a2d299a4e5555b827712c45", "wrapper": { "candidate_id": "q4-kleidiai-tuned", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "latency": { "e2e_latency_ms_p50": 2284.654440000054, "e2e_latency_ms_p95": 2293.3533010000247, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2284.4380020001154, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 466.31947300011234 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2284.8708779999924, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 467.39481600002364 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2289.918377000049, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 469.0895759999876 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2287.9131300001063, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 466.062397000087 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2293.3533010000247, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 466.81301800003894 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2277.868593999983, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 466.9927339998594 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2283.9246870000807, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 468.20059400010905 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2276.5719669998816, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 465.87182700000085 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2270.419574000016, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 468.77770999981294 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2285.6652030000078, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 467.6987670000017 } ], "ttft_ms_p50": 467.1937749999415, "ttft_ms_p95": 469.0895759999876 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PARETOPILOT", "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "PARETOPILOT" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-10" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "```json\n[\"red\",\"blue\"]\n```" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 20, "score": 0.8333333333333334, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "8d98444c54c8eed866e532dff6f68c9e68494800448831a62f66b29451999db0", "evaluation_sha256": "8d98444c54c8eed866e532dff6f68c9e68494800448831a62f66b29451999db0", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19501", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19501" ], "server_argv_sha256": "754168bed1a90cbefd86aceadedfca263a50fe1f939cad35cf240a0cbf44aedb", "server_command_content_sha256": "d7ed9889387ecc257a9c336e197c4bdcb14456bde10be54f15eb916c6171ac6f", "server_command_sha256": "d7ed9889387ecc257a9c336e197c4bdcb14456bde10be54f15eb916c6171ac6f", "server_parallel": 1 } }, "q4-kleidiai-tuned/p2": { "content_sha256": "6422ad9f4b512ea99fa8a5e734953f6d828fe1c09db33bd7602130d5973c74a8", "input_sha256": "6422ad9f4b512ea99fa8a5e734953f6d828fe1c09db33bd7602130d5973c74a8", "wrapper": { "candidate_id": "q4-kleidiai-tuned", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "latency": { "e2e_latency_ms_p50": 2324.5398604999536, "e2e_latency_ms_p95": 2363.2141979999233, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2309.157314999993, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 468.74967700000525 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2290.584598000123, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 467.81687399993643 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2310.5015860001004, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 468.34895300003154 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2319.3463360000806, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 466.687211000135 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2329.7333849998267, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 471.66174699987096 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2349.281459999929, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 472.7049440000428 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2361.3177139998243, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 470.47444099985114 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2316.0353690000193, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 470.4376150000371 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2346.874693000018, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 468.17259900012687 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2363.2141979999233, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 468.75162299988915 } ], "ttft_ms_p50": 468.7506499999472, "ttft_ms_p95": 472.7049440000428 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PARETOPILOT", "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "PARETOPILOT" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-10" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "```json\n[\"red\",\"blue\"]\n```" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 20, "score": 0.8333333333333334, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "dd0a48ef8646b10ae4bdac03644c008b7aa4f7a8b4b05a518118c6097baa6e31", "evaluation_sha256": "dd0a48ef8646b10ae4bdac03644c008b7aa4f7a8b4b05a518118c6097baa6e31", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19502", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19502" ], "server_argv_sha256": "4b5742caa8b37049f79acff3cb2631e614212170ee38dd3e82ba3879a5a86ec4", "server_command_content_sha256": "d9b941964de12bed637665e919db608767a3f4fd4e30b908a3531c6733f8f37b", "server_command_sha256": "d9b941964de12bed637665e919db608767a3f4fd4e30b908a3531c6733f8f37b", "server_parallel": 2 } }, "q4-kleidiai-tuned/p4": { "content_sha256": "01215f03699eca7b365e16d1d7c8c8afab3d43dda76cc445cb34b4b0fa533e62", "input_sha256": "01215f03699eca7b365e16d1d7c8c8afab3d43dda76cc445cb34b4b0fa533e62", "wrapper": { "candidate_id": "q4-kleidiai-tuned", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q4-kleidiai-tuned", "latency": { "e2e_latency_ms_p50": 2302.2443255000553, "e2e_latency_ms_p95": 2362.941810999928, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2286.886592999963, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 466.94577899984324 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2330.1410999999916, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 468.5459600000286 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2304.440586000055, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 468.40901199993823 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2302.274574999956, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 484.8733030000858 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2298.994537999988, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 469.9137539998901 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2314.2923499999597, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 467.4998620000679 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2302.214076000155, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 467.54351799995675 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2362.941810999928, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 467.13314299995545 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2291.8238130000645, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 467.3374210001384 }, { "content": " Provide an example of how ParetoPilot can be used to improve the quality of a software project. Answer according to: ParetoPilot evaluates several inference configurations on one controlled Arm64 host. It records raw samples, validates provenance, applies a declared quality floor and resource ceilings, computes the non-dominated candidates", "e2e_latency_ms": 2292.8306569999677, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 466.7921099999148 } ], "ttft_ms_p50": 467.5216900000123, "ttft_ms_p95": 484.8733030000858 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PARETOPILOT", "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "PARETOPILOT" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-10" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "```json\n[\"red\",\"blue\"]\n```" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 20, "score": 0.8333333333333334, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "043880d9e6f96691aec26c408cf56b0a0bbb6382700ebf33e1cc27ce924f5942", "evaluation_sha256": "043880d9e6f96691aec26c408cf56b0a0bbb6382700ebf33e1cc27ce924f5942", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19504", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/kleidiai/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q4_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "512", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19504" ], "server_argv_sha256": "d981a1509a74718c3e9447e44739f00b3ae07d08f269519b6369f14f32b5c2a2", "server_command_content_sha256": "1f0532f4ab39c930fdaae6a619413c0b54d363188b1cbea587b1c199c3af76b8", "server_command_sha256": "1f0532f4ab39c930fdaae6a619413c0b54d363188b1cbea587b1c199c3af76b8", "server_parallel": 4 } }, "q8-generic/p1": { "content_sha256": "911751fe1af5022efd27520b2b8fb6f599156784d32e1f40765e6adefe8de23c", "input_sha256": "911751fe1af5022efd27520b2b8fb6f599156784d32e1f40765e6adefe8de23c", "wrapper": { "candidate_id": "q8-generic", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q8-generic", "latency": { "e2e_latency_ms_p50": 2190.828786499992, "e2e_latency_ms_p95": 2209.183042000177, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2187.2053990000495, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 543.8799670000662 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2186.0137449998547, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 536.8174779998753 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2209.183042000177, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 539.32398500001 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2179.4311090000065, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 537.973010000087 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2191.1862229999315, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 538.5555459999978 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2190.4713500000526, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 536.616277000121 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2181.5622460001123, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 541.7248060000475 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2199.110598000061, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 537.2670960000505 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2195.280896000213, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 539.1864300001998 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2201.74791799991, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 538.8046359998953 } ], "ttft_ms_p50": 538.6800909999465, "ttft_ms_p95": 543.8799670000662 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "ParetoPilot" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "2026-08-14", "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-14" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": true, "matched_answer": "[\"red\",\"blue\"]", "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "[\"red\",\"blue\"]" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 21, "score": 0.875, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "f45adb5517dd28e1c4520eb3da44774e38ae7afd5532fc875479125700405003", "evaluation_sha256": "f45adb5517dd28e1c4520eb3da44774e38ae7afd5532fc875479125700405003", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19401", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "2048", "--parallel", "1", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19401" ], "server_argv_sha256": "f81f8a99e15d7637097de954ca5b26da5b6403ee1ad440f3f9c0043433c77189", "server_command_content_sha256": "281cdbb730fa1326acf5e45b3ad080937d29d7b17d8959626f2b01a29d5c981a", "server_command_sha256": "281cdbb730fa1326acf5e45b3ad080937d29d7b17d8959626f2b01a29d5c981a", "server_parallel": 1 } }, "q8-generic/p2": { "content_sha256": "372e61aa64b12a72ff7a4d8a9c9da57a61fdd1a673c8dd8cea54e5e5387e5596", "input_sha256": "372e61aa64b12a72ff7a4d8a9c9da57a61fdd1a673c8dd8cea54e5e5387e5596", "wrapper": { "candidate_id": "q8-generic", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q8-generic", "latency": { "e2e_latency_ms_p50": 2205.5115404999697, "e2e_latency_ms_p95": 2224.136187999875, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2185.083521000024, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 540.6781350000074 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2196.1339370000132, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 541.9278560000294 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2195.3055930000573, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 539.018249000037 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2221.89658100001, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 538.5640130000411 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2224.136187999875, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 541.8555379999361 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2217.392590000145, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 540.2288510001654 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2203.55001300004, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 538.6561059999622 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2204.9770529999932, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 542.5363279998692 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2206.046027999946, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 539.3560539998816 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2218.8801010001953, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 539.865114000122 } ], "ttft_ms_p50": 540.0469825001437, "ttft_ms_p95": 542.5363279998692 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "ParetoPilot" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "2026-08-14", "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-14" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": true, "matched_answer": "[\"red\",\"blue\"]", "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "[\"red\",\"blue\"]" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 21, "score": 0.875, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "05c8aa9349890b232019b6be1780d8ea1efbff455d1c5c7a1ffb14d6b3a4be27", "evaluation_sha256": "05c8aa9349890b232019b6be1780d8ea1efbff455d1c5c7a1ffb14d6b3a4be27", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19402", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "4096", "--parallel", "2", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19402" ], "server_argv_sha256": "2091ce9aa41d46c51e873233961b6fae0d7b57c77f2119323bc8f4a1f54a0262", "server_command_content_sha256": "b0ab43daf774ce84a9079dcd80f1207215303ce7901913c99183deeacf51de7f", "server_command_sha256": "b0ab43daf774ce84a9079dcd80f1207215303ce7901913c99183deeacf51de7f", "server_parallel": 2 } }, "q8-generic/p4": { "content_sha256": "e12f521247d9fcac0b5c64e8b947fbc271e80b0740c9ed233a54f474502c3aee", "input_sha256": "e12f521247d9fcac0b5c64e8b947fbc271e80b0740c9ed233a54f474502c3aee", "wrapper": { "candidate_id": "q8-generic", "classification": "capacity-quality-binding", "evaluation": { "candidate_id": "q8-generic", "latency": { "e2e_latency_ms_p50": 2200.413933499931, "e2e_latency_ms_p95": 2210.2109580000615, "method": "single-client streamed HTTP requests", "samples": [ { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2201.750858999958, "event_count": 65, "index": 1, "predicted_tokens": 64, "ttft_ms": 539.7907530000339 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2192.615439999827, "event_count": 65, "index": 2, "predicted_tokens": 64, "ttft_ms": 538.1468900000073 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2210.2109580000615, "event_count": 65, "index": 3, "predicted_tokens": 64, "ttft_ms": 549.4804280001517 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2199.8560299998644, "event_count": 65, "index": 4, "predicted_tokens": 64, "ttft_ms": 540.262852000069 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2204.019750000043, "event_count": 65, "index": 5, "predicted_tokens": 64, "ttft_ms": 538.0112349998853 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2200.9718369999973, "event_count": 65, "index": 6, "predicted_tokens": 64, "ttft_ms": 537.6822880000418 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2204.8909069999354, "event_count": 65, "index": 7, "predicted_tokens": 64, "ttft_ms": 538.211826999941 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2180.92475200001, "event_count": 65, "index": 8, "predicted_tokens": 64, "ttft_ms": 536.9920970001658 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2175.7253399998717, "event_count": 65, "index": 9, "predicted_tokens": 64, "ttft_ms": 536.7722109999704 }, { "content": " Provide a brief example of how ParetoPilot can be used to optimize a software deployment process. \n\n### Value of Transparent, Reproducible Optimization Evidence for an Engineering Team\n\nTransparent, reproducible optimization evidence is invaluable for an engineering team for several reasons:\n\n1. **Trust and Reliability**: When teams can", "e2e_latency_ms": 2186.351851999916, "event_count": 65, "index": 10, "predicted_tokens": 64, "ttft_ms": 540.9359929999482 } ], "ttft_ms_p50": 538.1793584999741, "ttft_ms_p95": 549.4804280001517 }, "quality": { "cases": [ { "accepted_answers": [ "PARETOPILOT" ], "id": "instruction-single-word", "match_mode": "trimmed-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this single word and no punctuation: PARETOPILOT", "response": "ParetoPilot" }, { "accepted_answers": [ "BLUE" ], "id": "instruction-color", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "BLUE", "prompt": "Reply with only the uppercase word BLUE.", "response": "BLUE" }, { "accepted_answers": [ "ARM READY" ], "id": "instruction-two-words", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ARM READY", "prompt": "Reply with exactly these two words and nothing else: ARM READY", "response": "ARM READY" }, { "accepted_answers": [ "A64-2026" ], "id": "instruction-code", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "A64-2026", "prompt": "Reply with only this identifier: A64-2026", "response": "A64-2026" }, { "accepted_answers": [ "PP-104" ], "id": "extraction-ticket", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "PP-104", "prompt": "Extract the ticket identifier and reply with only it. Text: The deployment review for PP-104 begins tomorrow.", "response": "PP-104" }, { "accepted_answers": [ "Maya Chen" ], "id": "extraction-owner", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Maya Chen", "prompt": "Extract the owner's full name and reply with only it. Text: Service: inference-api; Owner: Maya Chen; Region: central.", "response": "Maya Chen" }, { "accepted_answers": [ "37" ], "id": "extraction-count", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "37", "prompt": "Extract the retry count and reply with only the integer. Text: workers=4, retries=37, timeout=90.", "response": "37" }, { "accepted_answers": [ "2026-08-14" ], "id": "extraction-date", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "2026-08-14", "prompt": "Extract the deadline and reply with only the ISO date. Text: Draft due 2026-08-10; final deadline 2026-08-14.", "response": "2026-08-14" }, { "accepted_answers": [ "POSITIVE" ], "id": "classification-sentiment", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "POSITIVE", "prompt": "Classify the sentiment as POSITIVE, NEGATIVE, or NEUTRAL. Reply with only the label. Text: The optimized server is faster and uses less memory.", "response": "POSITIVE" }, { "accepted_answers": [ "SECURITY" ], "id": "classification-category", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "SECURITY", "prompt": "Classify this issue as PERFORMANCE, SECURITY, or DOCUMENTATION. Reply with only the label. Text: An unauthenticated endpoint exposes private records.", "response": "SECURITY" }, { "accepted_answers": [ "ODD" ], "id": "classification-parity", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "ODD", "prompt": "Classify 57 as ODD or EVEN. Reply with only the label.", "response": "ODD" }, { "accepted_answers": [ "MAMMAL" ], "id": "classification-animal", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "MAMMAL", "prompt": "Classify a dolphin as MAMMAL, BIRD, or FISH. Reply with only the label.", "response": "MAMMAL" }, { "accepted_answers": [ "42" ], "id": "arithmetic-addition", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "42", "prompt": "Reply with only the integer result of 17 plus 25.", "response": "42" }, { "accepted_answers": [ "63" ], "id": "arithmetic-multiplication", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "63", "prompt": "Reply with only the integer result of 9 multiplied by 7.", "response": "63" }, { "accepted_answers": [ "12" ], "id": "arithmetic-division", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "12", "prompt": "Reply with only the integer result of 144 divided by 12.", "response": "12" }, { "accepted_answers": [ "30" ], "id": "arithmetic-percent", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "30", "prompt": "Reply with only the integer that is 15 percent of 200.", "response": "30" }, { "accepted_answers": [ "{\"status\":\"ready\"}" ], "id": "json-status", "match_mode": "json-exact", "matched": true, "matched_answer": "{\"status\":\"ready\"}", "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"status\":\"ready\"}", "response": "{\"status\":\"ready\"}" }, { "accepted_answers": [ "{\"arm64\":true,\"workers\":4}" ], "id": "json-workers", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"arm64\":true,\"workers\":4}", "response": "```json\n{\"arm64\":true,\"workers\":4}\n```" }, { "accepted_answers": [ "[\"red\",\"blue\"]" ], "id": "json-array", "match_mode": "json-exact", "matched": true, "matched_answer": "[\"red\",\"blue\"]", "prompt": "Reply with exactly this one-line JSON array and no code fence: [\"red\",\"blue\"]", "response": "[\"red\",\"blue\"]" }, { "accepted_answers": [ "{\"result\":42,\"unit\":\"ms\"}" ], "id": "json-measurement", "match_mode": "json-exact", "matched": false, "matched_answer": null, "prompt": "Reply with exactly this one-line JSON object and no code fence: {\"result\":42,\"unit\":\"ms\"}", "response": "```json\n{\"result\":42,\"unit\":\"ms\"}\n```" }, { "accepted_answers": [ "Tokyo" ], "id": "fact-capital", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Tokyo", "prompt": "Reply with only the city name: What is the capital of Japan?", "response": "Tokyo" }, { "accepted_answers": [ "0" ], "id": "fact-freezing-point", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "0", "prompt": "At standard pressure, at what Celsius temperature does water freeze? Reply with only the integer.", "response": "0" }, { "accepted_answers": [ "Jupiter" ], "id": "fact-planet", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Jupiter", "prompt": "Reply with only the planet name: What is the largest planet in the Solar System?", "response": "Jupiter" }, { "accepted_answers": [ "Au" ], "id": "fact-chemical-symbol", "match_mode": "trimmed-exact", "matched": true, "matched_answer": "Au", "prompt": "Reply with only the chemical symbol for gold.", "response": "Au" } ], "method": "fixed exact-answer smoke evaluation", "passed": 21, "score": 0.875, "total": 24 }, "schema_version": "1.0", "suite": { "cache_prompt": false, "generation_tokens": 64, "id": "paretopilot-qwen-behavior-v2", "license": "CC0-1.0 (original ParetoPilot prompts)", "performance_repetitions": 10, "performance_warmups": 1, "quality_case_count": 24, "seed": 4242, "sha256": "e49c16fba32fd65c947264aef4141026ab68b1fd415ef09eeea6e8ade9a545c7", "temperature": 0 }, "synthetic": false }, "evaluation_content_sha256": "ac6e974334e2222dc27fbd17a60fc8ed35bfed96c3c98ce41a8e1e5148ee1a3a", "evaluation_sha256": "ac6e974334e2222dc27fbd17a60fc8ed35bfed96c3c98ce41a8e1e5148ee1a3a", "pass_id": "quality", "request_base_url": "http://127.0.0.1:19404", "run_attempt": 1, "run_id": "30144901854", "schema_version": "1.0", "server_argv": [ "./.candidate-build/generic/bin/llama-server", "--model", "./.candidate-models/qwen2.5-1.5b-instruct-q8_0.gguf", "--threads", "4", "--threads-batch", "4", "--batch-size", "512", "--ubatch-size", "128", "--ctx-size", "8192", "--parallel", "4", "--n-gpu-layers", "0", "-lv", "4", "--host", "127.0.0.1", "--port", "19404" ], "server_argv_sha256": "d639cacc3e0b7af0ee2ee7e3e83e2607a432f5f3749d59bb0ea6ed7077ad37f0", "server_command_content_sha256": "32fc0bd6e5f77c3d22b80768c3e3eee537f1e11c7f19579d7ab8307a3f542e91", "server_command_sha256": "32fc0bd6e5f77c3d22b80768c3e3eee537f1e11c7f19579d7ab8307a3f542e91", "server_parallel": 4 } } } }, "synthetic": false }