|
| 1 | +"""Keep current-facing performance figures aligned with the committed results.""" |
| 2 | + |
| 3 | +import json |
| 4 | +from pathlib import Path |
| 5 | + |
| 6 | +ROOT = Path(__file__).resolve().parents[1] |
| 7 | +RESULTS = ROOT / "benchmarks" / "results" |
| 8 | +HISTORICAL_REVISION = "49daed6609cb3da142d1a0c88e538dc07f00d974" |
| 9 | + |
| 10 | + |
| 11 | +def metric(model: str, questions: int, key: str) -> float: |
| 12 | + report = json.loads((RESULTS / f"{model}-mlx-float16.json").read_text()) |
| 13 | + row = next( |
| 14 | + result |
| 15 | + for result in report["results"] |
| 16 | + if result["workload"] == "short" and result["questions"] == questions |
| 17 | + ) |
| 18 | + return row["end_to_end"][key] |
| 19 | + |
| 20 | + |
| 21 | +def test_current_readmes_match_committed_benchmarks() -> None: |
| 22 | + english = ROOT.joinpath("README.md").read_text() |
| 23 | + chinese = ROOT.joinpath("README.zh-CN.md").read_text() |
| 24 | + laya = "laya" |
| 25 | + multilingual = "laya-multilingual" |
| 26 | + laya_p50 = f"{metric(laya, 1, 'p50_ms'):.2f}" |
| 27 | + multilingual_p50 = f"{metric(multilingual, 1, 'p50_ms'):.2f}" |
| 28 | + run_date = json.loads((RESULTS / "laya-mlx-float16.json").read_text())["created_at"].split("T")[ |
| 29 | + 0 |
| 30 | + ] |
| 31 | + |
| 32 | + assert ( |
| 33 | + f"**{laya_p50} ms** median end-to-end for a short English typed decision. " |
| 34 | + f"**{multilingual_p50} ms** with the multilingual checkpoint." |
| 35 | + in "\n".join(english.splitlines()[:8]) |
| 36 | + ) |
| 37 | + assert ( |
| 38 | + f"单个短问题端到端中位耗时 **{laya_p50} ms**；multilingual 检查点为 " |
| 39 | + f"**{multilingual_p50} ms**。" in "\n".join(chinese.splitlines()[:8]) |
| 40 | + ) |
| 41 | + assert f"上面的 {laya_p50} / {multilingual_p50} ms 来自" in chinese |
| 42 | + assert f"committed {run_date} benchmark run" in english |
| 43 | + assert f"{run_date} 的基准测试" in chinese |
| 44 | + |
| 45 | + for text, labels in ( |
| 46 | + (english, ("One short question, P50", "One short question, P95", "50-question throughput")), |
| 47 | + (chinese, ("单个短问题 P50", "单个短问题 P95", "50 问题吞吐量")), |
| 48 | + ): |
| 49 | + for label, questions, key, precision, unit in ( |
| 50 | + (labels[0], 1, "p50_ms", 2, "ms"), |
| 51 | + (labels[1], 1, "p95_ms", 2, "ms"), |
| 52 | + (labels[2], 50, "questions_per_second", 1, "q/s"), |
| 53 | + ): |
| 54 | + expected = ( |
| 55 | + f"| {label} | **{metric(laya, questions, key):.{precision}f} {unit}**" |
| 56 | + f" | **{metric(multilingual, questions, key):.{precision}f} {unit}** |" |
| 57 | + ) |
| 58 | + assert expected in text |
| 59 | + |
| 60 | + |
| 61 | +def test_release_copy_matches_committed_multilingual_latency() -> None: |
| 62 | + text = ROOT.joinpath("docs", "LAUNCH.md").read_text() |
| 63 | + latency = f"{metric('laya-multilingual', 1, 'p50_ms'):.2f} ms" |
| 64 | + run_date = json.loads((RESULTS / "laya-multilingual-mlx-float16.json").read_text())[ |
| 65 | + "created_at" |
| 66 | + ].split("T")[0] |
| 67 | + assert f"One-question API benchmark: {latency} p50" in text |
| 68 | + assert f"单问题基准 P50 为 {latency}" in text |
| 69 | + assert f"The {latency} headline describes" in text |
| 70 | + assert f"committed {run_date} run" in text |
| 71 | + |
| 72 | + |
| 73 | +def test_historical_research_links_keep_the_original_results() -> None: |
| 74 | + reports = { |
| 75 | + "PERFORMANCE_RESEARCH.md": ( |
| 76 | + "laya-mlx-float16", |
| 77 | + "laya-mlx-float32", |
| 78 | + "laya-torch-mps-float32", |
| 79 | + "laya-multilingual-mlx-float16", |
| 80 | + "laya-multilingual-mlx-float32", |
| 81 | + "laya-multilingual-torch-mps-float32", |
| 82 | + ), |
| 83 | + "MATH_10X_RESEARCH.md": ( |
| 84 | + "laya-mlx-float16", |
| 85 | + "laya-multilingual-mlx-float16", |
| 86 | + "laya-typed-decisions-mlx-float16", |
| 87 | + ), |
| 88 | + } |
| 89 | + for report, sources in reports.items(): |
| 90 | + text = ROOT.joinpath("docs", report).read_text() |
| 91 | + assert "`49daed6`" in text |
| 92 | + for source in sources: |
| 93 | + assert ( |
| 94 | + "https://github.com/mizorewww/laya-mlx/blob/" |
| 95 | + f"{HISTORICAL_REVISION}/benchmarks/results/{source}.json" |
| 96 | + ) in text |
| 97 | + assert f"(../benchmarks/results/{source}.json)" not in text |
0 commit comments