{ "meta": { "repo": "arx", "demo": "leaderboard-audit", "commit": "a2fb144", "worktree_dirty": false, "dirty_paths": [], "generated_at": "2026-10-22T07:54:30+00:00", "mode": "offline-numpy", "seed": 0, "python": "3.12.3", "numpy": "2.5.2", "note": "合成案例在构建时由 arx.audit 重算；报告 HTML 由 arx.audit.report_html 生成。" }, "summary": { "cases": 8, "verdicts": { "READABLE": 1, "MARGINAL": 1, "UNREADABLE": 1, "UNDECIDABLE": 5 } }, "cases": [ { "id": "readable", "title": "五名生成者，领先者高出噪声地板", "question": "这份榜面能分辨 strong 与 weak 的差距吗？", "verdict": "READABLE", "meaning": "差距大于测量误差。它不表示这个指标本身是对的。", "reasons": [], "reason_codes": [], "metrics": [ { "label": "噪声地板", "value": "0.9210", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "strong = 0.9600", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "+0.0390", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "0.0274", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "0.0274", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": { "above_chance": true, "z": 16.839999999999996, "critical_z": 1.9599639845400536, "n_items_needed": 6, "source": "AuditReport.extra.floor_vs_chance" }, "notes": [], "board": { "n_generators": 5, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "27602db8d518e5a1" }, "report": { "verdict": "READABLE", "reasons": [], "leader_system": "strong", "leader_score": 0.96, "floor_estimate": 0.9209999999999999, "margin": 0.039000000000000035, "delta_declared": 0.02, "mde": 0.027449817021202763, "threshold": 0.027449817021202763, "n_items": 400, "n_floor_items": 400, "n_generators": 5, "n_contestants": 2, "floor_method": "loo", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "per_generator_floor": [ 0.92, 0.9325, 0.92, 0.9025, 0.93 ], "chance_level": 0.5, "floor_vs_chance": { "z": 16.839999999999996, "critical_z": 1.9599639845400536, "se": 0.025, "n_items": 400, "n_items_needed": 6, "note": "6 items would put this floor above chance at alpha=0.05", "above_chance": true }, "mde_method": "binomial_approx" } }, "report_html": "\n\n\n\n\n五名生成者，领先者高出噪声地板 — READABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

五名生成者，领先者高出噪声地板

\n
READABLE
\n

The gap is larger than the measurement error. It does not mean the metric is the right one.

\n\n
\n
Leader
strong
score=0.9600
\n
Noise floor
0.9210
loo
\n
Margin over floor
0.0390
\n
Resolution bound
0.0274
declared delta=0.0200; MDE=0.0274
\n
Items
400
floor on 400
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "per_generator_floor": [\n    0.92,\n    0.9325,\n    0.92,\n    0.9025,\n    0.93\n  ],\n  "chance_level": 0.5,\n  "floor_vs_chance": {\n    "z": 16.839999999999996,\n    "critical_z": 1.9599639845400536,\n    "se": 0.025,\n    "n_items": 400,\n    "n_items_needed": 6,\n    "note": "6 items would put this floor above chance at alpha=0.05",\n    "above_chance": true\n  },\n  "mde_method": "binomial_approx"\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "READABLE",\n  "reasons": [],\n  "leader_system": "strong",\n  "leader_score": 0.96,\n  "floor_estimate": 0.9209999999999999,\n  "margin": 0.039000000000000035,\n  "delta_declared": 0.02,\n  "mde": 0.027449817021202763,\n  "threshold": 0.027449817021202763,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "loo",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "per_generator_floor": [\n      0.92,\n      0.9325,\n      0.92,\n      0.9025,\n      0.93\n    ],\n    "chance_level": 0.5,\n    "floor_vs_chance": {\n      "z": 16.839999999999996,\n      "critical_z": 1.9599639845400536,\n      "se": 0.025,\n      "n_items": 400,\n      "n_items_needed": 6,\n      "note": "6 items would put this floor above chance at alpha=0.05",\n      "above_chance": true\n    },\n    "mde_method": "binomial_approx"\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "marginal", "title": "同样的榜面，只有 120 个 item", "question": "差距没变，但样本分辨率变了吗？", "verdict": "MARGINAL", "meaning": "差距为正，但落在当前样本能分辨的范围内；需要更多 item。", "reasons": [ "margin +0.0267 is inside the resolution bound 0.0501" ], "reason_codes": [ "margin +0.0267 is inside the resolution bound 0.0501" ], "metrics": [ { "label": "噪声地板", "value": "0.9333", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "strong = 0.9600", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "+0.0267", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "0.0501", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "0.0501", "source": "AuditReport.mde" }, { "label": "items", "value": 120, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": { "above_chance": true, "z": 9.49385766342288, "critical_z": 1.9599639845400536, "n_items_needed": 6, "source": "AuditReport.extra.floor_vs_chance" }, "notes": [], "board": { "n_generators": 5, "n_items": 120, "agreement": 0.93, "seed": 3, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "0a62465bf19fb420" }, "report": { "verdict": "MARGINAL", "reasons": [ "margin +0.0267 is inside the resolution bound 0.0501" ], "leader_system": "strong", "leader_score": 0.96, "floor_estimate": 0.9333333333333333, "margin": 0.026666666666666616, "delta_declared": 0.02, "mde": 0.050116279939673394, "threshold": 0.050116279939673394, "n_items": 120, "n_floor_items": 120, "n_generators": 5, "n_contestants": 2, "floor_method": "loo", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "per_generator_floor": [ 0.9333, 0.95, 0.9167, 0.9333, 0.9333 ], "chance_level": 0.5, "floor_vs_chance": { "z": 9.49385766342288, "critical_z": 1.9599639845400536, "se": 0.04564354645876384, "n_items": 120, "n_items_needed": 6, "note": "6 items would put this floor above chance at alpha=0.05", "above_chance": true }, "mde_method": "binomial_approx" } }, "report_html": "\n\n\n\n\n同样的榜面，只有 120 个 item — MARGINAL\n\n\n\n
\n
arx / benchmark readability audit
\n

同样的榜面，只有 120 个 item

\n
MARGINAL
\n

The gap is positive but inside what this sample can resolve. Collect more items before claiming a difference.

\n\n
\n
Leader
strong
score=0.9600
\n
Noise floor
0.9333
loo
\n
Margin over floor
0.0267
\n
Resolution bound
0.0501
declared delta=0.0200; MDE=0.0501
\n
Items
120
floor on 120
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "per_generator_floor": [\n    0.9333,\n    0.95,\n    0.9167,\n    0.9333,\n    0.9333\n  ],\n  "chance_level": 0.5,\n  "floor_vs_chance": {\n    "z": 9.49385766342288,\n    "critical_z": 1.9599639845400536,\n    "se": 0.04564354645876384,\n    "n_items": 120,\n    "n_items_needed": 6,\n    "note": "6 items would put this floor above chance at alpha=0.05",\n    "above_chance": true\n  },\n  "mde_method": "binomial_approx"\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "MARGINAL",\n  "reasons": [\n    "margin +0.0267 is inside the resolution bound 0.0501"\n  ],\n  "leader_system": "strong",\n  "leader_score": 0.96,\n  "floor_estimate": 0.9333333333333333,\n  "margin": 0.026666666666666616,\n  "delta_declared": 0.02,\n  "mde": 0.050116279939673394,\n  "threshold": 0.050116279939673394,\n  "n_items": 120,\n  "n_floor_items": 120,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "loo",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "per_generator_floor": [\n      0.9333,\n      0.95,\n      0.9167,\n      0.9333,\n      0.9333\n    ],\n    "chance_level": 0.5,\n    "floor_vs_chance": {\n      "z": 9.49385766342288,\n      "critical_z": 1.9599639845400536,\n      "se": 0.04564354645876384,\n      "n_items": 120,\n      "n_items_needed": 6,\n      "note": "6 items would put this floor above chance at alpha=0.05",\n      "above_chance": true\n    },\n    "mde_method": "binomial_approx"\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "unreadable", "title": "领先者没有清过噪声地板", "question": "strong=0.86 真的高于这些标签自身的复现率吗？", "verdict": "UNREADABLE", "meaning": "领先者没有清过噪声地板；这份测量不支持这个排名。", "reasons": [ "leader strong (0.8600) is at or below the measured noise floor (0.9210)" ], "reason_codes": [ "leader strong (0.8600) is at or below the measured noise floor (0.9210)" ], "metrics": [ { "label": "噪声地板", "value": "0.9210", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "strong = 0.8600", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "-0.0610", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "0.0486", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "0.0486", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": { "above_chance": true, "z": 16.839999999999996, "critical_z": 1.9599639845400536, "n_items_needed": 6, "source": "AuditReport.extra.floor_vs_chance" }, "notes": [], "board": { "n_generators": 5, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.86, "weak": 0.82 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "27602db8d518e5a1" }, "report": { "verdict": "UNREADABLE", "reasons": [ "leader strong (0.8600) is at or below the measured noise floor (0.9210)" ], "leader_system": "strong", "leader_score": 0.86, "floor_estimate": 0.9209999999999999, "margin": -0.06099999999999994, "delta_declared": 0.02, "mde": 0.048605686910474524, "threshold": 0.048605686910474524, "n_items": 400, "n_floor_items": 400, "n_generators": 5, "n_contestants": 2, "floor_method": "loo", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "per_generator_floor": [ 0.92, 0.9325, 0.92, 0.9025, 0.93 ], "chance_level": 0.5, "floor_vs_chance": { "z": 16.839999999999996, "critical_z": 1.9599639845400536, "se": 0.025, "n_items": 400, "n_items_needed": 6, "note": "6 items would put this floor above chance at alpha=0.05", "above_chance": true }, "mde_method": "binomial_approx" } }, "report_html": "\n\n\n\n\n领先者没有清过噪声地板 — UNREADABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

领先者没有清过噪声地板

\n
UNREADABLE
\n

The leader does not clear the noise floor. The ranking is not supported by this measurement.

\n\n
\n
Leader
strong
score=0.8600
\n
Noise floor
0.9210
loo
\n
Margin over floor
-0.0610
\n
Resolution bound
0.0486
declared delta=0.0200; MDE=0.0486
\n
Items
400
floor on 400
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "per_generator_floor": [\n    0.92,\n    0.9325,\n    0.92,\n    0.9025,\n    0.93\n  ],\n  "chance_level": 0.5,\n  "floor_vs_chance": {\n    "z": 16.839999999999996,\n    "critical_z": 1.9599639845400536,\n    "se": 0.025,\n    "n_items": 400,\n    "n_items_needed": 6,\n    "note": "6 items would put this floor above chance at alpha=0.05",\n    "above_chance": true\n  },\n  "mde_method": "binomial_approx"\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNREADABLE",\n  "reasons": [\n    "leader strong (0.8600) is at or below the measured noise floor (0.9210)"\n  ],\n  "leader_system": "strong",\n  "leader_score": 0.86,\n  "floor_estimate": 0.9209999999999999,\n  "margin": -0.06099999999999994,\n  "delta_declared": 0.02,\n  "mde": 0.048605686910474524,\n  "threshold": 0.048605686910474524,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "loo",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "per_generator_floor": [\n      0.92,\n      0.9325,\n      0.92,\n      0.9025,\n      0.93\n    ],\n    "chance_level": 0.5,\n    "floor_vs_chance": {\n      "z": 16.839999999999996,\n      "critical_z": 1.9599639845400536,\n      "se": 0.025,\n      "n_items": 400,\n      "n_items_needed": 6,\n      "note": "6 items would put this floor above chance at alpha=0.05",\n      "above_chance": true\n    },\n    "mde_method": "binomial_approx"\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "two_generators", "title": "只有两名生成者", "question": "去掉一个评审者后，地板还能估计吗？", "verdict": "UNDECIDABLE", "meaning": "前提不成立，无法估计地板。这是一个结论，不是分析失败。", "reasons": [ "地板退化：没有可估计的独立复现地板，或地板未能高于随机水平。" ], "reason_codes": [ "floor_degenerate" ], "metrics": [ { "label": "噪声地板", "value": "—", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "—", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "—", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "—", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "—", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 2, "source": "AuditReport.n_generators" } ], "chance": null, "notes": [], "board": { "n_generators": 2, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "050ef5cccc5d7301" }, "report": { "verdict": "UNDECIDABLE", "reasons": [ "floor_degenerate" ], "leader_system": null, "leader_score": null, "floor_estimate": null, "margin": null, "delta_declared": 0.02, "mde": null, "threshold": null, "n_items": 400, "n_floor_items": 400, "n_generators": 2, "n_contestants": 2, "floor_method": "", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "note": "with two generators, leave-one-out leaves one, so the floor degenerates into pairwise agreement; it cannot be estimated" } }, "report_html": "\n\n\n\n\n只有两名生成者 — UNDECIDABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

只有两名生成者

\n
UNDECIDABLE
\n

The premises are violated; no floor is estimable. This is a finding, not a failure of the analysis.

\n\n
\n
Leader
—
score=—
\n
Noise floor
—
unknown method
\n
Margin over floor
—
\n
Resolution bound
—
declared delta=0.0200
\n
Items
400
floor on 400
\n
Gold generators
2
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "note": "with two generators, leave-one-out leaves one, so the floor degenerates into pairwise agreement; it cannot be estimated"\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNDECIDABLE",\n  "reasons": [\n    "floor_degenerate"\n  ],\n  "leader_system": null,\n  "leader_score": null,\n  "floor_estimate": null,\n  "margin": null,\n  "delta_declared": 0.02,\n  "mde": null,\n  "threshold": null,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 2,\n  "n_contestants": 2,\n  "floor_method": "",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "note": "with two generators, leave-one-out leaves one, so the floor degenerates into pairwise agreement; it cannot be estimated"\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "actors_overlap", "title": "评审者同时是参赛者", "question": "如果 strong 也参与了 gold，它是在和自己比吗？", "verdict": "UNDECIDABLE", "meaning": "前提不成立，无法估计地板。这是一个结论，不是分析失败。", "reasons": [ "评审者与参赛者有交集；地板测到的是同伴分布，不是测量误差。" ], "reason_codes": [ "actors_overlap" ], "metrics": [ { "label": "噪声地板", "value": "—", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "—", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "—", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "—", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "—", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": null, "notes": [ "重叠的 actor：strong" ], "board": { "n_generators": 5, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "strong", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "27602db8d518e5a1" }, "report": { "verdict": "UNDECIDABLE", "reasons": [ "actors_overlap" ], "leader_system": null, "leader_score": null, "floor_estimate": null, "margin": null, "delta_declared": 0.02, "mde": null, "threshold": null, "n_items": 400, "n_floor_items": 400, "n_generators": 5, "n_contestants": 2, "floor_method": "", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "overlapping_actors": [ "strong" ] } }, "report_html": "\n\n\n\n\n评审者同时是参赛者 — UNDECIDABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

评审者同时是参赛者

\n
UNDECIDABLE
\n

The premises are violated; no floor is estimable. This is a finding, not a failure of the analysis.

\n\n
\n
Leader
—
score=—
\n
Noise floor
—
unknown method
\n
Margin over floor
—
\n
Resolution bound
—
declared delta=0.0200
\n
Items
400
floor on 400
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "overlapping_actors": [\n    "strong"\n  ]\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNDECIDABLE",\n  "reasons": [\n    "actors_overlap"\n  ],\n  "leader_system": null,\n  "leader_score": null,\n  "floor_estimate": null,\n  "margin": null,\n  "delta_declared": 0.02,\n  "mde": null,\n  "threshold": null,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "overlapping_actors": [\n      "strong"\n    ]\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "noise", "title": "五名生成者只在掷硬币", "question": "地板接近随机时，领先者的大 margin 还意味着什么？", "verdict": "UNDECIDABLE", "meaning": "前提不成立，无法估计地板。这是一个结论，不是分析失败。", "reasons": [ "地板退化：没有可估计的独立复现地板，或地板未能高于随机水平。" ], "reason_codes": [ "floor_degenerate" ], "metrics": [ { "label": "噪声地板", "value": "0.4640", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "—", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "—", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "—", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "—", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": { "above_chance": false, "z": -1.4399999999999968, "critical_z": 1.9599639845400536, "n_items_needed": null, "source": "AuditReport.extra.floor_vs_chance" }, "notes": [ "the measured floor (0.4640) is not detectably above the chance level (0.5000) at n=400 items (z=-1.44 < 1.96): the generators do not demonstrably reproduce anything, so there is no gold for a score to be measured against" ], "board": { "n_generators": 5, "n_items": 400, "agreement": null, "seed": 7, "scores": { "padded": 0.965, "plain": 0.7 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "padded", "plain" ], "note": "pure coin-flip generators; floor is not above chance", "votes_sha256_16": "394cec73e0a84670" }, "report": { "verdict": "UNDECIDABLE", "reasons": [ "floor_degenerate" ], "leader_system": null, "leader_score": null, "floor_estimate": 0.4640000000000001, "margin": null, "delta_declared": 0.02, "mde": null, "threshold": null, "n_items": 400, "n_floor_items": 400, "n_generators": 5, "n_contestants": 2, "floor_method": "loo", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "per_generator_floor": [ 0.46, 0.44, 0.46, 0.49, 0.47 ], "chance_level": 0.5, "floor_vs_chance": { "z": -1.4399999999999968, "critical_z": 1.9599639845400536, "se": 0.025, "n_items": 400, "n_items_needed": null, "note": "the floor (0.4640) is at or below the chance level (0.5000); no sample size makes it informative", "above_chance": false }, "floor_error": "the measured floor (0.4640) is not detectably above the chance level (0.5000) at n=400 items (z=-1.44 < 1.96): the generators do not demonstrably reproduce anything, so there is no gold for a score to be measured against" } }, "report_html": "\n\n\n\n\n五名生成者只在掷硬币 — UNDECIDABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

五名生成者只在掷硬币

\n
UNDECIDABLE
\n

The premises are violated; no floor is estimable. This is a finding, not a failure of the analysis.

\n\n
\n
Leader
—
score=—
\n
Noise floor
0.4640
loo
\n
Margin over floor
—
\n
Resolution bound
—
declared delta=0.0200
\n
Items
400
floor on 400
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "per_generator_floor": [\n    0.46,\n    0.44,\n    0.46,\n    0.49,\n    0.47\n  ],\n  "chance_level": 0.5,\n  "floor_vs_chance": {\n    "z": -1.4399999999999968,\n    "critical_z": 1.9599639845400536,\n    "se": 0.025,\n    "n_items": 400,\n    "n_items_needed": null,\n    "note": "the floor (0.4640) is at or below the chance level (0.5000); no sample size makes it informative",\n    "above_chance": false\n  },\n  "floor_error": "the measured floor (0.4640) is not detectably above the chance level (0.5000) at n=400 items (z=-1.44 < 1.96): the generators do not demonstrably reproduce anything, so there is no gold for a score to be measured against"\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNDECIDABLE",\n  "reasons": [\n    "floor_degenerate"\n  ],\n  "leader_system": null,\n  "leader_score": null,\n  "floor_estimate": 0.4640000000000001,\n  "margin": null,\n  "delta_declared": 0.02,\n  "mde": null,\n  "threshold": null,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "loo",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "per_generator_floor": [\n      0.46,\n      0.44,\n      0.46,\n      0.49,\n      0.47\n    ],\n    "chance_level": 0.5,\n    "floor_vs_chance": {\n      "z": -1.4399999999999968,\n      "critical_z": 1.9599639845400536,\n      "se": 0.025,\n      "n_items": 400,\n      "n_items_needed": null,\n      "note": "the floor (0.4640) is at or below the chance level (0.5000); no sample size makes it informative",\n      "above_chance": false\n    },\n    "floor_error": "the measured floor (0.4640) is not detectably above the chance level (0.5000) at n=400 items (z=-1.44 < 1.96): the generators do not demonstrably reproduce anything, so there is no gold for a score to be measured against"\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "domain_mismatch", "title": "地板与榜面用了不同的 item 集", "question": "两个统计量是在同一批 item 上算的吗？", "verdict": "UNDECIDABLE", "meaning": "前提不成立，无法估计地板。这是一个结论，不是分析失败。", "reasons": [ "地板与榜面用了不同的 item 集；两个统计量不可相减。" ], "reason_codes": [ "domain_mismatch" ], "metrics": [ { "label": "噪声地板", "value": "—", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "—", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "—", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "—", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "—", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 5, "source": "AuditReport.n_generators" } ], "chance": null, "notes": [], "board": { "n_generators": 5, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "g1", "g2", "g3", "g4", "g5" ], "contestant_ids": [ "strong", "weak" ], "leaderboard_items": 150, "floor_items": 400, "votes_sha256_16": "27602db8d518e5a1" }, "report": { "verdict": "UNDECIDABLE", "reasons": [ "domain_mismatch" ], "leader_system": null, "leader_score": null, "floor_estimate": null, "margin": null, "delta_declared": 0.02, "mde": null, "threshold": null, "n_items": 400, "n_floor_items": 400, "n_generators": 5, "n_contestants": 2, "floor_method": "", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8, "leaderboard_items": 150, "floor_items": 400, "items_only_in_leaderboard": 0, "items_only_in_floor": 250 } }, "report_html": "\n\n\n\n\n地板与榜面用了不同的 item 集 — UNDECIDABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

地板与榜面用了不同的 item 集

\n
UNDECIDABLE
\n

The premises are violated; no floor is estimable. This is a finding, not a failure of the analysis.

\n\n
\n
Leader
—
score=—
\n
Noise floor
—
unknown method
\n
Margin over floor
—
\n
Resolution bound
—
declared delta=0.0200
\n
Items
400
floor on 400
\n
Gold generators
5
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8,\n  "leaderboard_items": 150,\n  "floor_items": 400,\n  "items_only_in_leaderboard": 0,\n  "items_only_in_floor": 250\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNDECIDABLE",\n  "reasons": [\n    "domain_mismatch"\n  ],\n  "leader_system": null,\n  "leader_score": null,\n  "floor_estimate": null,\n  "margin": null,\n  "delta_declared": 0.02,\n  "mde": null,\n  "threshold": null,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 5,\n  "n_contestants": 2,\n  "floor_method": "",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8,\n    "leaderboard_items": 150,\n    "floor_items": 400,\n    "items_only_in_leaderboard": 0,\n    "items_only_in_floor": 250\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } }, { "id": "single_generator", "title": "只有一名 gold 生成者", "question": "只有一个人产出的标签，能定义复现地板吗？", "verdict": "UNDECIDABLE", "meaning": "前提不成立，无法估计地板。这是一个结论，不是分析失败。", "reasons": [ "只有一名生成者，无法做 leave-one-out。" ], "reason_codes": [ "single_generator" ], "metrics": [ { "label": "噪声地板", "value": "—", "source": "AuditReport.floor_estimate" }, { "label": "领先者", "value": "—", "source": "AuditReport.leader_system / leader_score" }, { "label": "高出地板", "value": "—", "source": "AuditReport.margin" }, { "label": "分辨率下界", "value": "—", "source": "AuditReport.threshold" }, { "label": "MDE", "value": "—", "source": "AuditReport.mde" }, { "label": "items", "value": 400, "source": "AuditReport.n_items" }, { "label": "generators", "value": 1, "source": "AuditReport.n_generators" } ], "chance": null, "notes": [], "board": { "n_generators": 1, "n_items": 400, "agreement": 0.93, "seed": 0, "scores": { "strong": 0.96, "weak": 0.6 }, "delta_declared": 0.02, "judge_ids": [ "g1" ], "contestant_ids": [ "strong", "weak" ], "votes_sha256_16": "7327ab5da6bdb485" }, "report": { "verdict": "UNDECIDABLE", "reasons": [ "single_generator" ], "leader_system": null, "leader_score": null, "floor_estimate": null, "margin": null, "delta_declared": 0.02, "mde": null, "threshold": null, "n_items": 400, "n_floor_items": 400, "n_generators": 1, "n_contestants": 2, "floor_method": "", "sequential": null, "extra": { "alpha": 0.05, "power": 0.8 } }, "report_html": "\n\n\n\n\n只有一名 gold 生成者 — UNDECIDABLE\n\n\n\n
\n
arx / benchmark readability audit
\n

只有一名 gold 生成者

\n
UNDECIDABLE
\n

The premises are violated; no floor is estimable. This is a finding, not a failure of the analysis.

\n\n
\n
Leader
—
score=—
\n
Noise floor
—
unknown method
\n
Margin over floor
—
\n
Resolution bound
—
declared delta=0.0200
\n
Items
400
floor on 400
\n
Gold generators
1
\n
Contestants
2
\n
\n\n
\n

Why this verdict

\n \n
\n\n \n
Extra diagnostics
{\n  "alpha": 0.05,\n  "power": 0.8\n}
\n\n
\n Reproduce.\n arx audit <leaderboard.json> --html <report.html>\n
The verdict is computed from the input JSON only. No network, no model.
\n
\n\n
\n Raw report\n 
{\n  "verdict": "UNDECIDABLE",\n  "reasons": [\n    "single_generator"\n  ],\n  "leader_system": null,\n  "leader_score": null,\n  "floor_estimate": null,\n  "margin": null,\n  "delta_declared": 0.02,\n  "mde": null,\n  "threshold": null,\n  "n_items": 400,\n  "n_floor_items": 400,\n  "n_generators": 1,\n  "n_contestants": 2,\n  "floor_method": "",\n  "sequential": null,\n  "extra": {\n    "alpha": 0.05,\n    "power": 0.8\n  }\n}
\n
\n\n \n
\n\n\n", "source": { "kind": "recomputed", "audit": "arx.audit.audit_leaderboard", "renderer": "arx.audit.report_html.render_html" } } ], "sensitivity": { "n_items": [ 40, 80, 120, 200, 400 ], "deltas": [ 0.005, 0.01, 0.02, 0.05 ], "rows": [ { "n_items": 40, "cells": [ { "n_items": 40, "delta": 0.005, "verdict": "MARGINAL", "floor": 0.925, "margin": 0.03499999999999992, "threshold": 0.08680394314185923, "mde": 0.08680394314185923 }, { "n_items": 40, "delta": 0.01, "verdict": "MARGINAL", "floor": 0.925, "margin": 0.03499999999999992, "threshold": 0.08680394314185923, "mde": 0.08680394314185923 }, { "n_items": 40, "delta": 0.02, "verdict": "MARGINAL", "floor": 0.925, "margin": 0.03499999999999992, "threshold": 0.08680394314185923, "mde": 0.08680394314185923 }, { "n_items": 40, "delta": 0.05, "verdict": "MARGINAL", "floor": 0.925, "margin": 0.03499999999999992, "threshold": 0.08680394314185923, "mde": 0.08680394314185923 } ] }, { "n_items": 80, "cells": [ { "n_items": 80, "delta": 0.005, "verdict": "MARGINAL", "floor": 0.9024999999999999, "margin": 0.05750000000000011, "threshold": 0.06137965682934016, "mde": 0.06137965682934016 }, { "n_items": 80, "delta": 0.01, "verdict": "MARGINAL", "floor": 0.9024999999999999, "margin": 0.05750000000000011, "threshold": 0.06137965682934016, "mde": 0.06137965682934016 }, { "n_items": 80, "delta": 0.02, "verdict": "MARGINAL", "floor": 0.9024999999999999, "margin": 0.05750000000000011, "threshold": 0.06137965682934016, "mde": 0.06137965682934016 }, { "n_items": 80, "delta": 0.05, "verdict": "MARGINAL", "floor": 0.9024999999999999, "margin": 0.05750000000000011, "threshold": 0.06137965682934016, "mde": 0.06137965682934016 } ] }, { "n_items": 120, "cells": [ { "n_items": 120, "delta": 0.005, "verdict": "MARGINAL", "floor": 0.9116666666666667, "margin": 0.04833333333333323, "threshold": 0.050116279939673394, "mde": 0.050116279939673394 }, { "n_items": 120, "delta": 0.01, "verdict": "MARGINAL", "floor": 0.9116666666666667, "margin": 0.04833333333333323, "threshold": 0.050116279939673394, "mde": 0.050116279939673394 }, { "n_items": 120, "delta": 0.02, "verdict": "MARGINAL", "floor": 0.9116666666666667, "margin": 0.04833333333333323, "threshold": 0.050116279939673394, "mde": 0.050116279939673394 }, { "n_items": 120, "delta": 0.05, "verdict": "MARGINAL", "floor": 0.9116666666666667, "margin": 0.04833333333333323, "threshold": 0.050116279939673394, "mde": 0.050116279939673394 } ] }, { "n_items": 200, "cells": [ { "n_items": 200, "delta": 0.005, "verdict": "READABLE", "floor": 0.915, "margin": 0.04499999999999993, "threshold": 0.03881990351604478, "mde": 0.03881990351604478 }, { "n_items": 200, "delta": 0.01, "verdict": "READABLE", "floor": 0.915, "margin": 0.04499999999999993, "threshold": 0.03881990351604478, "mde": 0.03881990351604478 }, { "n_items": 200, "delta": 0.02, "verdict": "READABLE", "floor": 0.915, "margin": 0.04499999999999993, "threshold": 0.03881990351604478, "mde": 0.03881990351604478 }, { "n_items": 200, "delta": 0.05, "verdict": "MARGINAL", "floor": 0.915, "margin": 0.04499999999999993, "threshold": 0.05, "mde": 0.03881990351604478 } ] }, { "n_items": 400, "cells": [ { "n_items": 400, "delta": 0.005, "verdict": "READABLE", "floor": 0.9209999999999999, "margin": 0.039000000000000035, "threshold": 0.027449817021202763, "mde": 0.027449817021202763 }, { "n_items": 400, "delta": 0.01, "verdict": "READABLE", "floor": 0.9209999999999999, "margin": 0.039000000000000035, "threshold": 0.027449817021202763, "mde": 0.027449817021202763 }, { "n_items": 400, "delta": 0.02, "verdict": "READABLE", "floor": 0.9209999999999999, "margin": 0.039000000000000035, "threshold": 0.027449817021202763, "mde": 0.027449817021202763 }, { "n_items": 400, "delta": 0.05, "verdict": "MARGINAL", "floor": 0.9209999999999999, "margin": 0.039000000000000035, "threshold": 0.05, "mde": 0.027449817021202763 } ] } ], "note": "嵌套样本：同一个 400-item pool 的前 n 个 item；只变 n 与声明的 delta，n_boot=500。" }, "real_case": { "title": "ChaosNLI: same board, two floors", "url": "https://github.com/aidless/arx/blob/main/case-studies/chaosnli-noise-floor/README.md", "source": "case-studies/chaosnli-noise-floor/results_judge_panel.json", "audits": [ { "id": "human", "label": "100 human annotations", "verdict": "READABLE", "floor": 0.7467399999999997, "leader": "albert-xxlarge", "leader_score": 0.789, "margin": 0.0422600000000003, "mde": 0.03212404311266055, "n_items": 1000, "floor_source": "human", "source": "case-studies/chaosnli-noise-floor/results_judge_panel.json#/audits/human" }, { "id": "llm_judge_panel", "label": "32 named LLM judges", "verdict": "UNREADABLE", "floor": 0.86596875, "leader": "albert-xxlarge", "leader_score": 0.789, "margin": -0.07696874999999992, "mde": 0.03212404311266055, "n_items": 1000, "floor_source": "llm_judge_panel", "source": "case-studies/chaosnli-noise-floor/results_judge_panel.json#/audits/llm_judge_panel" } ], "note": "同一榜面，只换地板来源；本页回放 case study 结果，不重算。" }, "case_study": { "title": "ChaosNLI noise floor（真实数据案例）", "url": "https://github.com/aidless/arx/blob/main/case-studies/chaosnli-noise-floor/README.md", "note": "真实数据上的三条审计；本页不重算它，只链接到一手 case study。" }, "boundaries": [ "这些合成案例是合成投票矩阵，用来行使闸的四种判决；不是真实模型排名。", "READABLE 只说明差距超过测量误差，不说明指标本身是对的。", "闸审计的是榜面的可读性，不审计模型输出是否忠实、任务是否有意义。", "真实数据案例见 case-studies/chaosnli-noise-floor/；本页回放那份结果，不重算语料。", "报告 HTML 由 arx.audit.report_html 生成；页面不重新实现判决逻辑。" ], "verification": { "status": "MATCH", "cases": 8, "mismatches": [], "expected": { "readable": "READABLE", "marginal": "MARGINAL", "unreadable": "UNREADABLE", "two_generators": "UNDECIDABLE", "actors_overlap": "UNDECIDABLE", "noise": "UNDECIDABLE", "domain_mismatch": "UNDECIDABLE", "single_generator": "UNDECIDABLE" }, "checks": [ { "name": "sensitivity.40.0.005", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.40.0.01", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.40.0.02", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.40.0.05", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.80.0.005", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.80.0.01", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.80.0.02", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.80.0.05", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.120.0.005", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.120.0.01", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.120.0.02", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.120.0.05", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.200.0.005", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.200.0.01", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.200.0.02", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.200.0.05", "verdict": "MARGINAL", "ok": true }, { "name": "sensitivity.400.0.005", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.400.0.01", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.400.0.02", "verdict": "READABLE", "ok": true }, { "name": "sensitivity.400.0.05", "verdict": "MARGINAL", "ok": true }, { "name": "real_case.human.verdict", "expected": "READABLE", "actual": "READABLE", "ok": true }, { "name": "real_case.human.floor", "expected": 0.7467399999999997, "actual": 0.7467399999999997, "ok": true }, { "name": "real_case.human.margin", "expected": 0.0422600000000003, "actual": 0.0422600000000003, "ok": true }, { "name": "real_case.llm_judge_panel.verdict", "expected": "UNREADABLE", "actual": "UNREADABLE", "ok": true }, { "name": "real_case.llm_judge_panel.floor", "expected": 0.86596875, "actual": 0.86596875, "ok": true }, { "name": "real_case.llm_judge_panel.margin", "expected": -0.07696874999999992, "actual": -0.07696874999999992, "ok": true } ] } }