Oleks
70 lines
2.4 KiB
JSON
70 lines
2.4 KiB
JSON
{
|
|
"target": "fixture-plugin",
|
|
"note": "Synthetic. Two candidates: one that clears the FR-4.2 gate and the 2% filing threshold, one that is correctly left to inference. Shapes match tests/calibration/*.judgments.json.",
|
|
"audited_spend_weighted_tokens": 1000000,
|
|
"candidates": [
|
|
{
|
|
"candidate_id": "fixture-plugin/skill-a/label-derivation",
|
|
"skill": "fixture-plugin:skill-a",
|
|
"headline": "label derivation is a documented lookup table applied by inference",
|
|
"category": "usage",
|
|
"position": "llm-over-script-digest",
|
|
"determinism_tests": {
|
|
"T1": true,
|
|
"T2": true,
|
|
"T3": true,
|
|
"T4": true,
|
|
"T5": true
|
|
},
|
|
"falsifiability": {
|
|
"signature": "derive_labels(repo: str, issue: Issue) -> tuple[Labels, list[Ambiguity]]",
|
|
"examples": [
|
|
"repo='x', issue.title='fix crash' -> (['kind/bug'], [])",
|
|
"repo='x', issue.body has a checklist -> (['kind/epic'], [])",
|
|
"EDGE: repo='x', no signal at all -> ([], [Ambiguity('no axis-1 signal, defer to human')])"
|
|
],
|
|
"edge_case_index": 2,
|
|
"overrule_case": "An issue whose title reads as a bug but which the maintainer is deliberately tracking as a capability gap. The script returns kind/bug; a human says no. It cannot see that intent — it is not in the issue text."
|
|
},
|
|
"escalation_path": "Rows the table cannot resolve are emitted as Ambiguity and go to the model, never defaulted. The script proposes labels and never applies them; application stays behind the judgment gate (P4).",
|
|
"digest_schema": "One typed row per issue: (number, title, has_checklist, proposed_labels, ambiguities). Replaces reading every issue whole.",
|
|
"measurement": {
|
|
"offload_value": 200000,
|
|
"invocations": 12,
|
|
"attributed_turns": 200,
|
|
"mtr": 0.68,
|
|
"judgment_density": 0.11,
|
|
"ngram": {
|
|
"sig": ["list_issues(<str>)", "issue_read(<n>)", "label_read(<n>)"],
|
|
"recurrences": 17
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"candidate_id": "fixture-plugin/skill-b/naming",
|
|
"skill": "fixture-plugin:skill-b",
|
|
"category": "usage",
|
|
"position": "pure-inference",
|
|
"determinism_tests": {
|
|
"T1": false,
|
|
"T2": false,
|
|
"T3": true,
|
|
"T4": false,
|
|
"T5": false
|
|
},
|
|
"falsifiability": {
|
|
"signature": null,
|
|
"examples": [],
|
|
"overrule_case": null
|
|
},
|
|
"escalation_path": null,
|
|
"digest_schema": null,
|
|
"measurement": {
|
|
"offload_value": 90000,
|
|
"invocations": 8,
|
|
"attributed_turns": 120
|
|
}
|
|
}
|
|
]
|
|
}
|