audit-snapshot: fix wrong decline reason for a stability-downgraded candidate
decline_reason() only ever read a candidate's own grade, so a candidate boundary-classify graded high/complete-triple -- meaning it cleared FR-4.2 fine -- fell through to the generic 'did not clear the filing threshold' line when stability-classify downgraded it afterward. That's the opposite of what happened, and it directly contradicted the correct FR-4.5 section rendered a few lines below it in the same judgment-calls drawer, poisoning the next audit's step-2b recall with a false reason. Found live: real audit of anxious (2026-07-30) downgraded agent-wip/release-policy-derivation at 11.85% of audited spend -- the one gate it DID clear -- and the drawer said it didn't. Filed and now fixed as kotkan/claude-plugin-inference-arbitrage#16. Fix keeps FR-4.2's own check order first (a candidate can coincidentally be named in an unrelated stability finding without having actually cleared those checks) and only replaces the final fallback, since a candidate that reaches it cleared every earlier check on its own grade -- it cannot have also failed the filing threshold on its own merits. New regression case (h) in tests/memory.test.sh. Suite: all tests pass.
This commit is contained in:
Oleks
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "inference-arbitrage",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.1",
|
||||
"description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.",
|
||||
"author": {
|
||||
"name": "oleks"
|
||||
|
||||
+24
-3
@@ -653,9 +653,26 @@ def triple_line(f):
|
||||
for k in ("signature", "examples", "overrule_case"))
|
||||
|
||||
|
||||
def decline_reason(c):
|
||||
def decline_reason(c, downgraded_ids=frozenset()):
|
||||
"""Why the gate refused, read off the structured grade — never off the
|
||||
graders' prose `reasons`, whose provenance is mixed."""
|
||||
graders' prose `reasons`, whose provenance is mixed.
|
||||
|
||||
Checks stay in FR-4.2's own order (overrule case, tests, confidence)
|
||||
first — a candidate can be BOTH ungraded-for-filing AND coincidentally
|
||||
named in a stability finding built from unrelated synthetic data, and
|
||||
those real disqualifications must still win. Only the generic fallback at
|
||||
the end — "did not clear the filing threshold" — is replaced when the
|
||||
candidate is a genuine FR-4.5 downgrade, because a candidate that reaches
|
||||
that fallback cleared every earlier check on its own grade: complete
|
||||
triple, all determinism tests, `high` confidence. It cannot have failed
|
||||
the filing threshold on its own merits and ALSO be one `stability-classify`
|
||||
downgraded — real downgrades only ever touch candidates that already
|
||||
cleared `boundary-classify`'s gate (verdict `file`), so reaching this line
|
||||
while flagged in `downgraded_ids` means the fallback was always the wrong
|
||||
reason for it and FR-4.5 is the real one. Fix for
|
||||
kotkan/claude-plugin-inference-arbitrage#16, which contradicted the
|
||||
correct FR-4.5 section rendered a few lines below it in the same drawer.
|
||||
"""
|
||||
f = c.get("falsifiability") or {}
|
||||
tests = c.get("determinism_tests") or {}
|
||||
if f.get("overrule_case") is not True:
|
||||
@@ -667,6 +684,9 @@ def decline_reason(c):
|
||||
"yet located")
|
||||
if c.get("boundary_confidence") != "high":
|
||||
return f"boundary confidence {c.get('boundary_confidence')} — reported, not filed"
|
||||
if c.get("candidate_id") in downgraded_ids:
|
||||
return ("cleared boundary-classify's gate on its own grade — downgraded "
|
||||
"afterward by the evidence-stability gate (FR-4.5); see below")
|
||||
return "did not clear the filing threshold in this window"
|
||||
|
||||
|
||||
@@ -724,6 +744,7 @@ def render_memory_judgments(snap, findings):
|
||||
questions = snap.get("boundary_questions") or []
|
||||
if not questions and not findings:
|
||||
return None
|
||||
downgraded_ids = {f.get("candidate_id") for f in findings}
|
||||
lines = [
|
||||
f"# inference-arbitrage — judgment calls on {t.get('name')} (run {snap.get('run_id')})",
|
||||
"",
|
||||
@@ -744,7 +765,7 @@ def render_memory_judgments(snap, findings):
|
||||
f"audited spend over {m.get('invocations')} invocations.",
|
||||
f" - Determinism tests: {tests_line(c.get('determinism_tests'))}",
|
||||
f" - Falsifiability triple: {triple_line(c.get('falsifiability'))}",
|
||||
f" - Declined because: {decline_reason(c)}",
|
||||
f" - Declined because: {decline_reason(c, downgraded_ids)}",
|
||||
]
|
||||
lines.append("")
|
||||
if findings:
|
||||
|
||||
@@ -271,4 +271,52 @@ else
|
||||
echo " ok memory-notes exits non-zero with no snapshots recorded"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
echo "== (h) regression #16: a stability-downgraded candidate gets its real reason =="
|
||||
# The bug: decline_reason() only ever looked at a candidate's own grade, so a
|
||||
# candidate boundary-classify graded high/complete-triple (it cleared FR-4.2
|
||||
# fine) but that stability-classify then downgraded fell through to the
|
||||
# generic "did not clear the filing threshold" line -- the opposite of what
|
||||
# happened, and a direct contradiction of the FR-4.5 section rendered a few
|
||||
# lines below it in the same drawer.
|
||||
py "
|
||||
import json
|
||||
cand = {'candidate_id': 'synth/gamma/flip', 'skill': 'synth:gamma',
|
||||
'position': 'llm-over-script-digest',
|
||||
'determinism_tests': {t: True for t in ('T1','T2','T3','T4','T5')},
|
||||
'falsifiability_triple': {'signature': True, 'examples': True, 'overrule_case': True},
|
||||
'boundary_confidence': 'high', 'measurement_strength': 'thin',
|
||||
'share_of_audited_spend': 0.0, 'offload_value': 0,
|
||||
'escalation_path': 'escalate', 'digest_schema': 'typed rows',
|
||||
# rewritten by stability-classify in real use -- 'file' cleared, then downgraded
|
||||
'verdict': 'boundary-question', 'reasons': ['downgraded by stability-classify']}
|
||||
json.dump({'rubric_version': '1.0.0', 'target': 'synth', 'candidates': [cand]},
|
||||
open('$tmp/cls3.json','w'))
|
||||
json.dump({'stability_findings': [{
|
||||
'gate': 'FR-4.5', 'candidate_id': 'synth/gamma/flip', 'target': 'synth',
|
||||
'skill': 'synth:gamma', 'matched_by': 'slug',
|
||||
'prior': {'run_id': '2026-07-01T00-00-00Z', 'window': {'since': '2026-06-01'},
|
||||
'measurement_strength': 'thin', 'share_of_audited_spend': 0.0},
|
||||
'current': {'window': {'since': '2026-07-01'}, 'measurement_strength': 'measured',
|
||||
'share_of_audited_spend': 0.1185},
|
||||
'flips': [{'field': 'measurement_strength', 'prior': 'thin', 'current': 'measured'}],
|
||||
'self_report': {'repo': 'kotkan/claude-plugin-inference-arbitrage',
|
||||
'marker': '<!-- ia-stability: synth/gamma/flip -->'}}]},
|
||||
open('$tmp/stable3.json','w'))
|
||||
"
|
||||
store3=$tmp/store3
|
||||
$BIN --store "$store3" write --target synth --from "$tmp/inv.json" "$tmp/scan.json" \
|
||||
--classified "$tmp/cls3.json" --run-id 2026-07-29T12-00-00Z >/dev/null
|
||||
$BIN --store "$store3" memory-notes --target synth --stable "$tmp/stable3.json" >"$tmp/n3.json"
|
||||
py "
|
||||
import json
|
||||
c=[d for d in json.load(open('$tmp/n3.json'))['drawers'] if d['kind']=='judgment-calls'][0]['content']
|
||||
assert 'did not clear the filing threshold' not in c, \
|
||||
'regression: a candidate stability downgraded was never actually rejected by the threshold'
|
||||
assert 'cleared boundary-classify' in c and 'FR-4.5' in c, \
|
||||
'the decline reason must point at the real cause'
|
||||
assert 'no overrule case' not in c, 'this candidate has a complete triple, that reason is wrong too'
|
||||
print(' ok stability-downgraded candidate reports the real reason, not the filing-threshold line')" ||
|
||||
fail=1
|
||||
|
||||
exit "$fail"
|
||||
|
||||
Reference in New Issue
Block a user