diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 28c8948..830975c 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "inference-arbitrage", - "version": "0.5.0", + "version": "0.5.1", "description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.", "author": { "name": "oleks" diff --git a/bin/audit-snapshot b/bin/audit-snapshot index 42784bf..9036fdb 100755 --- a/bin/audit-snapshot +++ b/bin/audit-snapshot @@ -653,9 +653,26 @@ def triple_line(f): for k in ("signature", "examples", "overrule_case")) -def decline_reason(c): +def decline_reason(c, downgraded_ids=frozenset()): """Why the gate refused, read off the structured grade — never off the - graders' prose `reasons`, whose provenance is mixed.""" + graders' prose `reasons`, whose provenance is mixed. + + Checks stay in FR-4.2's own order (overrule case, tests, confidence) + first — a candidate can be BOTH ungraded-for-filing AND coincidentally + named in a stability finding built from unrelated synthetic data, and + those real disqualifications must still win. Only the generic fallback at + the end — "did not clear the filing threshold" — is replaced when the + candidate is a genuine FR-4.5 downgrade, because a candidate that reaches + that fallback cleared every earlier check on its own grade: complete + triple, all determinism tests, `high` confidence. It cannot have failed + the filing threshold on its own merits and ALSO be one `stability-classify` + downgraded — real downgrades only ever touch candidates that already + cleared `boundary-classify`'s gate (verdict `file`), so reaching this line + while flagged in `downgraded_ids` means the fallback was always the wrong + reason for it and FR-4.5 is the real one. Fix for + kotkan/claude-plugin-inference-arbitrage#16, which contradicted the + correct FR-4.5 section rendered a few lines below it in the same drawer. + """ f = c.get("falsifiability") or {} tests = c.get("determinism_tests") or {} if f.get("overrule_case") is not True: @@ -667,6 +684,9 @@ def decline_reason(c): "yet located") if c.get("boundary_confidence") != "high": return f"boundary confidence {c.get('boundary_confidence')} — reported, not filed" + if c.get("candidate_id") in downgraded_ids: + return ("cleared boundary-classify's gate on its own grade — downgraded " + "afterward by the evidence-stability gate (FR-4.5); see below") return "did not clear the filing threshold in this window" @@ -724,6 +744,7 @@ def render_memory_judgments(snap, findings): questions = snap.get("boundary_questions") or [] if not questions and not findings: return None + downgraded_ids = {f.get("candidate_id") for f in findings} lines = [ f"# inference-arbitrage — judgment calls on {t.get('name')} (run {snap.get('run_id')})", "", @@ -744,7 +765,7 @@ def render_memory_judgments(snap, findings): f"audited spend over {m.get('invocations')} invocations.", f" - Determinism tests: {tests_line(c.get('determinism_tests'))}", f" - Falsifiability triple: {triple_line(c.get('falsifiability'))}", - f" - Declined because: {decline_reason(c)}", + f" - Declined because: {decline_reason(c, downgraded_ids)}", ] lines.append("") if findings: diff --git a/tests/memory.test.sh b/tests/memory.test.sh index 053ef8d..7330713 100755 --- a/tests/memory.test.sh +++ b/tests/memory.test.sh @@ -271,4 +271,52 @@ else echo " ok memory-notes exits non-zero with no snapshots recorded" fi +# -------------------------------------------------------------------------- +echo "== (h) regression #16: a stability-downgraded candidate gets its real reason ==" +# The bug: decline_reason() only ever looked at a candidate's own grade, so a +# candidate boundary-classify graded high/complete-triple (it cleared FR-4.2 +# fine) but that stability-classify then downgraded fell through to the +# generic "did not clear the filing threshold" line -- the opposite of what +# happened, and a direct contradiction of the FR-4.5 section rendered a few +# lines below it in the same drawer. +py " +import json +cand = {'candidate_id': 'synth/gamma/flip', 'skill': 'synth:gamma', + 'position': 'llm-over-script-digest', + 'determinism_tests': {t: True for t in ('T1','T2','T3','T4','T5')}, + 'falsifiability_triple': {'signature': True, 'examples': True, 'overrule_case': True}, + 'boundary_confidence': 'high', 'measurement_strength': 'thin', + 'share_of_audited_spend': 0.0, 'offload_value': 0, + 'escalation_path': 'escalate', 'digest_schema': 'typed rows', + # rewritten by stability-classify in real use -- 'file' cleared, then downgraded + 'verdict': 'boundary-question', 'reasons': ['downgraded by stability-classify']} +json.dump({'rubric_version': '1.0.0', 'target': 'synth', 'candidates': [cand]}, + open('$tmp/cls3.json','w')) +json.dump({'stability_findings': [{ + 'gate': 'FR-4.5', 'candidate_id': 'synth/gamma/flip', 'target': 'synth', + 'skill': 'synth:gamma', 'matched_by': 'slug', + 'prior': {'run_id': '2026-07-01T00-00-00Z', 'window': {'since': '2026-06-01'}, + 'measurement_strength': 'thin', 'share_of_audited_spend': 0.0}, + 'current': {'window': {'since': '2026-07-01'}, 'measurement_strength': 'measured', + 'share_of_audited_spend': 0.1185}, + 'flips': [{'field': 'measurement_strength', 'prior': 'thin', 'current': 'measured'}], + 'self_report': {'repo': 'kotkan/claude-plugin-inference-arbitrage', + 'marker': ''}}]}, + open('$tmp/stable3.json','w')) +" +store3=$tmp/store3 +$BIN --store "$store3" write --target synth --from "$tmp/inv.json" "$tmp/scan.json" \ + --classified "$tmp/cls3.json" --run-id 2026-07-29T12-00-00Z >/dev/null +$BIN --store "$store3" memory-notes --target synth --stable "$tmp/stable3.json" >"$tmp/n3.json" +py " +import json +c=[d for d in json.load(open('$tmp/n3.json'))['drawers'] if d['kind']=='judgment-calls'][0]['content'] +assert 'did not clear the filing threshold' not in c, \ + 'regression: a candidate stability downgraded was never actually rejected by the threshold' +assert 'cleared boundary-classify' in c and 'FR-4.5' in c, \ + 'the decline reason must point at the real cause' +assert 'no overrule case' not in c, 'this candidate has a complete triple, that reason is wrong too' +print(' ok stability-downgraded candidate reports the real reason, not the filing-threshold line')" || + fail=1 + exit "$fail"