diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index da6c546..d3aee35 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "inference-arbitrage", - "version": "0.6.2", + "version": "0.6.3", "description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.", "author": { "name": "oleks" diff --git a/bin/audit-snapshot b/bin/audit-snapshot index 9036fdb..6634ecb 100755 --- a/bin/audit-snapshot +++ b/bin/audit-snapshot @@ -212,6 +212,7 @@ def build_snapshot(target, inventory, scan, classified, issue_map, explicit, run "weighted_tokens": scan_total.get("weighted_tokens", 0), "invocations": scan_total.get("invocations", 0), "share_mechanical": scan_total.get("mechanical_share", 0.0), + "invocation_caveat": scan_total.get("invocation_caveat"), }, "candidates": [snapshot_candidate(c, scan, explicit, issue_map) for c in filed], "boundary_questions": [snapshot_candidate(c, scan, explicit, issue_map) for c in questions], @@ -452,6 +453,12 @@ def render_run(snap, diffdoc): f"- Mechanical share of audited spend: {(snap['totals'].get('share_mechanical') or 0):.1%}", f"- Rubric v{snap.get('rubric_version')}, inference-arbitrage " f"v{snap.get('tool_versions', {}).get('inference-arbitrage')}", + ] + invocation_caveat = snap["totals"].get("invocation_caveat") + if invocation_caveat: + lines.append(f"- Note: {invocation_caveat}, so per-invocation figures below are " + "correspondingly inflated") + lines += [ "", "## Candidates", "", diff --git a/bin/offload-scan b/bin/offload-scan index a3b4f38..c4b0740 100755 --- a/bin/offload-scan +++ b/bin/offload-scan @@ -750,6 +750,17 @@ def main(): "mechanical_share": round( sum(1 for t in turns if classify(t, catalog) == "mechanical") / len(turns), 3) if turns else 0.0, + # reconstruct_invocations() only sees already-attributed turns and + # splits on a wall-clock idle gap; it has no signal for unrelated + # activity between two genuinely separate same-skill calls, so + # back-to-back invocations closer together than the gap collapse + # into one reported invocation. Counts here are a LOWER BOUND on + # the true invocation count, and per-invocation figures derived + # from them (e.g. cost/invocation) are correspondingly inflated. + "invocation_caveat": ( + "invocation counts are a lower bound: two same-skill calls " + f"within {int(th['idle_gap_minutes'])} minutes of each other on the wall " + "clock are merged into one reported invocation"), }, "skills": skills, "agents": agents, diff --git a/tests/scan.test.sh b/tests/scan.test.sh index 9da6889..aecd8c3 100755 --- a/tests/scan.test.sh +++ b/tests/scan.test.sh @@ -43,6 +43,16 @@ check "agent attributed via attributionAgent" "d['agents'][0]['agent'] == 'fixtu # --- 3.2 invocation reconstruction ---------------------------------------- check "idle gap splits mine-me into 3 invocations" "$(skill mine-me)['invocations'] == 3" check "think-hard is one contiguous invocation" "$(skill think-hard)['invocations'] == 1" +# kotkan/claude-plugin-inference-arbitrage#12: reconstruct_invocations() only +# sees already-attributed turns and splits on wall-clock idle gap alone, so +# it cannot tell two genuinely separate same-skill calls apart from one +# contiguous call if they land inside the gap. The emitted totals must say so +# rather than let the headline invocation/cost-per-invocation figures imply +# more confidence than the reconstruction can support. +check "totals carry a lower-bound caveat naming the idle gap threshold" " + 'lower bound' in d['totals']['invocation_caveat'] + and '10 minutes' in d['totals']['invocation_caveat'] +" # --- 3.3 turn classification ---------------------------------------------- check "mine-me is wholly mechanical" "$(skill mine-me)['mtr'] == 1.0" diff --git a/tests/snapshot.test.sh b/tests/snapshot.test.sh index 92a449c..c7395d6 100755 --- a/tests/snapshot.test.sh +++ b/tests/snapshot.test.sh @@ -317,6 +317,32 @@ else fail=1 fi +echo "== kotkan/claude-plugin-inference-arbitrage#12: scan's invocation_caveat reaches the summary page ==" +store6=$tmp/store6 +inventory "$tmp/inv6.json" 1.0.0 steady +scan "$tmp/scan6.json" 10000000 "steady:20:2000000:list_issues,issue_read" +python3 -c " +import json +d = json.load(open('$tmp/scan6.json')) +d['totals']['invocation_caveat'] = ('invocation counts are a lower bound: two same-skill ' + 'calls within 10 minutes of each other on the wall clock are merged into one reported ' + 'invocation') +json.dump(d, open('$tmp/scan6.json', 'w')) +" +classified "$tmp/cls6.json" "synth/steady/a:steady:2000000:0.20:file" +$BIN --store "$store6" write --target synth --run-id 2026-07-01T00:00:00Z \ + --from "$tmp/inv6.json" "$tmp/scan6.json" --classified "$tmp/cls6.json" >/dev/null +check "invocation_caveat stored in totals" "$(python3 -c " +import json +s=[json.loads(l) for l in open('$store6/Data/synth/snapshots.jsonl')][0] +print('lower bound' in s['totals']['invocation_caveat'])")" "True" +if grep -qF "lower bound" "$store6/Audits/synth/2026-07-01.md"; then + echo " ok invocation caveat rendered in the audit page" +else + echo " FAIL invocation caveat missing from the audit page" >&2 + fail=1 +fi + echo "== a re-run of the same run_id is refused ==" if $BIN --store "$store" write --target synth --run-id 2026-07-08T00:00:00Z \ --from "$tmp/inv2.json" "$tmp/scan2.json" --classified "$tmp/cls2.json" >/dev/null 2>&1; then