diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index da6c546..d3aee35 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "inference-arbitrage", - "version": "0.6.2", + "version": "0.6.3", "description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.", "author": { "name": "oleks" diff --git a/bin/stability-classify b/bin/stability-classify index 6d5d0bd..1a5482f 100755 --- a/bin/stability-classify +++ b/bin/stability-classify @@ -48,6 +48,26 @@ nothing to compare against. It files normally, marked of history would contradict FR-2.4 — a target with no history stays auditable on its own terms — and would make the first audit of every plugin file nothing. +AN EMPTY LOCAL STORE IS NOT THE SAME CLAIM AS AN EMPTY WIKI +------------------------------------------------------------- +This script only ever reads the local filesystem mirror of the store — by +design, no `bin/` script holds wiki credentials (see `audit-snapshot`'s +docstring). That means an empty `--store` is ambiguous: it might be a genuine +first audit (FR-2.4), or it might just be a cold local cache sitting in front +of real wiki history nobody imported yet (kotkan/claude-plugin-inference- +arbitrage#17). The two look identical to `find_prior` but are not the same +fact about the target, and only the caller — who can read the wiki — knows +which one is true. + +So when the store is empty, `--wiki-checked` is mandatory: `empty` asserts the +caller checked the wiki page `Data//snapshots.jsonl` and it genuinely +has no history (or the target truly is new), so `unchecked` is correct; +`imported` asserts the caller ran `audit-snapshot import` first — in which case +the store would no longer be empty and this path would not be taken at all. +Omitting the flag on an empty store is a hard error: it turns the SKILL.md +step 4b instruction to check the wiki first from a doc-only aside into +something the gate itself refuses to skip past. + The prior window is the most recent snapshot that contains this candidate by FR-5.2 identity (issue > signature hash > slug), not merely the most recent snapshot: a candidate that sat out one window is still comparable against the @@ -232,6 +252,13 @@ def main(): ap.add_argument("--signatures", help="JSON {candidate_id: [tool signature, ...]}") ap.add_argument("--issue-map", help='JSON {candidate_id: "owner/repo#N"}') ap.add_argument("--exclude-run", help="run_id to treat as not-yet-recorded (tests, re-runs)") + ap.add_argument("--wiki-checked", choices=["empty", "imported"], + help="required when the local store is empty for this target: 'empty' " + "asserts the wiki was checked and genuinely has no history; " + "'imported' asserts `audit-snapshot import` already ran. Without " + "this, an empty store is ambiguous between a genuine first audit " + "(FR-2.4) and a merely cold local cache in front of real wiki " + "history (kotkan/claude-plugin-inference-arbitrage#17).") ap.add_argument("--json", action="store_true", help="compact single-line JSON") args = ap.parse_args() @@ -239,6 +266,17 @@ def main(): history = [s for s in load_snapshots(args.store, args.target) if s.get("run_id") != args.exclude_run] + if not history and args.wiki_checked is None: + sys.exit( + "error: the store has no history for this target and --wiki-checked was not " + "given. This is ambiguous: it may be a genuine first audit (FR-2.4), or it may " + "just be a cold local cache in front of real wiki history " + "(kotkan/claude-plugin-inference-arbitrage#17). Check the wiki page " + f"Data/{args.target}/snapshots.jsonl first, then either pass " + "--wiki-checked=empty (the wiki genuinely has no history for this target), or " + "run `audit-snapshot import` to seed the store and re-run with " + "--wiki-checked=imported.") + result = gate(doc, history, load_json(args.scan) if args.scan else {}, load_json(args.signatures) if args.signatures else {}, diff --git a/skills/offload-audit/SKILL.md b/skills/offload-audit/SKILL.md index 918a313..b62d469 100644 --- a/skills/offload-audit/SKILL.md +++ b/skills/offload-audit/SKILL.md @@ -109,27 +109,53 @@ complete including the overrule case) lives in that script. ## 4b. Check the evidence against the previous window The second hard gate (FR-4.5, rubric §5b). It runs on every audit, before -anything is filed, and its output **replaces** `classified.json` downstream: +anything is filed, and its output **replaces** `classified.json` downstream. + +**First, unconditionally — not just when the store looks cold — fetch the +wiki's copy of this target's history and seed the local store from it.** The +local store this gate reads is a filesystem-only mirror (no `bin/` script +holds wiki credentials); it never talks to the wiki itself. That means a store +that is empty *locally* is not evidence the target has no history — it may +just be a cache nobody has seeded on this machine yet, and treating the two as +the same thing is exactly the bug behind +kotkan/claude-plugin-inference-arbitrage#17: a candidate silently filed as +`unchecked` when real prior-window history existed on the wiki all along. +`import` dedupes by `run_id`, so running this every time is safe and +idempotent — there is no "if cold" branch to reason about: + +```bash +Agent(subagent_type="cluster:gitea-agent", prompt= + "wiki_read Data//snapshots.jsonl on kotkan/claude-plugin-inference-arbitrage + (or the target's own repo, per how this target's store is organized). If the page + exists, save its content to a local file and report the path. If it does not + exist, say so plainly.") + +# if the page existed: +$IA/bin/audit-snapshot import --target --snapshots +``` + +Then run the gate: ```bash $IA/bin/stability-classify --classified classified.json --target \ - --scan scan.json > stable.json + --scan scan.json --wiki-checked > stable.json ``` +`--wiki-checked` is mandatory whenever the local store turns out empty for +this target: pass `empty` if the wiki fetch above genuinely found no page (or +no history), or `imported` if it did and you ran `audit-snapshot import`. The +script hard-errors if the store is empty and this flag is omitted — the wiki +check is no longer a documentation aside the gate quietly trusts you did. + It matches each `file` candidate against the most recent prior snapshot that recorded it (FR-5.2 identity) and downgrades any whose `measurement_strength` changed, or whose share crossed the 2% filing threshold, between the two windows. **Use `stable.json` for steps 5 and 6.** Never file from `classified.json` once this has run. -If the store has no history for this target — a first audit — every candidate -comes back `stability: unchecked` and files normally. That is the FR-2.4 -outcome, not a failure. If the store is cold but the wiki has history, seed it -first, or the gate has nothing to check against: - -```bash -$IA/bin/audit-snapshot import --target --snapshots -``` +If the store has no history for this target even after the wiki fetch above — +a genuine first audit — every candidate comes back `stability: unchecked` and +files normally. That is the FR-2.4 outcome, not a failure. ### 4b-i. A downgrade is a finding against *this* plugin diff --git a/tests/stability.test.sh b/tests/stability.test.sh index 3ec210b..5e57084 100755 --- a/tests/stability.test.sh +++ b/tests/stability.test.sh @@ -76,6 +76,8 @@ json.dump({ "candidates": [{ "candidate_id": "synth/skill/step", "skill": "synth:skill", + "position": "llm-over-script-digest", + "boundary_confidence": "high", "signature_hash": "slugonly", "measurement_strength": strength, "measurement": {"invocations": 10, "offload_value": 500000, @@ -141,9 +143,11 @@ print(' ok stability names the window it was checked against') # -------------------------------------------------------------------------- echo "== (c) no prior snapshot -> files normally, unchecked (FR-2.4) ==" # An absence of history is not evidence of instability. A first audit of a -# target must still be able to file. +# target must still be able to file. The caller must still assert the wiki +# was checked (kotkan/claude-plugin-inference-arbitrage#17) -- see (g)/(h). classified "$tmp/c.json" measured 0.12 -$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" >"$tmp/c.out.json" +$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" \ + --wiki-checked=empty >"$tmp/c.out.json" check "verdict" "$(field "$tmp/c.out.json" synth/skill/step verdict)" "file" check "stability" "$(field "$tmp/c.out.json" synth/skill/step stability)" \ "unchecked — no prior window to compare" @@ -215,4 +219,51 @@ assert d['summary']['to_file']==0, d['summary'] print(' ok nothing is left to file from that window') " || fail=1 +# -------------------------------------------------------------------------- +echo "== (g) empty store, no --wiki-checked -> hard error (kotkan/claude-plugin-inference-arbitrage#17) ==" +# The bug: a cold local cache and a genuinely history-free wiki look identical +# to this script, which only ever reads the filesystem. Without an explicit +# assertion from the caller, refuse rather than silently degrade to unchecked. +classified "$tmp/g.json" measured 0.12 +if $BIN --classified "$tmp/g.json" --target synth --store "$tmp/store-empty-g" \ + >"$tmp/g.out.json" 2>"$tmp/g.err"; then + echo " FAIL expected non-zero exit, got success" >&2 + fail=1 +else + echo " ok exits non-zero" +fi +if grep -q "wiki-checked" "$tmp/g.err"; then + echo " ok error names --wiki-checked" +else + echo " FAIL error does not mention --wiki-checked: $(cat "$tmp/g.err")" >&2 + fail=1 +fi + +# -------------------------------------------------------------------------- +echo "== (h) empty store, --wiki-checked=empty -> unchecked, as today ==" +# The caller checked the wiki and it really is history-free; behavior matches (c). +classified "$tmp/h.json" measured 0.12 +$BIN --classified "$tmp/h.json" --target synth --store "$tmp/store-empty-h" \ + --wiki-checked=empty >"$tmp/h.out.json" +check "verdict" "$(field "$tmp/h.out.json" synth/skill/step verdict)" "file" +check "stability" "$(field "$tmp/h.out.json" synth/skill/step stability)" \ + "unchecked — no prior window to compare" + +# -------------------------------------------------------------------------- +echo "== (i) empty store seeded via a real import, --wiki-checked=imported -> gate runs normally ==" +# After `audit-snapshot import` seeds the store from a wiki-fetched +# snapshots.jsonl, the store is no longer empty, so the gate compares against +# the imported history exactly as if it had been local all along. +store=$tmp/store-import-i +mkdir -p "$store" +prior "$tmp/seed-i" thin 0.0 +IMPORT_BIN=../bin/audit-snapshot +$IMPORT_BIN --store "$store" import --target synth \ + --snapshots "$tmp/seed-i/Data/synth/snapshots.jsonl" >/dev/null +classified "$tmp/i.json" measured 0.12 +$BIN --classified "$tmp/i.json" --target synth --store "$store" \ + --wiki-checked=imported >"$tmp/i.out.json" +check "verdict" "$(field "$tmp/i.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/i.out.json" downgraded)" "1" + exit "$fail"