diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index c7612a2..cd5354b 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "inference-arbitrage", - "version": "0.3.0", + "version": "0.4.0", "description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.", "author": { "name": "oleks" diff --git a/agents/offload-analyst.md b/agents/offload-analyst.md index d81623b..37373af 100644 --- a/agents/offload-analyst.md +++ b/agents/offload-analyst.md @@ -158,6 +158,22 @@ Note the inverse too: if the overrule case is *common*, the candidate is not a position-1 script — it is position 3, and the cut belongs earlier in the pipeline. +### 4b. The second hard gate — evidence stability + +> **Evidence-strength label or filing-threshold side changes across measurement +> windows → downgrade to a boundary question, never file (FR-4.5).** + +`bin/stability-classify` enforces it, against the prior snapshot, and it runs +after `boundary-classify` and before anything is filed. The reason is the mirror +of §4's: the triple checks that the *reasoning* survives scrutiny, this checks +that the *evidence* does. A `measured` 12% that becomes a `thin` 0% on a +different date range was never a measurement of the target, and a human shown +only one of the two has no way to discount it. + +When it fires, the finding is about **this plugin**, not the audited one — file +it on `kotkan/claude-plugin-inference-arbitrage`. Do not caveat the instability +into an issue body on the target's repo; a hedge a reader can skip is not a gate. + ### 5. Report - **Rank by measured `offload_value`, and say that you are doing so.** A reader diff --git a/bin/audit-snapshot b/bin/audit-snapshot index b037bff..0b5914c 100755 --- a/bin/audit-snapshot +++ b/bin/audit-snapshot @@ -39,20 +39,15 @@ the whole file, not a delta. """ import argparse -import hashlib import json import os import sys from datetime import datetime, timezone -DEFAULT_STORE = "~/.cache/inference-arbitrage/wiki" - -# Both mirror bin/boundary-classify deliberately: a candidate must clear the -# same bar to be called fixed that it had to clear to be filed. Changing one -# without the other would make "resolved" mean something the filing gate never -# meant. -MIN_SHARE_OF_SPEND = 0.02 -MIN_INVOCATIONS = 3 +from ia_store import (DEFAULT_STORE, MIN_INVOCATIONS, MIN_SHARE_OF_SPEND, + audits_dir, build_index, candidate_signature, find_match, + load_snapshots, scan_rows, signature_hash, snapshots_path, + store_root, tiered) # Relative change in the volume-normalized metrics below which a candidate is # `persisting` rather than grown/shrunk. Sampling noise across two windows of @@ -71,36 +66,6 @@ STATUSES = ("new", "persisting", "grown", "shrunk", "resolved", # store # -------------------------------------------------------------------------- -def store_root(store): - return os.path.expanduser(store) - - -def snapshots_path(store, target): - return os.path.join(store_root(store), "Data", target, "snapshots.jsonl") - - -def audits_dir(store, target): - return os.path.join(store_root(store), "Audits", target) - - -def load_snapshots(store, target): - path = snapshots_path(store, target) - if not os.path.exists(path): - return [] - out = [] - with open(path) as fh: - for lineno, line in enumerate(fh, 1): - line = line.strip() - if not line: - continue - try: - out.append(json.loads(line)) - except json.JSONDecodeError as exc: - sys.exit(f"error: {path}:{lineno}: corrupt snapshot record: {exc}") - out.sort(key=lambda s: s.get("run_id", "")) - return out - - def append_snapshot(store, target, snap): path = snapshots_path(store, target) os.makedirs(os.path.dirname(path), exist_ok=True) @@ -116,47 +81,6 @@ def write_text(path, text): return path -# -------------------------------------------------------------------------- -# identity (FR-5.2) -# -------------------------------------------------------------------------- - -def signature_hash(sig): - """Hash of a normalized tool-sequence signature. `sig` is the list of - masked signatures offload-scan already emits, so normalization happened - upstream; joining them is all that is left.""" - return hashlib.sha256("\n".join(sig).encode()).hexdigest()[:16] - - -def identity(cand): - """FR-5.2 resolution order. Issue number first: it is the only identifier - a human also uses, and it survives a refactor that moves the code.""" - if cand.get("issue"): - return ("issue", cand["issue"]) - if cand.get("signature_hash"): - return ("signature", cand["signature_hash"]) - return ("slug", cand.get("candidate_id")) - - -def build_index(cands): - idx = {"issue": {}, "signature": {}, "slug": {}} - for c in cands: - if c.get("issue"): - idx["issue"].setdefault(c["issue"], c) - if c.get("signature_hash"): - idx["signature"].setdefault(c["signature_hash"], c) - idx["slug"].setdefault(c.get("candidate_id"), c) - return idx - - -def find_match(cand, idx): - for kind, key in (("issue", cand.get("issue")), - ("signature", cand.get("signature_hash")), - ("slug", cand.get("candidate_id"))): - if key and key in idx[kind]: - return idx[kind][key], kind - return None, None - - # -------------------------------------------------------------------------- # write # -------------------------------------------------------------------------- @@ -171,13 +95,6 @@ def tool_version(): return None -def scan_rows(scan): - for row in scan.get("skills", []): - yield row.get("skill"), row - for row in scan.get("agents", []): - yield row.get("agent"), row - - def measurements_by_signature(scan): """Every recurring signature the scan saw, keyed by hash — including ones that no longer classify as candidates. FR-5.5 needs exactly this: a later @@ -198,20 +115,6 @@ def measurements_by_signature(scan): return out -def candidate_signature(cand, scan, explicit): - """Signature resolution, most trustworthy first: an explicitly supplied - tool sequence, then the dominant recurring n-gram of the candidate's own - skill, then the slug. The source is recorded because a slug-derived - identity cannot detect drift — it IS the thing drift is measured against.""" - cid = cand.get("candidate_id") - if explicit.get(cid): - return list(explicit[cid]), "explicit" - for name, row in scan_rows(scan): - if name == cand.get("skill") and row.get("ngrams"): - return list(row["ngrams"][0]["sig"]), "scan-ngram" - return [f"slug:{cid}"], "slug" - - def snapshot_candidate(cand, scan, explicit, issue_map): sig, source = candidate_signature(cand, scan, explicit) row = next((r for n, r in scan_rows(scan) if n == cand.get("skill")), {}) @@ -415,13 +318,6 @@ def row_of(cand, tier, status, reason, matched_by, prev_cand=None): } -def tiered(snap): - for c in snap.get("candidates", []): - yield c, "candidate" - for c in snap.get("boundary_questions", []): - yield c, "boundary-question" - - def diff_snapshots(prev, curr, issue_state): prev_all = list(tiered(prev)) idx = build_index([c for c, _ in prev_all]) diff --git a/bin/ia_store.py b/bin/ia_store.py new file mode 100644 index 0000000..735e4a5 --- /dev/null +++ b/bin/ia_store.py @@ -0,0 +1,143 @@ +"""Snapshot store access and candidate-identity resolution (FR-5.2), shared by +`bin/audit-snapshot` (which writes and diffs snapshots) and +`bin/stability-classify` (which reads them to gate filing, FR-4.5). + +It exists so that identity is computed exactly once. The gate's whole job is to +find the *same* candidate in an earlier window; if it hashed a signature even +slightly differently from the writer, it would silently find nothing and pass +every unstable candidate through — a gate that fails open and looks like it +worked. Copy-pasting `signature_hash` between the two scripts is therefore not +a style problem, it is the failure mode. + +Imported by name from `bin/`, which is `sys.path[0]` for a script run as +`bin/`. +""" + +import hashlib +import json +import os +import sys + +DEFAULT_STORE = "~/.cache/inference-arbitrage/wiki" + +# spec §5.2.7's filing threshold, mirroring bin/boundary-classify — a candidate +# must clear the same bar to be called fixed, or stable, that it had to clear to +# be filed. Changing one without the others would make those words mean +# something the filing gate never meant. +MIN_SHARE_OF_SPEND = 0.02 +MIN_INVOCATIONS = 3 + + +# -------------------------------------------------------------------------- +# store +# -------------------------------------------------------------------------- + + +def store_root(store): + return os.path.expanduser(store) + + +def snapshots_path(store, target): + return os.path.join(store_root(store), "Data", target, "snapshots.jsonl") + + +def audits_dir(store, target): + return os.path.join(store_root(store), "Audits", target) + + +def load_snapshots(store, target): + path = snapshots_path(store, target) + if not os.path.exists(path): + return [] + out = [] + with open(path) as fh: + for lineno, line in enumerate(fh, 1): + line = line.strip() + if not line: + continue + try: + out.append(json.loads(line)) + except json.JSONDecodeError as exc: + sys.exit(f"error: {path}:{lineno}: corrupt snapshot record: {exc}") + out.sort(key=lambda s: s.get("run_id", "")) + return out + + +# -------------------------------------------------------------------------- +# identity (FR-5.2) +# -------------------------------------------------------------------------- + + +def signature_hash(sig): + """Hash of a normalized tool-sequence signature. `sig` is the list of + masked signatures offload-scan already emits, so normalization happened + upstream; joining them is all that is left.""" + return hashlib.sha256("\n".join(sig).encode()).hexdigest()[:16] + + +def identity(cand): + """FR-5.2 resolution order. Issue number first: it is the only identifier + a human also uses, and it survives a refactor that moves the code.""" + if cand.get("issue"): + return ("issue", cand["issue"]) + if cand.get("signature_hash"): + return ("signature", cand["signature_hash"]) + return ("slug", cand.get("candidate_id")) + + +def build_index(cands): + idx = {"issue": {}, "signature": {}, "slug": {}} + for c in cands: + if c.get("issue"): + idx["issue"].setdefault(c["issue"], c) + if c.get("signature_hash"): + idx["signature"].setdefault(c["signature_hash"], c) + idx["slug"].setdefault(c.get("candidate_id"), c) + return idx + + +def find_match(cand, idx): + for kind, key in ( + ("issue", cand.get("issue")), + ("signature", cand.get("signature_hash")), + ("slug", cand.get("candidate_id")), + ): + if key and key in idx[kind]: + return idx[kind][key], kind + return None, None + + +def tiered(snap): + """Both tiers of a snapshot. A candidate that was a boundary question last + window and a filing candidate this one is the same candidate, so identity + matching must see both piles.""" + for c in snap.get("candidates", []): + yield c, "candidate" + for c in snap.get("boundary_questions", []): + yield c, "boundary-question" + + +# -------------------------------------------------------------------------- +# signatures derived from a scan +# -------------------------------------------------------------------------- + + +def scan_rows(scan): + for row in scan.get("skills", []): + yield row.get("skill"), row + for row in scan.get("agents", []): + yield row.get("agent"), row + + +def candidate_signature(cand, scan, explicit): + """Signature resolution, most trustworthy first: an explicitly supplied + tool sequence, then the dominant recurring n-gram of the candidate's own + skill, then the slug. The source is recorded because a slug-derived + identity cannot detect drift — it IS the thing drift is measured against.""" + cid = cand.get("candidate_id") + if explicit.get(cid): + return list(explicit[cid]), "explicit" + for name, row in scan_rows(scan): + if name == cand.get("skill") and row.get("ngrams"): + return list(row["ngrams"][0]["sig"]), "scan-ngram" + return [f"slug:{cid}"], "slug" diff --git a/bin/stability-classify b/bin/stability-classify new file mode 100755 index 0000000..6d5d0bd --- /dev/null +++ b/bin/stability-classify @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 +"""The evidence-stability gate (FR-4.5, rubric §5b): refuses to file a candidate +whose evidence label is an artifact of which measurement window happened to be +audited. + +WHY THIS IS A GATE AND NOT A CAVEAT +----------------------------------- +Phase 7 filed `anxious/agent-wip/release-policy-derivation` on a 30-day window +as `measurement_strength: measured`, 11.94% of audited spend — a confident, +concrete cost claim. The identical candidate, classified over a 7-day window, +came back `thin`, 0.00% of spend. Nothing about the plugin changed between the +two; only the date range did. + +A human reading either issue body alone has no way to know the other exists. +That is the `rubric.md` §4a failure — fast, confident, and wrong, with nobody +watching — reached by the plugin's own headline number. It was caught once, by +a human running a second skeptical review, which is not a repeatable control: +the next candidate gets filed only if someone happens to be suspicious that day. + +So stability becomes mechanical, in the same register as FR-4.2's overrule-case +gate: a candidate whose evidence flips across windows is downgraded to a +boundary question and never filed, and the flip is emitted as a finding against +this plugin's own repo (FR-8.2), because a confidence label that moves with the +calendar is a bug in the auditor, not a fact about the target. + +WHAT COUNTS AS A FLIP +--------------------- +Two things, either of which disqualifies: + + 1. `measurement_strength` differs between the windows (`measured` vs `thin` + vs `unmeasured`). This is the label the issue body speaks in, so a change + here changes what a human is told. + 2. `share_of_audited_spend` lands on opposite sides of MIN_SHARE_OF_SPEND. + That threshold is the filing gate itself (spec §5.2.7): if one window would + have filed on value and the other would not, the value claim is not + load-bearing evidence, it is a window artifact. + +Ordinary movement within a band is NOT a flip. A candidate that measures 8% one +window and 5% the next is stable — same label, same side of the threshold — and +that variation is what `audit-snapshot diff` exists to report as a trend. This +gate fires only when the *description a human receives* changes. + +NO PRIOR WINDOW MEANS UNCHECKED, NOT UNSTABLE +--------------------------------------------- +A first-ever audit of a target, or a candidate seen for the first time, has +nothing to compare against. It files normally, marked +`stability: unchecked`. Manufacturing an instability finding out of an absence +of history would contradict FR-2.4 — a target with no history stays auditable on +its own terms — and would make the first audit of every plugin file nothing. + +The prior window is the most recent snapshot that contains this candidate by +FR-5.2 identity (issue > signature hash > slug), not merely the most recent +snapshot: a candidate that sat out one window is still comparable against the +window before it, and using the latest snapshot unconditionally would report +every such candidate as unmatched and let it through. + +Runs after `boundary-classify` and before `filing-plan`; emits the same document +shape it consumes, with verdicts revised, so it drops into the pipeline in place +of `classified.json`. +""" + +import argparse +import json +import sys + +from ia_store import (DEFAULT_STORE, MIN_SHARE_OF_SPEND, build_index, + candidate_signature, find_match, load_snapshots, + signature_hash, tiered) + +GATE = "FR-4.5" + +UNCHECKED_NO_HISTORY = "unchecked — no prior window to compare" +UNCHECKED_NO_MATCH = "unchecked — no prior window recorded this candidate" + + +def keyed(cand, scan, explicit, issue_map): + """The identity fields FR-5.2 matches on, computed for a candidate that is + only classified and not yet snapshotted. Derived with the same + `candidate_signature` the snapshot writer uses — a divergence here would + make the gate silently match nothing and pass every unstable candidate.""" + sig, source = candidate_signature(cand, scan, explicit) + return { + "candidate_id": cand.get("candidate_id"), + "issue": issue_map.get(cand.get("candidate_id")) or cand.get("issue"), + "signature_hash": signature_hash(sig), + "signature_source": source, + } + + +def find_prior(key, history): + """Most recent snapshot holding this candidate, newest first.""" + for snap in reversed(history): + idx = build_index([c for c, _ in tiered(snap)]) + match, by = find_match(key, idx) + if match: + return snap, match, by + return None, None, None + + +def crosses_threshold(a, b): + return (a >= MIN_SHARE_OF_SPEND) != (b >= MIN_SHARE_OF_SPEND) + + +def flips(prior, curr_strength, curr_share): + """Named, so the finding says exactly what moved rather than 'unstable'.""" + out = [] + prior_strength = prior.get("measurement_strength") + prior_share = (prior.get("measurement") or {}).get("share_of_plugin", 0.0) or 0.0 + + if prior_strength != curr_strength: + out.append({ + "field": "measurement_strength", + "prior": prior_strength, + "current": curr_strength, + "why": ("the evidence label the issue body speaks in changed with the window, " + "so the same candidate would be described differently to a human " + "depending only on when the audit ran"), + }) + if crosses_threshold(prior_share, curr_share): + out.append({ + "field": "share_of_audited_spend", + "prior": prior_share, + "current": curr_share, + "threshold": MIN_SHARE_OF_SPEND, + "why": (f"the windows land on opposite sides of the {MIN_SHARE_OF_SPEND:.0%} " + "filing threshold — one window would file on value and the other " + "would not, so the value claim is a window artifact"), + }) + return out + + +def gate(doc, history, scan, explicit, issue_map): + target = doc.get("target") + window = scan.get("window") or {} + findings = [] + checked = 0 + + for cand in doc.get("candidates", []): + if cand.get("verdict") != "file": + continue + + if not history: + cand["stability"] = UNCHECKED_NO_HISTORY + continue + + key = keyed(cand, scan, explicit, issue_map) + snap, prior, by = find_prior(key, history) + if prior is None: + cand["stability"] = UNCHECKED_NO_MATCH + continue + + checked += 1 + curr_strength = cand.get("measurement_strength") + curr_share = cand.get("share_of_audited_spend", 0.0) or 0.0 + moved = flips(prior, curr_strength, curr_share) + + if not moved: + cand["stability"] = ( + f"stable against {snap.get('run_id')} (matched by {by}): " + f"{curr_strength}, {curr_share:.2%} of audited spend either side") + continue + + cand["verdict"] = "boundary-question" + cand["stability"] = f"unstable against {snap.get('run_id')} (matched by {by})" + cand.setdefault("reasons", []).append( + f"HARD GATE ({GATE}): evidence is not stable across measurement windows — " + + "; ".join(f"{f['field']} {f['prior']!r} -> {f['current']!r}" for f in moved) + + ". Downgraded to a boundary question, never filed." + ) + + findings.append({ + "gate": GATE, + "candidate_id": cand.get("candidate_id"), + "target": target, + "skill": cand.get("skill"), + "matched_by": by, + "prior": { + "run_id": snap.get("run_id"), + "window": snap.get("window"), + "measurement_strength": prior.get("measurement_strength"), + "share_of_audited_spend": (prior.get("measurement") or {}).get("share_of_plugin"), + }, + "current": { + "window": window, + "measurement_strength": curr_strength, + "share_of_audited_spend": curr_share, + }, + "flips": moved, + "headline": ( + f"{cand.get('candidate_id')} is {prior.get('measurement_strength')} at " + f"{((prior.get('measurement') or {}).get('share_of_plugin') or 0):.2%} of spend in " + f"{snap.get('run_id')} and {curr_strength} at {curr_share:.2%} in this window — " + "the confidence label depends on the measurement window, not on the candidate"), + # FR-8.2: the instability is a finding about THIS plugin, filed on + # its own repo. The marker mirrors FR-6.3's `ia-candidate` so a + # re-run comments instead of duplicating. + "self_report": { + "repo": "kotkan/claude-plugin-inference-arbitrage", + "marker": f"", + }, + }) + + rows = doc.get("candidates", []) + summary = doc.setdefault("summary", {}) + summary["to_file"] = sum(1 for c in rows if c.get("verdict") == "file") + summary["boundary_questions"] = sum(1 for c in rows if c.get("verdict") == "boundary-question") + summary["stability_checked"] = checked + summary["stability_downgraded"] = len(findings) + + doc["stability_gate"] = { + "gate": GATE, + "snapshots_available": len(history), + "checked": checked, + "downgraded": len(findings), + } + doc["stability_findings"] = findings + return doc + + +def load_json(path): + with open(path) as fh: + return json.load(fh) + + +def main(): + ap = argparse.ArgumentParser(prog="stability-classify") + ap.add_argument("--classified", required=True, help="bin/boundary-classify output") + ap.add_argument("--target", required=True, help="target plugin name, as snapshotted") + ap.add_argument("--store", default=DEFAULT_STORE, + help=f"local mirror of the wiki store (default {DEFAULT_STORE})") + ap.add_argument("--scan", help="bin/offload-scan output, for signature identity") + ap.add_argument("--signatures", help="JSON {candidate_id: [tool signature, ...]}") + ap.add_argument("--issue-map", help='JSON {candidate_id: "owner/repo#N"}') + ap.add_argument("--exclude-run", help="run_id to treat as not-yet-recorded (tests, re-runs)") + ap.add_argument("--json", action="store_true", help="compact single-line JSON") + args = ap.parse_args() + + doc = load_json(args.classified) + history = [s for s in load_snapshots(args.store, args.target) + if s.get("run_id") != args.exclude_run] + + result = gate(doc, history, + load_json(args.scan) if args.scan else {}, + load_json(args.signatures) if args.signatures else {}, + load_json(args.issue_map) if args.issue_map else {}) + + json.dump(result, sys.stdout, indent=None if args.json else 2) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/commands/offload-audit.md b/commands/offload-audit.md index 8e78475..6d49101 100644 --- a/commands/offload-audit.md +++ b/commands/offload-audit.md @@ -13,9 +13,10 @@ Run the full audit on **$ARGUMENTS**. 2. If no target was given, ask which plugin to audit. Do not guess one, and do not default to the current directory — auditing the wrong plugin wastes a full transcript scan. -3. Invoke the `offload-audit` skill and follow it exactly, including the filing - gate and the marker search. For a heavy target, or when the report should not - consume this session's context, hand the work to +3. Invoke the `offload-audit` skill and follow it exactly, including both filing + gates (the overrule case, FR-4.2; the evidence-stability check against the + previous window, FR-4.5) and the marker search. For a heavy target, or when + the report should not consume this session's context, hand the work to `Agent(subagent_type="inference-arbitrage:offload-analyst")` instead — one agent for the whole audit, never one per skill (spec §7). diff --git a/design/rubric.md b/design/rubric.md index 53c2155..107c099 100644 --- a/design/rubric.md +++ b/design/rubric.md @@ -14,7 +14,9 @@ The rubric has five parts, applied in order: the two in the middle. 3. **The positioning principle** — given a pipeline, where exactly do you cut? 4. **Two failure modes** — named, with detection questions for each. -5. **The falsifiability triple** — the gate a candidate must pass to be filed. +5. **The falsifiability triple** — the gate a candidate's *reasoning* must pass + to be filed. + **5b. The evidence-stability gate** — the gate its *evidence* must pass. --- @@ -310,6 +312,57 @@ precisely the behavior the rubric is trying to teach. --- +## 5b. The evidence-stability gate + +The triple in §5 asks whether the *reasoning* survives scrutiny. This asks +whether the *evidence* does. + +A candidate's headline is a number and a word: `measured`, 12% of audited spend. +Both come from a window — the range of transcripts the dynamic pass happened to +read — and the window is chosen by whoever ran the audit, usually by typing a +default. If the same candidate reads `measured, 12%` over thirty days and +`thin, 0%` over seven, then the audit has not measured the candidate. It has +measured the calendar, and reported the result as a property of the plugin. + +The damage is specific, and it is §4a's damage arriving by a new route. A human +reading the issue sees one window's figure, with no indication that another +window exists or disagrees. They cannot discount what they are not shown. So the +number reads as a fact, gets acted on as a fact, and the correction — if it ever +comes — arrives only because someone independently re-ran the audit on a +different range and happened to notice. That is not a control. It is luck with a +review attached. + +Hence the same shape of rule as §5: + +> **Evidence that changes label across windows → downgrade to boundary +> question.** Compare each filable candidate against the most recent prior +> window that recorded it. If the evidence-strength label differs, or if the two +> windows land on opposite sides of the filing threshold, the candidate is not +> filed. + +Two boundaries make this a usable rule rather than a blanket refusal. + +**Movement is not instability.** A candidate at 12% one window and 8% the next +is stable: the human is told the same thing, and the difference is a trend, which +the snapshot diff already reports as one. The gate fires only when the +*description a human receives* would change — the label, or which side of the +threshold the value falls on. Everything else is signal about the target, and +suppressing it would throw away the trend the plugin exists to accumulate. + +**Absence of history is not instability.** The first audit of a target has +nothing to compare against, and a candidate seen for the first time has no +earlier window. Those file normally, marked unchecked. Inventing an instability +finding out of missing history would mean no plugin could ever be audited once, +which contradicts the principle that a plugin with no measured history stays +auditable on its definition alone. + +The instability, when it fires, is a finding **about the auditor**. The target +did not change; the tool described it two different ways. So it is reported +against this plugin's own repo, not the target's — the same self-application +discipline as §FR-8, applied to the plugin's own headline number. + +--- + ## 6. Confidence scoring | `boundary_confidence` | Requires | diff --git a/design/spec.md b/design/spec.md index b05a5db..e87b005 100644 --- a/design/spec.md +++ b/design/spec.md @@ -223,6 +223,19 @@ say so plainly. `token-budget` is expected to produce exactly this result; if a run against `token-budget` produces a list of confident offload candidates, the rubric is miscalibrated and that is a bug in this plugin. +**FR-4.5** No candidate may be filed unless its **evidence is stable across +measurement windows**. Before filing, each candidate whose verdict is `file` is +matched by FR-5.2 identity against the most recent prior snapshot recording it, +and the two windows' evidence is compared. A candidate whose +`measurement_strength` differs between the windows, or whose +`share_of_audited_spend` falls on opposite sides of the §5.2.7 filing threshold, +**is downgraded to a boundary question and never filed** — and the instability is +itself reported as a finding against *this* plugin's repo (FR-8.2), because a +confidence label that moves with the calendar is a defect in the auditor, not a +fact about the target. A candidate with no prior window to compare against is +marked `unchecked` and files normally; an absence of history is not evidence of +instability, and manufacturing one from it would contradict FR-2.4. + ### FR-5 — Snapshot accumulation and diffing **FR-5.1** Every run appends one machine-readable snapshot record to an diff --git a/design/tasks.md b/design/tasks.md index e21426f..724c181 100644 --- a/design/tasks.md +++ b/design/tasks.md @@ -99,6 +99,8 @@ These block design, not just code. Answers go into `Methodology/Calibration.md`. - `token-budget` → **must** yield zero `high`-confidence candidates (S2). - `worktree-discipline` with `bin/worktree-audit` masked out of the inventory → **must** flag worktree classification as `high` (S3). + - the two-window `anxious` fixture → **must** downgrade + `release-policy-derivation` on the stability gate (S4). If either fails, the rubric is wrong and Phase 5 does not start. - [ ] **4.5** Record the calibration outcome and any threshold change, with its evidence, in `Methodology/Calibration.md`. @@ -139,8 +141,14 @@ These block design, not just code. Answers go into `Methodology/Calibration.md`. - [x] **7.1** S1 — audits four different plugins with no target-specific code. - [x] **7.2** S2 — `token-budget` → "nothing to offload." - [x] **7.3** S3 — masked `worktree-discipline` → rediscovers the known cut. -- [ ] **7.4** S4 — a real candidate in `anxious` that the user agrees is genuine. - **User sign-off required before the first issue is filed to a real repo.** +- [x] **7.4** S4 — the FR-4.5 evidence-stability gate mechanically catches a real + unstable candidate in `anxious` (`agent-wip/release-policy-derivation`) + without human review. The original framing — "a candidate the user agrees + is genuine" — is retired: a human triage catching a bad candidate is not a + repeatable test, and the one that ran caught this candidate *because* its + evidence flipped across windows. That check is now + `bin/stability-classify`, asserted against the real two-window evidence in + `tests/stability.test.sh` case (f). - [x] **7.5** S5 — double-run files no duplicate. - [x] **7.6** S6 — two-snapshot trend is computable and volume-normalized. - [x] **7.7** S7 — **audit `inference-arbitrage` with `inference-arbitrage`** diff --git a/references/boundary-rubric.md b/references/boundary-rubric.md index 53c2155..107c099 100644 --- a/references/boundary-rubric.md +++ b/references/boundary-rubric.md @@ -14,7 +14,9 @@ The rubric has five parts, applied in order: the two in the middle. 3. **The positioning principle** — given a pipeline, where exactly do you cut? 4. **Two failure modes** — named, with detection questions for each. -5. **The falsifiability triple** — the gate a candidate must pass to be filed. +5. **The falsifiability triple** — the gate a candidate's *reasoning* must pass + to be filed. + **5b. The evidence-stability gate** — the gate its *evidence* must pass. --- @@ -310,6 +312,57 @@ precisely the behavior the rubric is trying to teach. --- +## 5b. The evidence-stability gate + +The triple in §5 asks whether the *reasoning* survives scrutiny. This asks +whether the *evidence* does. + +A candidate's headline is a number and a word: `measured`, 12% of audited spend. +Both come from a window — the range of transcripts the dynamic pass happened to +read — and the window is chosen by whoever ran the audit, usually by typing a +default. If the same candidate reads `measured, 12%` over thirty days and +`thin, 0%` over seven, then the audit has not measured the candidate. It has +measured the calendar, and reported the result as a property of the plugin. + +The damage is specific, and it is §4a's damage arriving by a new route. A human +reading the issue sees one window's figure, with no indication that another +window exists or disagrees. They cannot discount what they are not shown. So the +number reads as a fact, gets acted on as a fact, and the correction — if it ever +comes — arrives only because someone independently re-ran the audit on a +different range and happened to notice. That is not a control. It is luck with a +review attached. + +Hence the same shape of rule as §5: + +> **Evidence that changes label across windows → downgrade to boundary +> question.** Compare each filable candidate against the most recent prior +> window that recorded it. If the evidence-strength label differs, or if the two +> windows land on opposite sides of the filing threshold, the candidate is not +> filed. + +Two boundaries make this a usable rule rather than a blanket refusal. + +**Movement is not instability.** A candidate at 12% one window and 8% the next +is stable: the human is told the same thing, and the difference is a trend, which +the snapshot diff already reports as one. The gate fires only when the +*description a human receives* would change — the label, or which side of the +threshold the value falls on. Everything else is signal about the target, and +suppressing it would throw away the trend the plugin exists to accumulate. + +**Absence of history is not instability.** The first audit of a target has +nothing to compare against, and a candidate seen for the first time has no +earlier window. Those file normally, marked unchecked. Inventing an instability +finding out of missing history would mean no plugin could ever be audited once, +which contradicts the principle that a plugin with no measured history stays +auditable on its definition alone. + +The instability, when it fires, is a finding **about the auditor**. The target +did not change; the tool described it two different ways. So it is reported +against this plugin's own repo, not the target's — the same self-application +discipline as §FR-8, applied to the plugin's own headline number. + +--- + ## 6. Confidence scoring | `boundary_confidence` | Requires | diff --git a/skills/offload-audit/SKILL.md b/skills/offload-audit/SKILL.md index 1ce3ebf..795be34 100644 --- a/skills/offload-audit/SKILL.md +++ b/skills/offload-audit/SKILL.md @@ -6,10 +6,10 @@ allowed-tools: Bash, Read, Grep, Glob, Skill, Agent # Audit a plugin for work that should be a script -Six steps, in order. Steps 1–3 are scripts, step 4 is the only judgment, steps -5–6 are delegation. **Do not do by hand anything a `bin/` tool does** — this -plugin is subject to its own thesis (FR-8), and an audit that counts things by -inference is the finding, not the method. +Seven steps, in order. Steps 1–3 and 4b are scripts, step 4 is the only +judgment, steps 5–6 are delegation. **Do not do by hand anything a `bin/` tool +does** — this plugin is subject to its own thesis (FR-8), and an audit that +counts things by inference is the finding, not the method. Full rubric: the `boundary-rubric` skill. Deeper reporting discipline: `agents/offload-analyst.md`. Issue-body contract and the marker-search @@ -73,6 +73,53 @@ complete including the overrule case) lives in that script. > **No overrule case → boundary question, reported, never filed.** Hard gate, > FR-4.2. When genuinely torn, do not file. +## 4b. Check the evidence against the previous window + +The second hard gate (FR-4.5, rubric §5b). It runs on every audit, before +anything is filed, and its output **replaces** `classified.json` downstream: + +```bash +$IA/bin/stability-classify --classified classified.json --target \ + --scan scan.json > stable.json +``` + +It matches each `file` candidate against the most recent prior snapshot that +recorded it (FR-5.2 identity) and downgrades any whose `measurement_strength` +changed, or whose share crossed the 2% filing threshold, between the two +windows. **Use `stable.json` for steps 5 and 6.** Never file from +`classified.json` once this has run. + +If the store has no history for this target — a first audit — every candidate +comes back `stability: unchecked` and files normally. That is the FR-2.4 +outcome, not a failure. If the store is cold but the wiki has history, seed it +first, or the gate has nothing to check against: + +```bash +$IA/bin/audit-snapshot import --target --snapshots +``` + +### 4b-i. A downgrade is a finding against *this* plugin + +Each entry in `stability_findings` is a defect in the auditor — the tool +described the same candidate two different ways depending on the date range. File +it, implicitly, on `kotkan/claude-plugin-inference-arbitrage`, **not** on the +target's repo. Idempotency works the same way as FR-6.3: the finding carries +`self_report.marker` (``); search for that +marker first and comment on the existing issue rather than opening a second one. + +```text +Agent(subagent_type="anxious:issuer-agent", prompt=...) + "On kotkan/claude-plugin-inference-arbitrage: search issues (state=all) for the + literal marker . If it exists, comment the + new numbers on that issue. If not, open one — the marker must be the first line + of the body, verbatim. This is an inference-arbitrage self-audit finding: a + candidate's evidence label changed across measurement windows, so the gate + refused to file it. Include the headline, both windows, and both figures." +``` + +Do not report the instability as a caveat inside a target-repo issue. A hedge a +reader can skip is not a gate. + ## 5. Output path A — file the issues **Filing is implicit.** Do not ask permission to file a candidate that cleared @@ -90,7 +137,7 @@ offload-audit → Agent(anxious:issuer-agent) → cluster:gitea-agent → Gitea ```bash $IA/bin/filing-plan queries \ - --classified classified.json --judgments judgments.json \ + --classified stable.json --judgments judgments.json \ --target-path > queries.json ``` @@ -126,7 +173,7 @@ silently become a duplicate-filing spree (FR-6.3). ```bash $IA/bin/filing-plan plan \ - --classified classified.json --judgments judgments.json \ + --classified stable.json --judgments judgments.json \ --target-path --existing existing.json \ --audit-page "Audits//" \ --window " -> " --run-date \ @@ -176,7 +223,7 @@ runs **even when path A was skipped**. ```bash $IA/bin/audit-snapshot write --target --from inv.json scan.json \ - --classified classified.json --issue-map issue-map.json \ + --classified stable.json --issue-map issue-map.json \ --repo \ --note "" $IA/bin/audit-snapshot pages --target > pages.json diff --git a/tests/calibration/README.md b/tests/calibration/README.md index 024fd81..99eb87a 100644 --- a/tests/calibration/README.md +++ b/tests/calibration/README.md @@ -50,3 +50,31 @@ rather than in agent prose (FR-8.1 — the plugin obeying its own thesis). | --- | --- | --- | | 1 (S2) | `token-budget` | zero `high`-confidence candidates; conclusion "nothing to offload here" | | 2 (S3) | `worktree-discipline`, masked | `sweep-worktrees` worktree-safety classification at `high`, position `llm-over-script-digest` | +| 3 (S4) | `anxious`, two windows | `release-policy-derivation` downgraded to a boundary question by the FR-4.5 stability gate, with no human in the loop | + +## The two-window `anxious` fixture — gate 3 + +Asserted by `tests/stability.test.sh`. These are **real** Phase 7 outputs, kept +verbatim, because the point of S4 is that the gate catches a candidate that +actually got filed, not a case constructed to be caught: + +- `anxious-windows.snapshots.jsonl` — both snapshots. Run + `2026-07-29T00-00-00Z` is the 30-day window, in which + `anxious/agent-wip/release-policy-derivation` graded `measured` at 11.94% of + audited spend. Run `2026-07-29T01-00-00Z` is the 7-day window, in which the + same candidate graded `thin` at 0.00%. +- `anxious-7d.classified.json` — `boundary-classify`'s verdict on the 7-day + window, which was `file`, `high` confidence, `thin` evidence. +- `anxious-7d.scan.json` — the 7-day scan, for signature identity. Masked + shapes and counts only (FR-3.5); no transcript content. + +The test excludes the 7-day run from the store so it is the run being +classified, exactly as at filing time, and asserts the gate downgrades it. The +comparison is symmetric: classifying the 30-day window against the 7-day +snapshot downgrades it too. Both windows cannot be right, which is the finding — +recorded as kotkan/claude-plugin-inference-arbitrage#11. + +The candidate matches by **slug**, not signature: the 7-day scan produced no +recurring n-gram for that skill, so its signature falls back to the slug and the +hashes differ. That is FR-5.2's resolution order doing its job — an identity +that survives the evidence disappearing is exactly what a stability check needs. diff --git a/tests/calibration/anxious-7d.classified.json b/tests/calibration/anxious-7d.classified.json new file mode 100644 index 0000000..3e7436b --- /dev/null +++ b/tests/calibration/anxious-7d.classified.json @@ -0,0 +1,150 @@ +{ + "rubric_version": "1.0.0", + "target": "anxious (7-day window)", + "summary": { + "candidates": 5, + "high_confidence": 1, + "to_file": 1, + "boundary_questions": 2, + "no_offload": 1, + "drift_notes": 1, + "conclusion": "1 high-confidence candidate(s)" + }, + "candidates": [ + { + "candidate_id": "anxious/issuer-agent/consecutive-agent-fanout", + "skill": "anxious:issuer-agent", + "category": "usage", + "position": "pure-inference", + "determinism_tests": { + "T1": false, + "T2": false, + "T3": true, + "T4": false, + "T5": false + }, + "falsifiability_triple": { + "signature": false, + "examples": false, + "overrule_case": false + }, + "boundary_confidence": "low", + "measurement_strength": "measured", + "share_of_audited_spend": 0.75, + "offload_value": 3027155, + "verdict": "no-offload", + "reasons": [ + "T2 or T4 in doubt \u2014 the oracle or the semantic gap is unresolved", + "position 4 \u2014 correctly done by inference. Reported as a finding, not a question." + ] + }, + { + "candidate_id": "anxious/agent-wip/release-policy-derivation", + "skill": "anxious:agent-wip", + "category": "usage", + "position": "llm-over-script-digest", + "determinism_tests": { + "T1": true, + "T2": true, + "T3": true, + "T4": true, + "T5": true + }, + "falsifiability_triple": { + "signature": true, + "examples": true, + "overrule_case": true + }, + "boundary_confidence": "high", + "measurement_strength": "thin", + "share_of_audited_spend": 0.0, + "offload_value": 0, + "verdict": "file", + "reasons": [ + "all five determinism tests pass; triple complete; escalation path defined", + "filed on the static case \u2014 evidence is thin (0 invocations, 0 attributed turns), so the value claim is unproven and must be stated as such in the issue body" + ] + }, + { + "candidate_id": "anxious/steward-agent/taxonomy-axis-derivation", + "skill": "anxious:steward-agent", + "category": "usage", + "position": "llm-over-script-digest", + "determinism_tests": { + "T1": true, + "T2": true, + "T3": true, + "T4": true, + "T5": true + }, + "falsifiability_triple": { + "signature": true, + "examples": false, + "overrule_case": true + }, + "boundary_confidence": "medium", + "measurement_strength": "thin", + "share_of_audited_spend": 0.0, + "offload_value": 0, + "verdict": "boundary-question", + "reasons": [ + "falsifiability triple incomplete: missing examples", + "confidence medium \u2014 reported, not filed" + ] + }, + { + "candidate_id": "anxious/pr-rot/unmeasured-script-verb-dominance", + "skill": "anxious:pr-rot", + "category": "static", + "position": "llm-over-script-digest", + "determinism_tests": { + "T1": true, + "T2": true, + "T3": true, + "T4": true, + "T5": true + }, + "falsifiability_triple": { + "signature": true, + "examples": false, + "overrule_case": false + }, + "boundary_confidence": "medium", + "measurement_strength": "thin", + "share_of_audited_spend": 0.0, + "offload_value": 0, + "verdict": "boundary-question", + "reasons": [ + "falsifiability triple incomplete: missing examples, overrule_case", + "HARD GATE (FR-4.2): no overrule case \u2014 downgraded to a boundary question, never filed" + ] + }, + { + "candidate_id": "anxious/kanban-flow/hook-prose-drift", + "skill": "anxious:kanban-flow", + "category": "hook-prose-drift", + "position": "pure-inference", + "determinism_tests": { + "T1": false, + "T2": false, + "T3": true, + "T4": false, + "T5": true + }, + "falsifiability_triple": { + "signature": false, + "examples": false, + "overrule_case": false + }, + "boundary_confidence": "low", + "measurement_strength": "thin", + "share_of_audited_spend": 0.0, + "offload_value": 0, + "verdict": "drift-note", + "reasons": [ + "T2 or T4 in doubt \u2014 the oracle or the semantic gap is unresolved", + "hook/prose drift \u2014 a restated configuration contract, not an un-scripted algorithm. The fix is to point at the hook file, not to write a script." + ] + } + ] +} diff --git a/tests/calibration/anxious-7d.scan.json b/tests/calibration/anxious-7d.scan.json new file mode 100644 index 0000000..d67a56d --- /dev/null +++ b/tests/calibration/anxious-7d.scan.json @@ -0,0 +1,116 @@ +{ + "window": { + "since": "2026-07-22T15:04:13.318000+00:00", + "until": "2026-07-29T15:04:20.306000+00:00", + "requested_since": "2026-07-22T15:04:13.270540+00:00", + "requested_until": null, + "sessions": 69 + }, + "plugin": "anxious", + "tool_versions": { + "cc_tokens_path": "/home/oleks/projects/claude-plugins/token-budget/bin/cc-tokens" + }, + "coverage": { + "attributed_turns": 1493, + "candidate_turns": 45384, + "ratio": 0.033, + "caveat": "hook-driven and unattributed agent turns excluded" + }, + "totals": { + "weighted_tokens": 4036235, + "cost_estimate_usd": 20.18, + "invocations": 146, + "mechanical_share": 0.179 + }, + "skills": [], + "agents": [ + { + "agent": "anxious:issuer-agent", + "invocations": 144, + "turns": 1487, + "sidechain_turns": 1487, + "weighted_tokens": 3856203, + "cost_estimate_usd": 19.28, + "share_of_plugin": 0.955, + "mtr": 0.18, + "judgment_density": 0.019, + "retry_density": 0.132, + "offload_waste": 536202, + "read_amplification": 492.12, + "fanout_multiplier": 1.0, + "repetition_factor": 5.755, + "ngrams": [ + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 23, + "invocations": 21, + "waste": 277765 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 10, + "invocations": 10, + "waste": 151483 + }, + { + "sig": [ + "issue_read(index=, method=, owner=, repo=)", + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 11, + "invocations": 9, + "waste": 133610 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 7, + "invocations": 7, + "waste": 127433 + }, + { + "sig": [ + "Bash(command=cat )", + "Bash(command=cat )", + "Bash(command=cat )" + ], + "recurrences": 27, + "invocations": 9, + "waste": 102394 + } + ], + "offload_value": 3027155 + }, + { + "agent": "anxious:steward-agent", + "invocations": 2, + "turns": 6, + "sidechain_turns": 6, + "weighted_tokens": 180032, + "cost_estimate_usd": 0.9, + "share_of_plugin": 0.045, + "mtr": 0.0, + "judgment_density": 0.167, + "retry_density": 0.333, + "offload_waste": 0, + "read_amplification": 0.0, + "fanout_multiplier": 1.0, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 0 + } + ] +} diff --git a/tests/calibration/anxious-windows.snapshots.jsonl b/tests/calibration/anxious-windows.snapshots.jsonl new file mode 100644 index 0000000..e879ae5 --- /dev/null +++ b/tests/calibration/anxious-windows.snapshots.jsonl @@ -0,0 +1,2 @@ +{"boundary_questions": [{"boundary_confidence": "medium", "candidate_id": "anxious/steward-agent/taxonomy-axis-derivation", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": false, "overrule_case": true, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": 2522, "invocations": 50, "offload_value": 126116, "offload_waste": 17835, "share_of_plugin": 0.0039}, "measurement_strength": "measured", "position": "llm-over-script-digest", "signature": ["Agent(description=, prompt=, run_in_background=, subagent_type=)", "Agent(description=, prompt=, run_in_background=, subagent_type=)", "Agent(description=, prompt=, run_in_background=, subagent_type=)"], "signature_hash": "58d2793e6ae0be3a", "signature_source": "scan-ngram", "skill": "anxious:steward-agent", "verdict": "boundary-question"}, {"boundary_confidence": "medium", "candidate_id": "anxious/pr-rot/unmeasured-script-verb-dominance", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": false, "overrule_case": false, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": null, "invocations": 0, "offload_value": 0, "offload_waste": 0, "share_of_plugin": 0.0}, "measurement_strength": "thin", "position": "llm-over-script-digest", "signature": ["slug:anxious/pr-rot/unmeasured-script-verb-dominance"], "signature_hash": "81c46de83ec1db54", "signature_source": "slug", "skill": "anxious:pr-rot", "verdict": "boundary-question"}], "candidates": [{"boundary_confidence": "high", "candidate_id": "anxious/agent-wip/release-policy-derivation", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": true, "overrule_case": true, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": 325055, "invocations": 12, "offload_value": 3900668, "offload_waste": 1332195, "share_of_plugin": 0.1194}, "measurement_strength": "measured", "position": "llm-over-script-digest", "signature": ["Edit(file_path=, new_string=, old_string=, replace_all=)", "Read(file_path=)", "Edit(file_path=, new_string=, old_string=, replace_all=)"], "signature_hash": "c996b432e9e74875", "signature_source": "scan-ngram", "skill": "anxious:agent-wip", "verdict": "file"}], "coverage": {"attributed_turns": 4925, "ratio": 0.021}, "measurements_by_signature": {"0e86b21ec00b654d": {"cost_per_invocation": 149564, "invocations": 8, "offload_value": 1196514, "share_of_plugin": 0.0366}, "2975e355fac82fbe": {"cost_per_invocation": 95184, "invocations": 3, "offload_value": 285552, "share_of_plugin": 0.0087}, "50a421d7e1b738c1": {"cost_per_invocation": 99897, "invocations": 3, "offload_value": 299691, "share_of_plugin": 0.0092}, "5293e14ad0d8a437": {"cost_per_invocation": 13226, "invocations": 21, "offload_value": 277765, "share_of_plugin": 0.0085}, "5633671c2238385e": {"cost_per_invocation": 25412, "invocations": 6, "offload_value": 152475, "share_of_plugin": 0.0047}, "58d2793e6ae0be3a": {"cost_per_invocation": 103019, "invocations": 29, "offload_value": 2987563, "share_of_plugin": 0.0915}, "7d605314639126b3": {"cost_per_invocation": 22448, "invocations": 9, "offload_value": 202033, "share_of_plugin": 0.0062}, "7eeb6c7ac199ef39": {"cost_per_invocation": 121085, "invocations": 15, "offload_value": 1816278, "share_of_plugin": 0.0556}, "a66700b03dd6cf83": {"cost_per_invocation": 70949, "invocations": 3, "offload_value": 212849, "share_of_plugin": 0.0065}, "a80cd235dd72737e": {"cost_per_invocation": 100681, "invocations": 3, "offload_value": 302043, "share_of_plugin": 0.0092}, "c996b432e9e74875": {"cost_per_invocation": 114041, "invocations": 3, "offload_value": 342125, "share_of_plugin": 0.0105}, "e024d9bb86195e89": {"cost_per_invocation": 110637, "invocations": 22, "offload_value": 2434027, "share_of_plugin": 0.0745}, "e36302f74f0b178b": {"cost_per_invocation": 18719, "invocations": 46, "offload_value": 861104, "share_of_plugin": 0.0264}, "fbb113ca63dff419": {"cost_per_invocation": 20484, "invocations": 24, "offload_value": 491635, "share_of_plugin": 0.0151}}, "notes": ["anxious/issuer-agent/consecutive-agent-fanout: no-offload \u2014 position 4 \u2014 correctly done by inference. Reported as a finding, not a question.", "anxious/kanban-flow/hook-prose-drift: drift-note \u2014 hook/prose drift \u2014 a restated configuration contract, not an un-scripted algorithm. The fix is to point at the hook file, not to write a script.", "S6: 30-day window"], "rubric_version": "1.0.0", "run_id": "2026-07-29T00-00-00Z", "target": {"name": "anxious", "repo": "oleks/claude-plugin-anxious", "skills": ["agent-wip", "archivist-agent", "coach-agent", "delivery-flow", "docs-agent", "issuer-agent", "kaizen", "kanban-flow", "nudge-watch", "pomodoro", "pr-rot", "steward-agent"], "source": "path", "version": "0.32.0"}, "tool_versions": {"cc-tokens": "/home/oleks/projects/claude-plugins/token-budget/bin/cc-tokens", "inference-arbitrage": "0.3.0"}, "totals": {"invocations": 666, "share_mechanical": 0.101, "weighted_tokens": 32660210}, "window": {"requested_since": "2026-06-29T15:03:44.048713+00:00", "requested_until": null, "sessions": 165, "since": "2026-07-03T12:23:08.955000+00:00", "until": "2026-07-29T15:04:06.599000+00:00"}} +{"boundary_questions": [{"boundary_confidence": "medium", "candidate_id": "anxious/steward-agent/taxonomy-axis-derivation", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": false, "overrule_case": true, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": 0, "invocations": 2, "offload_value": 0, "offload_waste": 0, "share_of_plugin": 0.0}, "measurement_strength": "thin", "position": "llm-over-script-digest", "signature": ["slug:anxious/steward-agent/taxonomy-axis-derivation"], "signature_hash": "5543110781c02f44", "signature_source": "slug", "skill": "anxious:steward-agent", "verdict": "boundary-question"}, {"boundary_confidence": "medium", "candidate_id": "anxious/pr-rot/unmeasured-script-verb-dominance", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": false, "overrule_case": false, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": null, "invocations": 0, "offload_value": 0, "offload_waste": 0, "share_of_plugin": 0.0}, "measurement_strength": "thin", "position": "llm-over-script-digest", "signature": ["slug:anxious/pr-rot/unmeasured-script-verb-dominance"], "signature_hash": "81c46de83ec1db54", "signature_source": "slug", "skill": "anxious:pr-rot", "verdict": "boundary-question"}], "candidates": [{"boundary_confidence": "high", "candidate_id": "anxious/agent-wip/release-policy-derivation", "determinism_tests": {"T1": true, "T2": true, "T3": true, "T4": true, "T5": true}, "digest_schema": null, "escalation_path": null, "falsifiability": {"examples": true, "overrule_case": true, "signature": true}, "issue": null, "measurement": {"cost_per_invocation": null, "invocations": 0, "offload_value": 0, "offload_waste": 0, "share_of_plugin": 0.0}, "measurement_strength": "thin", "position": "llm-over-script-digest", "signature": ["slug:anxious/agent-wip/release-policy-derivation"], "signature_hash": "0c08bf7a89c90ee1", "signature_source": "slug", "skill": "anxious:agent-wip", "verdict": "file"}], "coverage": {"attributed_turns": 1493, "ratio": 0.033}, "measurements_by_signature": {"5293e14ad0d8a437": {"cost_per_invocation": 13226, "invocations": 21, "offload_value": 277765, "share_of_plugin": 0.0688}, "7086e4bab6f9f6ec": {"cost_per_invocation": 14845, "invocations": 9, "offload_value": 133610, "share_of_plugin": 0.0331}, "975f49e56c74c70f": {"cost_per_invocation": 11377, "invocations": 9, "offload_value": 102394, "share_of_plugin": 0.0254}, "97f4fd883b379f3b": {"cost_per_invocation": 18204, "invocations": 7, "offload_value": 127433, "share_of_plugin": 0.0316}, "a22e46cfcc0b628f": {"cost_per_invocation": 15148, "invocations": 10, "offload_value": 151483, "share_of_plugin": 0.0375}}, "notes": ["anxious/issuer-agent/consecutive-agent-fanout: no-offload \u2014 position 4 \u2014 correctly done by inference. Reported as a finding, not a question.", "anxious/kanban-flow/hook-prose-drift: drift-note \u2014 hook/prose drift \u2014 a restated configuration contract, not an un-scripted algorithm. The fix is to point at the hook file, not to write a script.", "S6: 7-day window"], "rubric_version": "1.0.0", "run_id": "2026-07-29T01-00-00Z", "target": {"name": "anxious", "repo": "oleks/claude-plugin-anxious", "skills": ["agent-wip", "archivist-agent", "coach-agent", "delivery-flow", "docs-agent", "issuer-agent", "kaizen", "kanban-flow", "nudge-watch", "pomodoro", "pr-rot", "steward-agent"], "source": "path", "version": "0.32.0"}, "tool_versions": {"cc-tokens": "/home/oleks/projects/claude-plugins/token-budget/bin/cc-tokens", "inference-arbitrage": "0.3.0"}, "totals": {"invocations": 146, "share_mechanical": 0.179, "weighted_tokens": 4036235}, "window": {"requested_since": "2026-07-22T15:04:13.270540+00:00", "requested_until": null, "sessions": 69, "since": "2026-07-22T15:04:13.318000+00:00", "until": "2026-07-29T15:04:20.306000+00:00"}} diff --git a/tests/stability.test.sh b/tests/stability.test.sh new file mode 100755 index 0000000..3ec210b --- /dev/null +++ b/tests/stability.test.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash +# bin/stability-classify: the FR-4.5 evidence-stability gate. Three synthetic +# cases with hand-written snapshots, then the real two-window `anxious` evidence +# that motivated the gate (kotkan/claude-plugin-inference-arbitrage#11) — the +# repeatable form of acceptance criterion S4. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + +BIN=../bin/stability-classify +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +fail=0 + +check() { + local what=$1 got=$2 want=$3 + if [ "$got" = "$want" ]; then + echo " ok $what = $want" + else + echo " FAIL $what = $got (want $want)" >&2 + fail=1 + fi +} + +# field +field() { + python3 -c " +import json,sys +d=json.load(open(sys.argv[1])) +c=[x for x in d['candidates'] if x['candidate_id']==sys.argv[2]][0] +print(c.get(sys.argv[3]))" "$@" +} +gatefield() { + python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['stability_gate'][sys.argv[2]])" "$@" +} + +# classified +# One position-3 candidate the boundary gate already graded `file`. +classified() { + python3 - "$1" "$2" "$3" <<'EOF' +import json, sys +out, strength, share = sys.argv[1:] +json.dump({ + "rubric_version": "1.0.0", + "target": "synth", + "summary": {"candidates": 1, "high_confidence": 1, "to_file": 1, + "boundary_questions": 0, "no_offload": 0, "drift_notes": 0}, + "candidates": [{ + "candidate_id": "synth/skill/step", + "skill": "synth:skill", + "position": "llm-over-script-digest", + "boundary_confidence": "high", + "measurement_strength": strength, + "share_of_audited_spend": float(share), + "offload_value": 500000, + "verdict": "file", + "reasons": ["all five determinism tests pass; triple complete"], + }], +}, open(out, "w")) +EOF +} + +# snapshot-store +# A prior snapshot recording the same candidate by slug identity. +prior() { + local store=$1 + mkdir -p "$store/Data/synth" + python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" <<'EOF' +import json, sys +out, strength, share = sys.argv[1:] +json.dump({ + "run_id": "2026-07-01T00-00-00Z", + "window": {"since": "2026-06-01", "until": "2026-07-01"}, + "target": {"name": "synth", "version": "1.0.0", "skills": ["synth:skill"]}, + "coverage": {"ratio": 0.1, "attributed_turns": 500}, + "totals": {"weighted_tokens": 5000000}, + "candidates": [{ + "candidate_id": "synth/skill/step", + "skill": "synth:skill", + "signature_hash": "slugonly", + "measurement_strength": strength, + "measurement": {"invocations": 10, "offload_value": 500000, + "share_of_plugin": float(share), "cost_per_invocation": 50000}, + "issue": None, + "verdict": "file", + }], + "boundary_questions": [], + "measurements_by_signature": {}, + "notes": [], +}, open(out, "w")) +EOF +} + +# -------------------------------------------------------------------------- +echo "== (a) measured now, thin in the prior window -> downgraded, with a finding ==" +# The shape of the real anxious case: same candidate, same rubric grade, two +# different evidence labels because the windows differ. +classified "$tmp/a.json" measured 0.12 +prior "$tmp/store-a" thin 0.0 +$BIN --classified "$tmp/a.json" --target synth --store "$tmp/store-a" >"$tmp/a.out.json" +check "verdict" "$(field "$tmp/a.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/a.out.json" downgraded)" "1" +check "checked" "$(gatefield "$tmp/a.out.json" checked)" "1" +python3 -c " +import json +d=json.load(open('$tmp/a.out.json')) +f=d['stability_findings'][0] +fields={x['field'] for x in f['flips']} +assert fields == {'measurement_strength','share_of_audited_spend'}, fields +assert f['prior']['measurement_strength']=='thin' +assert f['current']['measurement_strength']=='measured' +assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage' +assert f['self_report']['marker']=='' +print(' ok finding names both flips, and self-reports with a marker') +" || fail=1 +python3 -c " +import json +d=json.load(open('$tmp/a.out.json')) +r=d['candidates'][0]['reasons'][-1] +assert 'HARD GATE (FR-4.5)' in r and 'never filed' in r, r +assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==1, d['summary'] +print(' ok reason cites the gate; summary counts are re-derived') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (b) same label, same side of the threshold -> stays filed ==" +# 12% then 8% is a trend, not an instability: `audit-snapshot diff` reports the +# movement, and the human is told the same thing either way. +classified "$tmp/b.json" measured 0.08 +prior "$tmp/store-b" measured 0.12 +$BIN --classified "$tmp/b.json" --target synth --store "$tmp/store-b" >"$tmp/b.out.json" +check "verdict" "$(field "$tmp/b.out.json" synth/skill/step verdict)" "file" +check "downgraded" "$(gatefield "$tmp/b.out.json" downgraded)" "0" +check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/b.out.json'))['stability_findings']))")" "0" +python3 -c " +import json +s=json.load(open('$tmp/b.out.json'))['candidates'][0]['stability'] +assert s.startswith('stable against 2026-07-01T00-00-00Z'), s +print(' ok stability names the window it was checked against') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (c) no prior snapshot -> files normally, unchecked (FR-2.4) ==" +# An absence of history is not evidence of instability. A first audit of a +# target must still be able to file. +classified "$tmp/c.json" measured 0.12 +$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" >"$tmp/c.out.json" +check "verdict" "$(field "$tmp/c.out.json" synth/skill/step verdict)" "file" +check "stability" "$(field "$tmp/c.out.json" synth/skill/step stability)" \ + "unchecked — no prior window to compare" +check "checked" "$(gatefield "$tmp/c.out.json" checked)" "0" + +# -------------------------------------------------------------------------- +echo "== (d) history exists but not for this candidate -> unchecked, still filed ==" +classified "$tmp/d.json" measured 0.12 +python3 -c " +import json +d=json.load(open('$tmp/d.json')) +d['candidates'][0]['candidate_id']='synth/other/step' +json.dump(d,open('$tmp/d.json','w'))" +prior "$tmp/store-d" thin 0.0 +$BIN --classified "$tmp/d.json" --target synth --store "$tmp/store-d" >"$tmp/d.out.json" +check "verdict" "$(field "$tmp/d.out.json" synth/other/step verdict)" "file" +check "stability" "$(field "$tmp/d.out.json" synth/other/step stability)" \ + "unchecked — no prior window recorded this candidate" + +# -------------------------------------------------------------------------- +echo "== (e) only 'file' verdicts are gated ==" +# A boundary question is already not being filed; re-gating it would double-count +# and would put a stability finding on something no human is being shown. +classified "$tmp/e.json" measured 0.12 +python3 -c " +import json +d=json.load(open('$tmp/e.json')) +d['candidates'][0]['verdict']='boundary-question' +json.dump(d,open('$tmp/e.json','w'))" +prior "$tmp/store-e" thin 0.0 +$BIN --classified "$tmp/e.json" --target synth --store "$tmp/store-e" >"$tmp/e.out.json" +check "untouched verdict" "$(field "$tmp/e.out.json" synth/skill/step verdict)" "boundary-question" +check "not counted as checked" "$(gatefield "$tmp/e.out.json" checked)" "0" +check "no finding" "$(gatefield "$tmp/e.out.json" downgraded)" "0" + +# -------------------------------------------------------------------------- +echo "== (f) S4 — the real anxious candidate is caught with no human in the loop ==" +# The exact evidence from kotkan/claude-plugin-inference-arbitrage#11: the +# 30-day window graded `anxious/agent-wip/release-policy-derivation` measured at +# 11.94% of audited spend, the 7-day window graded the same candidate thin at +# 0.00%. Both windows' snapshots are in the store; --exclude-run makes the 7-day +# run the one being classified, exactly as it is at filing time. +store=$tmp/store-anxious +mkdir -p "$store/Data/anxious" +cp calibration/anxious-windows.snapshots.jsonl "$store/Data/anxious/snapshots.jsonl" +$BIN --classified calibration/anxious-7d.classified.json \ + --target anxious --store "$store" \ + --scan calibration/anxious-7d.scan.json \ + --exclude-run 2026-07-29T01-00-00Z >"$tmp/f.out.json" +check "verdict" "$(field "$tmp/f.out.json" anxious/agent-wip/release-policy-derivation verdict)" \ + "boundary-question" +check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "1" +python3 -c " +import json +d=json.load(open('$tmp/f.out.json')) +f=[x for x in d['stability_findings'] + if x['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0] +assert f['prior']['measurement_strength']=='measured', f['prior'] +assert abs(f['prior']['share_of_audited_spend']-0.1194) < 1e-9, f['prior'] +assert f['current']['measurement_strength']=='thin', f['current'] +assert f['current']['share_of_audited_spend']==0.0, f['current'] +assert f['prior']['run_id']=='2026-07-29T00-00-00Z', f['prior'] +print(' ok finding carries the real numbers: measured 11.94% vs thin 0.00%') +" || fail=1 +python3 -c " +import json +d=json.load(open('$tmp/f.out.json')) +assert d['summary']['to_file']==0, d['summary'] +print(' ok nothing is left to file from that window') +" || fail=1 + +exit "$fail"