kotkan/claude-plugin-inference-arbitrage
3
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

FR-4.5: evidence-stability gate — refuse to file a candidate whose confidence moved with the window

Browse Source 
Phase 7 filed anxious/agent-wip/release-policy-derivation as `measured` at
11.94% of audited spend on a 30-day window. The same candidate over 7 days came
back `thin` at 0.00%. Only the date range differed. A human reading either issue
body alone cannot know the other exists, which is rubric.md §4a's failure mode
reached through the plugin's own headline number.

That was caught by a human running a second skeptical review — luck, not a
control. It is now mechanical, in the same register as FR-4.2's overrule-case
gate:

- bin/stability-classify matches each `file` candidate against the most recent
  prior snapshot recording it (FR-5.2 identity) and downgrades it to a boundary
  question if `measurement_strength` changed or the share crossed the 2% filing
  threshold between windows. Movement within a label is a trend, not an
  instability; no prior window is `unchecked`, not unstable (FR-2.4).
- The downgrade is a finding against THIS repo, not the target's — the target
  did not change, the auditor described it two ways. Carries an `ia-stability`
  marker so a re-run comments rather than duplicating.
- bin/ia_store.py factors the store and identity resolution out of
  audit-snapshot so both scripts hash signatures identically. A divergence there
  would make the gate match nothing and fail open.

Documented as spec FR-4.5 and rubric §5b (shipped verbatim in references/).
Wired into skills/offload-audit as step 4b, before filing-plan; steps 5-6 now
consume its output rather than classified.json.

S4 is retired in its old form — a human triage catching a bad candidate is not a
repeatable test — and re-passed as: the gate catches the real
release-policy-derivation case with no human in the loop. Asserted in
tests/stability.test.sh case (f) against the verbatim two-window Phase 7 output.

Closes kotkan/claude-plugin-inference-arbitrage#11
Suite: 178 assertions, exit 0.
This commit is contained in:
oleks
2026-07-29 20:24:31 +03:00
committed by Oleks
parent 105e9f5bae
commit dcec9cabfb
16 changed files with 1119 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/audit-snapshot
+4 -108
View File
@@ -39,20 +39,15 @@ the whole file, not a delta.
"""
import argparse
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
DEFAULT_STORE = "~/.cache/inference-arbitrage/wiki"
# Both mirror bin/boundary-classify deliberately: a candidate must clear the
# same bar to be called fixed that it had to clear to be filed. Changing one
# without the other would make "resolved" mean something the filing gate never
# meant.
MIN_SHARE_OF_SPEND = 0.02
MIN_INVOCATIONS = 3
from ia_store import (DEFAULT_STORE, MIN_INVOCATIONS, MIN_SHARE_OF_SPEND,
audits_dir, build_index, candidate_signature, find_match,
load_snapshots, scan_rows, signature_hash, snapshots_path,
store_root, tiered)
# Relative change in the volume-normalized metrics below which a candidate is
# `persisting` rather than grown/shrunk. Sampling noise across two windows of
@@ -71,36 +66,6 @@ STATUSES = ("new", "persisting", "grown", "shrunk", "resolved",
# store
# --------------------------------------------------------------------------
def store_root(store):
return os.path.expanduser(store)
def snapshots_path(store, target):
return os.path.join(store_root(store), "Data", target, "snapshots.jsonl")
def audits_dir(store, target):
return os.path.join(store_root(store), "Audits", target)
def load_snapshots(store, target):
path = snapshots_path(store, target)
if not os.path.exists(path):
return []
out = []
with open(path) as fh:
for lineno, line in enumerate(fh, 1):
line = line.strip()
if not line:
continue
try:
out.append(json.loads(line))
except json.JSONDecodeError as exc:
sys.exit(f"error: {path}:{lineno}: corrupt snapshot record: {exc}")
out.sort(key=lambda s: s.get("run_id", ""))
return out
def append_snapshot(store, target, snap):
path = snapshots_path(store, target)
os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -116,47 +81,6 @@ def write_text(path, text):
return path
# --------------------------------------------------------------------------
# identity (FR-5.2)
# --------------------------------------------------------------------------
def signature_hash(sig):
"""Hash of a normalized tool-sequence signature. `sig` is the list of
masked signatures offload-scan already emits, so normalization happened
upstream; joining them is all that is left."""
return hashlib.sha256("\n".join(sig).encode()).hexdigest()[:16]
def identity(cand):
"""FR-5.2 resolution order. Issue number first: it is the only identifier
a human also uses, and it survives a refactor that moves the code."""
if cand.get("issue"):
return ("issue", cand["issue"])
if cand.get("signature_hash"):
return ("signature", cand["signature_hash"])
return ("slug", cand.get("candidate_id"))
def build_index(cands):
idx = {"issue": {}, "signature": {}, "slug": {}}
for c in cands:
if c.get("issue"):
idx["issue"].setdefault(c["issue"], c)
if c.get("signature_hash"):
idx["signature"].setdefault(c["signature_hash"], c)
idx["slug"].setdefault(c.get("candidate_id"), c)
return idx
def find_match(cand, idx):
for kind, key in (("issue", cand.get("issue")),
("signature", cand.get("signature_hash")),
("slug", cand.get("candidate_id"))):
if key and key in idx[kind]:
return idx[kind][key], kind
return None, None
# --------------------------------------------------------------------------
# write
# --------------------------------------------------------------------------
@@ -171,13 +95,6 @@ def tool_version():
return None
def scan_rows(scan):
for row in scan.get("skills", []):
yield row.get("skill"), row
for row in scan.get("agents", []):
yield row.get("agent"), row
def measurements_by_signature(scan):
"""Every recurring signature the scan saw, keyed by hash — including ones
that no longer classify as candidates. FR-5.5 needs exactly this: a later
@@ -198,20 +115,6 @@ def measurements_by_signature(scan):
return out
def candidate_signature(cand, scan, explicit):
"""Signature resolution, most trustworthy first: an explicitly supplied
tool sequence, then the dominant recurring n-gram of the candidate's own
skill, then the slug. The source is recorded because a slug-derived
identity cannot detect drift — it IS the thing drift is measured against."""
cid = cand.get("candidate_id")
if explicit.get(cid):
return list(explicit[cid]), "explicit"
for name, row in scan_rows(scan):
if name == cand.get("skill") and row.get("ngrams"):
return list(row["ngrams"][0]["sig"]), "scan-ngram"
return [f"slug:{cid}"], "slug"
def snapshot_candidate(cand, scan, explicit, issue_map):
sig, source = candidate_signature(cand, scan, explicit)
row = next((r for n, r in scan_rows(scan) if n == cand.get("skill")), {})
@@ -415,13 +318,6 @@ def row_of(cand, tier, status, reason, matched_by, prev_cand=None):
}
def tiered(snap):
for c in snap.get("candidates", []):
yield c, "candidate"
for c in snap.get("boundary_questions", []):
yield c, "boundary-question"
def diff_snapshots(prev, curr, issue_state):
prev_all = list(tiered(prev))
idx = build_index([c for c, _ in prev_all])
bin/ia_store.py
+143
View File
@@ -0,0 +1,143 @@
"""Snapshot store access and candidate-identity resolution (FR-5.2), shared by
`bin/audit-snapshot` (which writes and diffs snapshots) and
`bin/stability-classify` (which reads them to gate filing, FR-4.5).
It exists so that identity is computed exactly once. The gate's whole job is to
find the *same* candidate in an earlier window; if it hashed a signature even
slightly differently from the writer, it would silently find nothing and pass
every unstable candidate through — a gate that fails open and looks like it
worked. Copy-pasting `signature_hash` between the two scripts is therefore not
a style problem, it is the failure mode.
Imported by name from `bin/`, which is `sys.path[0]` for a script run as
`bin/<tool>`.
"""
import hashlib
import json
import os
import sys
DEFAULT_STORE = "~/.cache/inference-arbitrage/wiki"
# spec §5.2.7's filing threshold, mirroring bin/boundary-classify — a candidate
# must clear the same bar to be called fixed, or stable, that it had to clear to
# be filed. Changing one without the others would make those words mean
# something the filing gate never meant.
MIN_SHARE_OF_SPEND = 0.02
MIN_INVOCATIONS = 3
# --------------------------------------------------------------------------
# store
# --------------------------------------------------------------------------
def store_root(store):
return os.path.expanduser(store)
def snapshots_path(store, target):
return os.path.join(store_root(store), "Data", target, "snapshots.jsonl")
def audits_dir(store, target):
return os.path.join(store_root(store), "Audits", target)
def load_snapshots(store, target):
path = snapshots_path(store, target)
if not os.path.exists(path):
return []
out = []
with open(path) as fh:
for lineno, line in enumerate(fh, 1):
line = line.strip()
if not line:
continue
try:
out.append(json.loads(line))
except json.JSONDecodeError as exc:
sys.exit(f"error: {path}:{lineno}: corrupt snapshot record: {exc}")
out.sort(key=lambda s: s.get("run_id", ""))
return out
# --------------------------------------------------------------------------
# identity (FR-5.2)
# --------------------------------------------------------------------------
def signature_hash(sig):
"""Hash of a normalized tool-sequence signature. `sig` is the list of
masked signatures offload-scan already emits, so normalization happened
upstream; joining them is all that is left."""
return hashlib.sha256("\n".join(sig).encode()).hexdigest()[:16]
def identity(cand):
"""FR-5.2 resolution order. Issue number first: it is the only identifier
a human also uses, and it survives a refactor that moves the code."""
if cand.get("issue"):
return ("issue", cand["issue"])
if cand.get("signature_hash"):
return ("signature", cand["signature_hash"])
return ("slug", cand.get("candidate_id"))
def build_index(cands):
idx = {"issue": {}, "signature": {}, "slug": {}}
for c in cands:
if c.get("issue"):
idx["issue"].setdefault(c["issue"], c)
if c.get("signature_hash"):
idx["signature"].setdefault(c["signature_hash"], c)
idx["slug"].setdefault(c.get("candidate_id"), c)
return idx
def find_match(cand, idx):
for kind, key in (
("issue", cand.get("issue")),
("signature", cand.get("signature_hash")),
("slug", cand.get("candidate_id")),
):
if key and key in idx[kind]:
return idx[kind][key], kind
return None, None
def tiered(snap):
"""Both tiers of a snapshot. A candidate that was a boundary question last
window and a filing candidate this one is the same candidate, so identity
matching must see both piles."""
for c in snap.get("candidates", []):
yield c, "candidate"
for c in snap.get("boundary_questions", []):
yield c, "boundary-question"
# --------------------------------------------------------------------------
# signatures derived from a scan
# --------------------------------------------------------------------------
def scan_rows(scan):
for row in scan.get("skills", []):
yield row.get("skill"), row
for row in scan.get("agents", []):
yield row.get("agent"), row
def candidate_signature(cand, scan, explicit):
"""Signature resolution, most trustworthy first: an explicitly supplied
tool sequence, then the dominant recurring n-gram of the candidate's own
skill, then the slug. The source is recorded because a slug-derived
identity cannot detect drift — it IS the thing drift is measured against."""
cid = cand.get("candidate_id")
if explicit.get(cid):
return list(explicit[cid]), "explicit"
for name, row in scan_rows(scan):
if name == cand.get("skill") and row.get("ngrams"):
return list(row["ngrams"][0]["sig"]), "scan-ngram"
return [f"slug:{cid}"], "slug"
bin/stability-classify
Executable
+252
View File
@@ -0,0 +1,252 @@
#!/usr/bin/env python3
"""The evidence-stability gate (FR-4.5, rubric §5b): refuses to file a candidate
whose evidence label is an artifact of which measurement window happened to be
audited.
WHY THIS IS A GATE AND NOT A CAVEAT
-----------------------------------
Phase 7 filed `anxious/agent-wip/release-policy-derivation` on a 30-day window
as `measurement_strength: measured`, 11.94% of audited spend — a confident,
concrete cost claim. The identical candidate, classified over a 7-day window,
came back `thin`, 0.00% of spend. Nothing about the plugin changed between the
two; only the date range did.
A human reading either issue body alone has no way to know the other exists.
That is the `rubric.md` §4a failure — fast, confident, and wrong, with nobody
watching — reached by the plugin's own headline number. It was caught once, by
a human running a second skeptical review, which is not a repeatable control:
the next candidate gets filed only if someone happens to be suspicious that day.
So stability becomes mechanical, in the same register as FR-4.2's overrule-case
gate: a candidate whose evidence flips across windows is downgraded to a
boundary question and never filed, and the flip is emitted as a finding against
this plugin's own repo (FR-8.2), because a confidence label that moves with the
calendar is a bug in the auditor, not a fact about the target.
WHAT COUNTS AS A FLIP
---------------------
Two things, either of which disqualifies:
1. `measurement_strength` differs between the windows (`measured` vs `thin`
vs `unmeasured`). This is the label the issue body speaks in, so a change
here changes what a human is told.
2. `share_of_audited_spend` lands on opposite sides of MIN_SHARE_OF_SPEND.
That threshold is the filing gate itself (spec §5.2.7): if one window would
have filed on value and the other would not, the value claim is not
load-bearing evidence, it is a window artifact.
Ordinary movement within a band is NOT a flip. A candidate that measures 8% one
window and 5% the next is stable — same label, same side of the threshold — and
that variation is what `audit-snapshot diff` exists to report as a trend. This
gate fires only when the *description a human receives* changes.
NO PRIOR WINDOW MEANS UNCHECKED, NOT UNSTABLE
---------------------------------------------
A first-ever audit of a target, or a candidate seen for the first time, has
nothing to compare against. It files normally, marked
`stability: unchecked`. Manufacturing an instability finding out of an absence
of history would contradict FR-2.4 — a target with no history stays auditable on
its own terms — and would make the first audit of every plugin file nothing.
The prior window is the most recent snapshot that contains this candidate by
FR-5.2 identity (issue > signature hash > slug), not merely the most recent
snapshot: a candidate that sat out one window is still comparable against the
window before it, and using the latest snapshot unconditionally would report
every such candidate as unmatched and let it through.
Runs after `boundary-classify` and before `filing-plan`; emits the same document
shape it consumes, with verdicts revised, so it drops into the pipeline in place
of `classified.json`.
"""
import argparse
import json
import sys
from ia_store import (DEFAULT_STORE, MIN_SHARE_OF_SPEND, build_index,
candidate_signature, find_match, load_snapshots,
signature_hash, tiered)
GATE = "FR-4.5"
UNCHECKED_NO_HISTORY = "unchecked — no prior window to compare"
UNCHECKED_NO_MATCH = "unchecked — no prior window recorded this candidate"
def keyed(cand, scan, explicit, issue_map):
"""The identity fields FR-5.2 matches on, computed for a candidate that is
only classified and not yet snapshotted. Derived with the same
`candidate_signature` the snapshot writer uses — a divergence here would
make the gate silently match nothing and pass every unstable candidate."""
sig, source = candidate_signature(cand, scan, explicit)
return {
"candidate_id": cand.get("candidate_id"),
"issue": issue_map.get(cand.get("candidate_id")) or cand.get("issue"),
"signature_hash": signature_hash(sig),
"signature_source": source,
}
def find_prior(key, history):
"""Most recent snapshot holding this candidate, newest first."""
for snap in reversed(history):
idx = build_index([c for c, _ in tiered(snap)])
match, by = find_match(key, idx)
if match:
return snap, match, by
return None, None, None
def crosses_threshold(a, b):
return (a >= MIN_SHARE_OF_SPEND) != (b >= MIN_SHARE_OF_SPEND)
def flips(prior, curr_strength, curr_share):
"""Named, so the finding says exactly what moved rather than 'unstable'."""
out = []
prior_strength = prior.get("measurement_strength")
prior_share = (prior.get("measurement") or {}).get("share_of_plugin", 0.0) or 0.0
if prior_strength != curr_strength:
out.append({
"field": "measurement_strength",
"prior": prior_strength,
"current": curr_strength,
"why": ("the evidence label the issue body speaks in changed with the window, "
"so the same candidate would be described differently to a human "
"depending only on when the audit ran"),
})
if crosses_threshold(prior_share, curr_share):
out.append({
"field": "share_of_audited_spend",
"prior": prior_share,
"current": curr_share,
"threshold": MIN_SHARE_OF_SPEND,
"why": (f"the windows land on opposite sides of the {MIN_SHARE_OF_SPEND:.0%} "
"filing threshold — one window would file on value and the other "
"would not, so the value claim is a window artifact"),
})
return out
def gate(doc, history, scan, explicit, issue_map):
target = doc.get("target")
window = scan.get("window") or {}
findings = []
checked = 0
for cand in doc.get("candidates", []):
if cand.get("verdict") != "file":
continue
if not history:
cand["stability"] = UNCHECKED_NO_HISTORY
continue
key = keyed(cand, scan, explicit, issue_map)
snap, prior, by = find_prior(key, history)
if prior is None:
cand["stability"] = UNCHECKED_NO_MATCH
continue
checked += 1
curr_strength = cand.get("measurement_strength")
curr_share = cand.get("share_of_audited_spend", 0.0) or 0.0
moved = flips(prior, curr_strength, curr_share)
if not moved:
cand["stability"] = (
f"stable against {snap.get('run_id')} (matched by {by}): "
f"{curr_strength}, {curr_share:.2%} of audited spend either side")
continue
cand["verdict"] = "boundary-question"
cand["stability"] = f"unstable against {snap.get('run_id')} (matched by {by})"
cand.setdefault("reasons", []).append(
f"HARD GATE ({GATE}): evidence is not stable across measurement windows — "
+ "; ".join(f"{f['field']} {f['prior']!r} -> {f['current']!r}" for f in moved)
+ ". Downgraded to a boundary question, never filed."
)
findings.append({
"gate": GATE,
"candidate_id": cand.get("candidate_id"),
"target": target,
"skill": cand.get("skill"),
"matched_by": by,
"prior": {
"run_id": snap.get("run_id"),
"window": snap.get("window"),
"measurement_strength": prior.get("measurement_strength"),
"share_of_audited_spend": (prior.get("measurement") or {}).get("share_of_plugin"),
},
"current": {
"window": window,
"measurement_strength": curr_strength,
"share_of_audited_spend": curr_share,
},
"flips": moved,
"headline": (
f"{cand.get('candidate_id')} is {prior.get('measurement_strength')} at "
f"{((prior.get('measurement') or {}).get('share_of_plugin') or 0):.2%} of spend in "
f"{snap.get('run_id')} and {curr_strength} at {curr_share:.2%} in this window — "
"the confidence label depends on the measurement window, not on the candidate"),
# FR-8.2: the instability is a finding about THIS plugin, filed on
# its own repo. The marker mirrors FR-6.3's `ia-candidate` so a
# re-run comments instead of duplicating.
"self_report": {
"repo": "kotkan/claude-plugin-inference-arbitrage",
"marker": f"<!-- ia-stability: {cand.get('candidate_id')} -->",
},
})
rows = doc.get("candidates", [])
summary = doc.setdefault("summary", {})
summary["to_file"] = sum(1 for c in rows if c.get("verdict") == "file")
summary["boundary_questions"] = sum(1 for c in rows if c.get("verdict") == "boundary-question")
summary["stability_checked"] = checked
summary["stability_downgraded"] = len(findings)
doc["stability_gate"] = {
"gate": GATE,
"snapshots_available": len(history),
"checked": checked,
"downgraded": len(findings),
}
doc["stability_findings"] = findings
return doc
def load_json(path):
with open(path) as fh:
return json.load(fh)
def main():
ap = argparse.ArgumentParser(prog="stability-classify")
ap.add_argument("--classified", required=True, help="bin/boundary-classify output")
ap.add_argument("--target", required=True, help="target plugin name, as snapshotted")
ap.add_argument("--store", default=DEFAULT_STORE,
help=f"local mirror of the wiki store (default {DEFAULT_STORE})")
ap.add_argument("--scan", help="bin/offload-scan output, for signature identity")
ap.add_argument("--signatures", help="JSON {candidate_id: [tool signature, ...]}")
ap.add_argument("--issue-map", help='JSON {candidate_id: "owner/repo#N"}')
ap.add_argument("--exclude-run", help="run_id to treat as not-yet-recorded (tests, re-runs)")
ap.add_argument("--json", action="store_true", help="compact single-line JSON")
args = ap.parse_args()
doc = load_json(args.classified)
history = [s for s in load_snapshots(args.store, args.target)
if s.get("run_id") != args.exclude_run]
result = gate(doc, history,
load_json(args.scan) if args.scan else {},
load_json(args.signatures) if args.signatures else {},
load_json(args.issue_map) if args.issue_map else {})
json.dump(result, sys.stdout, indent=None if args.json else 2)
sys.stdout.write("\n")
if __name__ == "__main__":
main()