Oleks
69ff443bd2
bin/stability-classify only ever reads the local filesystem snapshot store
(by design, no bin/ script holds wiki credentials), but SKILL.md step 4b
documented seeding it from the wiki as a conditional "if the store is cold"
aside. That made a cold-but-not-actually-empty local cache indistinguishable
from a genuine first audit: both silently emit stability: unchecked and file
unstable candidates, as observed on the 2026-07-30 anxious run.
- bin/stability-classify: add a required --wiki-checked {empty,imported}
flag; hard-error when the store is empty and the flag is omitted, instead
of silently degrading to unchecked.
- tests/stability.test.sh: (g) no flag on an empty store errors, (h)
--wiki-checked=empty behaves as before, (i) --wiki-checked=imported after a
real `audit-snapshot import` runs the gate normally against the imported
history.
- skills/offload-audit/SKILL.md: step 4b now fetches the wiki's
Data/<target>/snapshots.jsonl and runs `audit-snapshot import`
unconditionally (it dedupes by run_id, so this is safe every run) before
invoking the gate.
- Bump plugin.json to 0.6.3.
270 lines
12 KiB
Bash
Executable File
270 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/stability-classify: the FR-4.5 evidence-stability gate. Three synthetic
|
|
# cases with hand-written snapshots, then the real two-window `anxious` evidence
|
|
# that motivated the gate (kotkan/claude-plugin-inference-arbitrage#11) — the
|
|
# repeatable form of acceptance criterion S4.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/stability-classify
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# field <json> <candidate_id> <key>
|
|
field() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
c=[x for x in d['candidates'] if x['candidate_id']==sys.argv[2]][0]
|
|
print(c.get(sys.argv[3]))" "$@"
|
|
}
|
|
gatefield() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['stability_gate'][sys.argv[2]])" "$@"
|
|
}
|
|
|
|
# classified <file> <strength> <share>
|
|
# One position-3 candidate the boundary gate already graded `file`.
|
|
classified() {
|
|
python3 - "$1" "$2" "$3" <<'EOF'
|
|
import json, sys
|
|
out, strength, share = sys.argv[1:]
|
|
json.dump({
|
|
"rubric_version": "1.0.0",
|
|
"target": "synth",
|
|
"summary": {"candidates": 1, "high_confidence": 1, "to_file": 1,
|
|
"boundary_questions": 0, "no_offload": 0, "drift_notes": 0},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"position": "llm-over-script-digest",
|
|
"boundary_confidence": "high",
|
|
"measurement_strength": strength,
|
|
"share_of_audited_spend": float(share),
|
|
"offload_value": 500000,
|
|
"verdict": "file",
|
|
"reasons": ["all five determinism tests pass; triple complete"],
|
|
}],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# snapshot-store <store> <strength> <share>
|
|
# A prior snapshot recording the same candidate by slug identity.
|
|
prior() {
|
|
local store=$1
|
|
mkdir -p "$store/Data/synth"
|
|
python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" <<'EOF'
|
|
import json, sys
|
|
out, strength, share = sys.argv[1:]
|
|
json.dump({
|
|
"run_id": "2026-07-01T00-00-00Z",
|
|
"window": {"since": "2026-06-01", "until": "2026-07-01"},
|
|
"target": {"name": "synth", "version": "1.0.0", "skills": ["synth:skill"]},
|
|
"coverage": {"ratio": 0.1, "attributed_turns": 500},
|
|
"totals": {"weighted_tokens": 5000000},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"position": "llm-over-script-digest",
|
|
"boundary_confidence": "high",
|
|
"signature_hash": "slugonly",
|
|
"measurement_strength": strength,
|
|
"measurement": {"invocations": 10, "offload_value": 500000,
|
|
"share_of_plugin": float(share), "cost_per_invocation": 50000},
|
|
"issue": None,
|
|
"verdict": "file",
|
|
}],
|
|
"boundary_questions": [],
|
|
"measurements_by_signature": {},
|
|
"notes": [],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (a) measured now, thin in the prior window -> downgraded, with a finding =="
|
|
# The shape of the real anxious case: same candidate, same rubric grade, two
|
|
# different evidence labels because the windows differ.
|
|
classified "$tmp/a.json" measured 0.12
|
|
prior "$tmp/store-a" thin 0.0
|
|
$BIN --classified "$tmp/a.json" --target synth --store "$tmp/store-a" >"$tmp/a.out.json"
|
|
check "verdict" "$(field "$tmp/a.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/a.out.json" downgraded)" "1"
|
|
check "checked" "$(gatefield "$tmp/a.out.json" checked)" "1"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
f=d['stability_findings'][0]
|
|
fields={x['field'] for x in f['flips']}
|
|
assert fields == {'measurement_strength','share_of_audited_spend'}, fields
|
|
assert f['prior']['measurement_strength']=='thin'
|
|
assert f['current']['measurement_strength']=='measured'
|
|
assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage'
|
|
assert f['self_report']['marker']=='<!-- ia-stability: synth/skill/step -->'
|
|
print(' ok finding names both flips, and self-reports with a marker')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
r=d['candidates'][0]['reasons'][-1]
|
|
assert 'HARD GATE (FR-4.5)' in r and 'never filed' in r, r
|
|
assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==1, d['summary']
|
|
print(' ok reason cites the gate; summary counts are re-derived')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (b) same label, same side of the threshold -> stays filed =="
|
|
# 12% then 8% is a trend, not an instability: `audit-snapshot diff` reports the
|
|
# movement, and the human is told the same thing either way.
|
|
classified "$tmp/b.json" measured 0.08
|
|
prior "$tmp/store-b" measured 0.12
|
|
$BIN --classified "$tmp/b.json" --target synth --store "$tmp/store-b" >"$tmp/b.out.json"
|
|
check "verdict" "$(field "$tmp/b.out.json" synth/skill/step verdict)" "file"
|
|
check "downgraded" "$(gatefield "$tmp/b.out.json" downgraded)" "0"
|
|
check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/b.out.json'))['stability_findings']))")" "0"
|
|
python3 -c "
|
|
import json
|
|
s=json.load(open('$tmp/b.out.json'))['candidates'][0]['stability']
|
|
assert s.startswith('stable against 2026-07-01T00-00-00Z'), s
|
|
print(' ok stability names the window it was checked against')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (c) no prior snapshot -> files normally, unchecked (FR-2.4) =="
|
|
# An absence of history is not evidence of instability. A first audit of a
|
|
# target must still be able to file. The caller must still assert the wiki
|
|
# was checked (kotkan/claude-plugin-inference-arbitrage#17) -- see (g)/(h).
|
|
classified "$tmp/c.json" measured 0.12
|
|
$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" \
|
|
--wiki-checked=empty >"$tmp/c.out.json"
|
|
check "verdict" "$(field "$tmp/c.out.json" synth/skill/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/c.out.json" synth/skill/step stability)" \
|
|
"unchecked — no prior window to compare"
|
|
check "checked" "$(gatefield "$tmp/c.out.json" checked)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (d) history exists but not for this candidate -> unchecked, still filed =="
|
|
classified "$tmp/d.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/d.json'))
|
|
d['candidates'][0]['candidate_id']='synth/other/step'
|
|
json.dump(d,open('$tmp/d.json','w'))"
|
|
prior "$tmp/store-d" thin 0.0
|
|
$BIN --classified "$tmp/d.json" --target synth --store "$tmp/store-d" >"$tmp/d.out.json"
|
|
check "verdict" "$(field "$tmp/d.out.json" synth/other/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/d.out.json" synth/other/step stability)" \
|
|
"unchecked — no prior window recorded this candidate"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (e) only 'file' verdicts are gated =="
|
|
# A boundary question is already not being filed; re-gating it would double-count
|
|
# and would put a stability finding on something no human is being shown.
|
|
classified "$tmp/e.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/e.json'))
|
|
d['candidates'][0]['verdict']='boundary-question'
|
|
json.dump(d,open('$tmp/e.json','w'))"
|
|
prior "$tmp/store-e" thin 0.0
|
|
$BIN --classified "$tmp/e.json" --target synth --store "$tmp/store-e" >"$tmp/e.out.json"
|
|
check "untouched verdict" "$(field "$tmp/e.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "not counted as checked" "$(gatefield "$tmp/e.out.json" checked)" "0"
|
|
check "no finding" "$(gatefield "$tmp/e.out.json" downgraded)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (f) S4 — the real anxious candidate is caught with no human in the loop =="
|
|
# The exact evidence from kotkan/claude-plugin-inference-arbitrage#11: the
|
|
# 30-day window graded `anxious/agent-wip/release-policy-derivation` measured at
|
|
# 11.94% of audited spend, the 7-day window graded the same candidate thin at
|
|
# 0.00%. Both windows' snapshots are in the store; --exclude-run makes the 7-day
|
|
# run the one being classified, exactly as it is at filing time.
|
|
store=$tmp/store-anxious
|
|
mkdir -p "$store/Data/anxious"
|
|
cp calibration/anxious-windows.snapshots.jsonl "$store/Data/anxious/snapshots.jsonl"
|
|
$BIN --classified calibration/anxious-7d.classified.json \
|
|
--target anxious --store "$store" \
|
|
--scan calibration/anxious-7d.scan.json \
|
|
--exclude-run 2026-07-29T01-00-00Z >"$tmp/f.out.json"
|
|
check "verdict" "$(field "$tmp/f.out.json" anxious/agent-wip/release-policy-derivation verdict)" \
|
|
"boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "1"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
f=[x for x in d['stability_findings']
|
|
if x['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0]
|
|
assert f['prior']['measurement_strength']=='measured', f['prior']
|
|
assert abs(f['prior']['share_of_audited_spend']-0.1194) < 1e-9, f['prior']
|
|
assert f['current']['measurement_strength']=='thin', f['current']
|
|
assert f['current']['share_of_audited_spend']==0.0, f['current']
|
|
assert f['prior']['run_id']=='2026-07-29T00-00-00Z', f['prior']
|
|
print(' ok finding carries the real numbers: measured 11.94% vs thin 0.00%')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
assert d['summary']['to_file']==0, d['summary']
|
|
print(' ok nothing is left to file from that window')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (g) empty store, no --wiki-checked -> hard error (kotkan/claude-plugin-inference-arbitrage#17) =="
|
|
# The bug: a cold local cache and a genuinely history-free wiki look identical
|
|
# to this script, which only ever reads the filesystem. Without an explicit
|
|
# assertion from the caller, refuse rather than silently degrade to unchecked.
|
|
classified "$tmp/g.json" measured 0.12
|
|
if $BIN --classified "$tmp/g.json" --target synth --store "$tmp/store-empty-g" \
|
|
>"$tmp/g.out.json" 2>"$tmp/g.err"; then
|
|
echo " FAIL expected non-zero exit, got success" >&2
|
|
fail=1
|
|
else
|
|
echo " ok exits non-zero"
|
|
fi
|
|
if grep -q "wiki-checked" "$tmp/g.err"; then
|
|
echo " ok error names --wiki-checked"
|
|
else
|
|
echo " FAIL error does not mention --wiki-checked: $(cat "$tmp/g.err")" >&2
|
|
fail=1
|
|
fi
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (h) empty store, --wiki-checked=empty -> unchecked, as today =="
|
|
# The caller checked the wiki and it really is history-free; behavior matches (c).
|
|
classified "$tmp/h.json" measured 0.12
|
|
$BIN --classified "$tmp/h.json" --target synth --store "$tmp/store-empty-h" \
|
|
--wiki-checked=empty >"$tmp/h.out.json"
|
|
check "verdict" "$(field "$tmp/h.out.json" synth/skill/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/h.out.json" synth/skill/step stability)" \
|
|
"unchecked — no prior window to compare"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (i) empty store seeded via a real import, --wiki-checked=imported -> gate runs normally =="
|
|
# After `audit-snapshot import` seeds the store from a wiki-fetched
|
|
# snapshots.jsonl, the store is no longer empty, so the gate compares against
|
|
# the imported history exactly as if it had been local all along.
|
|
store=$tmp/store-import-i
|
|
mkdir -p "$store"
|
|
prior "$tmp/seed-i" thin 0.0
|
|
IMPORT_BIN=../bin/audit-snapshot
|
|
$IMPORT_BIN --store "$store" import --target synth \
|
|
--snapshots "$tmp/seed-i/Data/synth/snapshots.jsonl" >/dev/null
|
|
classified "$tmp/i.json" measured 0.12
|
|
$BIN --classified "$tmp/i.json" --target synth --store "$store" \
|
|
--wiki-checked=imported >"$tmp/i.out.json"
|
|
check "verdict" "$(field "$tmp/i.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/i.out.json" downgraded)" "1"
|
|
|
|
exit "$fail"
|