oleks
dcec9cabfb
Phase 7 filed anxious/agent-wip/release-policy-derivation as `measured` at 11.94% of audited spend on a 30-day window. The same candidate over 7 days came back `thin` at 0.00%. Only the date range differed. A human reading either issue body alone cannot know the other exists, which is rubric.md §4a's failure mode reached through the plugin's own headline number. That was caught by a human running a second skeptical review — luck, not a control. It is now mechanical, in the same register as FR-4.2's overrule-case gate: - bin/stability-classify matches each `file` candidate against the most recent prior snapshot recording it (FR-5.2 identity) and downgrades it to a boundary question if `measurement_strength` changed or the share crossed the 2% filing threshold between windows. Movement within a label is a trend, not an instability; no prior window is `unchecked`, not unstable (FR-2.4). - The downgrade is a finding against THIS repo, not the target's — the target did not change, the auditor described it two ways. Carries an `ia-stability` marker so a re-run comments rather than duplicating. - bin/ia_store.py factors the store and identity resolution out of audit-snapshot so both scripts hash signatures identically. A divergence there would make the gate match nothing and fail open. Documented as spec FR-4.5 and rubric §5b (shipped verbatim in references/). Wired into skills/offload-audit as step 4b, before filing-plan; steps 5-6 now consume its output rather than classified.json. S4 is retired in its old form — a human triage catching a bad candidate is not a repeatable test — and re-passed as: the gate catches the real release-policy-derivation case with no human in the loop. Asserted in tests/stability.test.sh case (f) against the verbatim two-window Phase 7 output. Closes kotkan/claude-plugin-inference-arbitrage#11 Suite: 178 assertions, exit 0.
219 lines
9.2 KiB
Bash
Executable File
219 lines
9.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/stability-classify: the FR-4.5 evidence-stability gate. Three synthetic
|
|
# cases with hand-written snapshots, then the real two-window `anxious` evidence
|
|
# that motivated the gate (kotkan/claude-plugin-inference-arbitrage#11) — the
|
|
# repeatable form of acceptance criterion S4.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/stability-classify
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# field <json> <candidate_id> <key>
|
|
field() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
c=[x for x in d['candidates'] if x['candidate_id']==sys.argv[2]][0]
|
|
print(c.get(sys.argv[3]))" "$@"
|
|
}
|
|
gatefield() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['stability_gate'][sys.argv[2]])" "$@"
|
|
}
|
|
|
|
# classified <file> <strength> <share>
|
|
# One position-3 candidate the boundary gate already graded `file`.
|
|
classified() {
|
|
python3 - "$1" "$2" "$3" <<'EOF'
|
|
import json, sys
|
|
out, strength, share = sys.argv[1:]
|
|
json.dump({
|
|
"rubric_version": "1.0.0",
|
|
"target": "synth",
|
|
"summary": {"candidates": 1, "high_confidence": 1, "to_file": 1,
|
|
"boundary_questions": 0, "no_offload": 0, "drift_notes": 0},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"position": "llm-over-script-digest",
|
|
"boundary_confidence": "high",
|
|
"measurement_strength": strength,
|
|
"share_of_audited_spend": float(share),
|
|
"offload_value": 500000,
|
|
"verdict": "file",
|
|
"reasons": ["all five determinism tests pass; triple complete"],
|
|
}],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# snapshot-store <store> <strength> <share>
|
|
# A prior snapshot recording the same candidate by slug identity.
|
|
prior() {
|
|
local store=$1
|
|
mkdir -p "$store/Data/synth"
|
|
python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" <<'EOF'
|
|
import json, sys
|
|
out, strength, share = sys.argv[1:]
|
|
json.dump({
|
|
"run_id": "2026-07-01T00-00-00Z",
|
|
"window": {"since": "2026-06-01", "until": "2026-07-01"},
|
|
"target": {"name": "synth", "version": "1.0.0", "skills": ["synth:skill"]},
|
|
"coverage": {"ratio": 0.1, "attributed_turns": 500},
|
|
"totals": {"weighted_tokens": 5000000},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"signature_hash": "slugonly",
|
|
"measurement_strength": strength,
|
|
"measurement": {"invocations": 10, "offload_value": 500000,
|
|
"share_of_plugin": float(share), "cost_per_invocation": 50000},
|
|
"issue": None,
|
|
"verdict": "file",
|
|
}],
|
|
"boundary_questions": [],
|
|
"measurements_by_signature": {},
|
|
"notes": [],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (a) measured now, thin in the prior window -> downgraded, with a finding =="
|
|
# The shape of the real anxious case: same candidate, same rubric grade, two
|
|
# different evidence labels because the windows differ.
|
|
classified "$tmp/a.json" measured 0.12
|
|
prior "$tmp/store-a" thin 0.0
|
|
$BIN --classified "$tmp/a.json" --target synth --store "$tmp/store-a" >"$tmp/a.out.json"
|
|
check "verdict" "$(field "$tmp/a.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/a.out.json" downgraded)" "1"
|
|
check "checked" "$(gatefield "$tmp/a.out.json" checked)" "1"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
f=d['stability_findings'][0]
|
|
fields={x['field'] for x in f['flips']}
|
|
assert fields == {'measurement_strength','share_of_audited_spend'}, fields
|
|
assert f['prior']['measurement_strength']=='thin'
|
|
assert f['current']['measurement_strength']=='measured'
|
|
assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage'
|
|
assert f['self_report']['marker']=='<!-- ia-stability: synth/skill/step -->'
|
|
print(' ok finding names both flips, and self-reports with a marker')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
r=d['candidates'][0]['reasons'][-1]
|
|
assert 'HARD GATE (FR-4.5)' in r and 'never filed' in r, r
|
|
assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==1, d['summary']
|
|
print(' ok reason cites the gate; summary counts are re-derived')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (b) same label, same side of the threshold -> stays filed =="
|
|
# 12% then 8% is a trend, not an instability: `audit-snapshot diff` reports the
|
|
# movement, and the human is told the same thing either way.
|
|
classified "$tmp/b.json" measured 0.08
|
|
prior "$tmp/store-b" measured 0.12
|
|
$BIN --classified "$tmp/b.json" --target synth --store "$tmp/store-b" >"$tmp/b.out.json"
|
|
check "verdict" "$(field "$tmp/b.out.json" synth/skill/step verdict)" "file"
|
|
check "downgraded" "$(gatefield "$tmp/b.out.json" downgraded)" "0"
|
|
check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/b.out.json'))['stability_findings']))")" "0"
|
|
python3 -c "
|
|
import json
|
|
s=json.load(open('$tmp/b.out.json'))['candidates'][0]['stability']
|
|
assert s.startswith('stable against 2026-07-01T00-00-00Z'), s
|
|
print(' ok stability names the window it was checked against')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (c) no prior snapshot -> files normally, unchecked (FR-2.4) =="
|
|
# An absence of history is not evidence of instability. A first audit of a
|
|
# target must still be able to file.
|
|
classified "$tmp/c.json" measured 0.12
|
|
$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" >"$tmp/c.out.json"
|
|
check "verdict" "$(field "$tmp/c.out.json" synth/skill/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/c.out.json" synth/skill/step stability)" \
|
|
"unchecked — no prior window to compare"
|
|
check "checked" "$(gatefield "$tmp/c.out.json" checked)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (d) history exists but not for this candidate -> unchecked, still filed =="
|
|
classified "$tmp/d.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/d.json'))
|
|
d['candidates'][0]['candidate_id']='synth/other/step'
|
|
json.dump(d,open('$tmp/d.json','w'))"
|
|
prior "$tmp/store-d" thin 0.0
|
|
$BIN --classified "$tmp/d.json" --target synth --store "$tmp/store-d" >"$tmp/d.out.json"
|
|
check "verdict" "$(field "$tmp/d.out.json" synth/other/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/d.out.json" synth/other/step stability)" \
|
|
"unchecked — no prior window recorded this candidate"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (e) only 'file' verdicts are gated =="
|
|
# A boundary question is already not being filed; re-gating it would double-count
|
|
# and would put a stability finding on something no human is being shown.
|
|
classified "$tmp/e.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/e.json'))
|
|
d['candidates'][0]['verdict']='boundary-question'
|
|
json.dump(d,open('$tmp/e.json','w'))"
|
|
prior "$tmp/store-e" thin 0.0
|
|
$BIN --classified "$tmp/e.json" --target synth --store "$tmp/store-e" >"$tmp/e.out.json"
|
|
check "untouched verdict" "$(field "$tmp/e.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "not counted as checked" "$(gatefield "$tmp/e.out.json" checked)" "0"
|
|
check "no finding" "$(gatefield "$tmp/e.out.json" downgraded)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (f) S4 — the real anxious candidate is caught with no human in the loop =="
|
|
# The exact evidence from kotkan/claude-plugin-inference-arbitrage#11: the
|
|
# 30-day window graded `anxious/agent-wip/release-policy-derivation` measured at
|
|
# 11.94% of audited spend, the 7-day window graded the same candidate thin at
|
|
# 0.00%. Both windows' snapshots are in the store; --exclude-run makes the 7-day
|
|
# run the one being classified, exactly as it is at filing time.
|
|
store=$tmp/store-anxious
|
|
mkdir -p "$store/Data/anxious"
|
|
cp calibration/anxious-windows.snapshots.jsonl "$store/Data/anxious/snapshots.jsonl"
|
|
$BIN --classified calibration/anxious-7d.classified.json \
|
|
--target anxious --store "$store" \
|
|
--scan calibration/anxious-7d.scan.json \
|
|
--exclude-run 2026-07-29T01-00-00Z >"$tmp/f.out.json"
|
|
check "verdict" "$(field "$tmp/f.out.json" anxious/agent-wip/release-policy-derivation verdict)" \
|
|
"boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "1"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
f=[x for x in d['stability_findings']
|
|
if x['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0]
|
|
assert f['prior']['measurement_strength']=='measured', f['prior']
|
|
assert abs(f['prior']['share_of_audited_spend']-0.1194) < 1e-9, f['prior']
|
|
assert f['current']['measurement_strength']=='thin', f['current']
|
|
assert f['current']['share_of_audited_spend']==0.0, f['current']
|
|
assert f['prior']['run_id']=='2026-07-29T00-00-00Z', f['prior']
|
|
print(' ok finding carries the real numbers: measured 11.94% vs thin 0.00%')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
assert d['summary']['to_file']==0, d['summary']
|
|
print(' ok nothing is left to file from that window')
|
|
" || fail=1
|
|
|
|
exit "$fail"
|