Oleks
88fc58b2fd
ci/woodpecker/push/test Pipeline was successful
The stability gate treated every evidence flip as symmetric, and that produced a false downgrade on the candidate it was written for (kotkan/claude-plugin-inference-arbitrage#15). Re-measuring anxious/agent-wip/release-policy-derivation over four real windows with the same script gave 0 -> 2 -> 12 -> 15 invocations at 7/14/27/60 days: monotonic accumulation of a step that runs about once every two days, not a label flipping about. The 7-day window had not caught an unstable candidate, it had failed to observe a real one. So a thin/unmeasured reading now only overturns a `measured` one when its window is at least 90% as wide (observed since->until span). Below that the comparison takes a third verdict, `insufficient-window`: not filed, not downgraded to a boundary question, no FR-6.5 self-report, and carried forward with the wider snapshot left as the standing comparison point. The parity is relative rather than an absolute day count because the adequate width is a property of the candidate's invocation rate, which the auditor does not know in advance. Unchanged, and tested: the reverse direction (thin prior -> measured now) still downgrades, a flip between comparably wide windows still downgrades, and a threshold crossing with `measured` on both sides is never excused by narrowness. An unreadable window is not an exemption either — the flip stands and window_parity records that the check could not run. - bin/stability-classify: window_days/window_parity/deferrable, the insufficient-window verdict, stability_deferrals, window_parity on findings too, insufficient_window + window_parity_ratio in the gate block - bin/audit-snapshot: record deferrals in the snapshot notes so the row is visible on the wiki page instead of silently dropped - design/spec.md: FR-4.5.1 as a testable requirement; S4 restated - design/rubric.md: the third boundary in §5b - skills/offload-audit/SKILL.md: do not file and do not self-report a deferral - tests: real 60-day anxious fixture; cases (f)/(f2) on real data, (j)-(n) synthetic — 7 test files pass Verified against the live store (~/.cache/inference-arbitrage/wiki): the real 7.0d-vs-26.49d comparison now reports insufficient-window with zero findings, while the real 33.4d-vs-26.49d one reports stable at 13.87% and files normally.
443 lines
21 KiB
Bash
Executable File
443 lines
21 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/stability-classify: the FR-4.5 evidence-stability gate. Three synthetic
|
|
# cases with hand-written snapshots, then the real two-window `anxious` evidence
|
|
# that motivated the gate (kotkan/claude-plugin-inference-arbitrage#11) — the
|
|
# repeatable form of acceptance criterion S4.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/stability-classify
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# field <json> <candidate_id> <key>
|
|
field() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
c=[x for x in d['candidates'] if x['candidate_id']==sys.argv[2]][0]
|
|
print(c.get(sys.argv[3]))" "$@"
|
|
}
|
|
gatefield() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['stability_gate'][sys.argv[2]])" "$@"
|
|
}
|
|
|
|
# scanwin <file> <since> <until>
|
|
# The only field the gate reads out of a scan for the FR-4.5 parity rule is the
|
|
# window's observed span, so the fixture carries exactly that.
|
|
scanwin() {
|
|
python3 -c "
|
|
import json, sys
|
|
json.dump({'window': {'since': sys.argv[2], 'until': sys.argv[3], 'sessions': 10}},
|
|
open(sys.argv[1], 'w'))" "$@"
|
|
}
|
|
|
|
# classified <file> <strength> <share>
|
|
# One position-3 candidate the boundary gate already graded `file`.
|
|
classified() {
|
|
python3 - "$1" "$2" "$3" <<'EOF'
|
|
import json, sys
|
|
out, strength, share = sys.argv[1:]
|
|
json.dump({
|
|
"rubric_version": "1.0.0",
|
|
"target": "synth",
|
|
"summary": {"candidates": 1, "high_confidence": 1, "to_file": 1,
|
|
"boundary_questions": 0, "no_offload": 0, "drift_notes": 0},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"position": "llm-over-script-digest",
|
|
"boundary_confidence": "high",
|
|
"measurement_strength": strength,
|
|
"share_of_audited_spend": float(share),
|
|
"offload_value": 500000,
|
|
"verdict": "file",
|
|
"reasons": ["all five determinism tests pass; triple complete"],
|
|
}],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# snapshot-store <store> <strength> <share> [since] [until]
|
|
# A prior snapshot recording the same candidate by slug identity. The window
|
|
# defaults to 30 days; the width matters for the FR-4.5 parity rule (cases j-m).
|
|
prior() {
|
|
local store=$1
|
|
mkdir -p "$store/Data/synth"
|
|
python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" "${4:-2026-06-01}" "${5:-2026-07-01}" <<'EOF'
|
|
import json, sys
|
|
out, strength, share, since, until = sys.argv[1:]
|
|
json.dump({
|
|
"run_id": "2026-07-01T00-00-00Z",
|
|
"window": {"since": since, "until": until},
|
|
"target": {"name": "synth", "version": "1.0.0", "skills": ["synth:skill"]},
|
|
"coverage": {"ratio": 0.1, "attributed_turns": 500},
|
|
"totals": {"weighted_tokens": 5000000},
|
|
"candidates": [{
|
|
"candidate_id": "synth/skill/step",
|
|
"skill": "synth:skill",
|
|
"position": "llm-over-script-digest",
|
|
"boundary_confidence": "high",
|
|
"signature_hash": "slugonly",
|
|
"measurement_strength": strength,
|
|
"measurement": {"invocations": 10, "offload_value": 500000,
|
|
"share_of_plugin": float(share), "cost_per_invocation": 50000},
|
|
"issue": None,
|
|
"verdict": "file",
|
|
}],
|
|
"boundary_questions": [],
|
|
"measurements_by_signature": {},
|
|
"notes": [],
|
|
}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (a) measured now, thin in the prior window -> downgraded, with a finding =="
|
|
# The shape of the real anxious case: same candidate, same rubric grade, two
|
|
# different evidence labels because the windows differ.
|
|
classified "$tmp/a.json" measured 0.12
|
|
prior "$tmp/store-a" thin 0.0
|
|
$BIN --classified "$tmp/a.json" --target synth --store "$tmp/store-a" >"$tmp/a.out.json"
|
|
check "verdict" "$(field "$tmp/a.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/a.out.json" downgraded)" "1"
|
|
check "checked" "$(gatefield "$tmp/a.out.json" checked)" "1"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
f=d['stability_findings'][0]
|
|
fields={x['field'] for x in f['flips']}
|
|
assert fields == {'measurement_strength','share_of_audited_spend'}, fields
|
|
assert f['prior']['measurement_strength']=='thin'
|
|
assert f['current']['measurement_strength']=='measured'
|
|
assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage'
|
|
assert f['self_report']['marker']=='<!-- ia-stability: synth/skill/step -->'
|
|
print(' ok finding names both flips, and self-reports with a marker')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/a.out.json'))
|
|
r=d['candidates'][0]['reasons'][-1]
|
|
assert 'HARD GATE (FR-4.5)' in r and 'never filed' in r, r
|
|
assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==1, d['summary']
|
|
print(' ok reason cites the gate; summary counts are re-derived')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (b) same label, same side of the threshold -> stays filed =="
|
|
# 12% then 8% is a trend, not an instability: `audit-snapshot diff` reports the
|
|
# movement, and the human is told the same thing either way.
|
|
classified "$tmp/b.json" measured 0.08
|
|
prior "$tmp/store-b" measured 0.12
|
|
$BIN --classified "$tmp/b.json" --target synth --store "$tmp/store-b" >"$tmp/b.out.json"
|
|
check "verdict" "$(field "$tmp/b.out.json" synth/skill/step verdict)" "file"
|
|
check "downgraded" "$(gatefield "$tmp/b.out.json" downgraded)" "0"
|
|
check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/b.out.json'))['stability_findings']))")" "0"
|
|
python3 -c "
|
|
import json
|
|
s=json.load(open('$tmp/b.out.json'))['candidates'][0]['stability']
|
|
assert s.startswith('stable against 2026-07-01T00-00-00Z'), s
|
|
print(' ok stability names the window it was checked against')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (c) no prior snapshot -> files normally, unchecked (FR-2.4) =="
|
|
# An absence of history is not evidence of instability. A first audit of a
|
|
# target must still be able to file. The caller must still assert the wiki
|
|
# was checked (kotkan/claude-plugin-inference-arbitrage#17) -- see (g)/(h).
|
|
classified "$tmp/c.json" measured 0.12
|
|
$BIN --classified "$tmp/c.json" --target synth --store "$tmp/store-empty" \
|
|
--wiki-checked=empty >"$tmp/c.out.json"
|
|
check "verdict" "$(field "$tmp/c.out.json" synth/skill/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/c.out.json" synth/skill/step stability)" \
|
|
"unchecked — no prior window to compare"
|
|
check "checked" "$(gatefield "$tmp/c.out.json" checked)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (d) history exists but not for this candidate -> unchecked, still filed =="
|
|
classified "$tmp/d.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/d.json'))
|
|
d['candidates'][0]['candidate_id']='synth/other/step'
|
|
json.dump(d,open('$tmp/d.json','w'))"
|
|
prior "$tmp/store-d" thin 0.0
|
|
$BIN --classified "$tmp/d.json" --target synth --store "$tmp/store-d" >"$tmp/d.out.json"
|
|
check "verdict" "$(field "$tmp/d.out.json" synth/other/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/d.out.json" synth/other/step stability)" \
|
|
"unchecked — no prior window recorded this candidate"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (e) only 'file' verdicts are gated =="
|
|
# A boundary question is already not being filed; re-gating it would double-count
|
|
# and would put a stability finding on something no human is being shown.
|
|
classified "$tmp/e.json" measured 0.12
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/e.json'))
|
|
d['candidates'][0]['verdict']='boundary-question'
|
|
json.dump(d,open('$tmp/e.json','w'))"
|
|
prior "$tmp/store-e" thin 0.0
|
|
$BIN --classified "$tmp/e.json" --target synth --store "$tmp/store-e" >"$tmp/e.out.json"
|
|
check "untouched verdict" "$(field "$tmp/e.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "not counted as checked" "$(gatefield "$tmp/e.out.json" checked)" "0"
|
|
check "no finding" "$(gatefield "$tmp/e.out.json" downgraded)" "0"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (f) S4 — the real anxious 7-day window is refused, as insufficient-window =="
|
|
# The exact evidence from kotkan/claude-plugin-inference-arbitrage#11: the
|
|
# ~26-day window graded `anxious/agent-wip/release-policy-derivation` measured at
|
|
# 11.94% of audited spend, the 7-day window graded the same candidate thin at
|
|
# 0.00%. Both windows' snapshots are in the store; --exclude-run makes the 7-day
|
|
# run the one being classified, exactly as it is at filing time.
|
|
#
|
|
# S4 still holds — the candidate is mechanically refused with no human in the
|
|
# loop — but per kotkan/claude-plugin-inference-arbitrage#15 the honest reason is
|
|
# not "unstable". Re-measuring the same candidate over 14/27/60 days gave 0 -> 2
|
|
# -> 12 -> 15 invocations: a monotonic accumulation of a ~once-every-two-days
|
|
# step, so the 7-day window under-sampled it rather than contradicting it. A
|
|
# 7.0-day window cannot overturn a 26.1-day `measured` reading, so this is a
|
|
# deferral, not a downgrade, and nothing is self-reported against this plugin.
|
|
store=$tmp/store-anxious
|
|
mkdir -p "$store/Data/anxious"
|
|
cp calibration/anxious-windows.snapshots.jsonl "$store/Data/anxious/snapshots.jsonl"
|
|
$BIN --classified calibration/anxious-7d.classified.json \
|
|
--target anxious --store "$store" \
|
|
--scan calibration/anxious-7d.scan.json \
|
|
--exclude-run 2026-07-29T01-00-00Z >"$tmp/f.out.json"
|
|
check "verdict" "$(field "$tmp/f.out.json" anxious/agent-wip/release-policy-derivation verdict)" \
|
|
"insufficient-window"
|
|
check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "0"
|
|
check "insufficient_window" "$(gatefield "$tmp/f.out.json" insufficient_window)" "1"
|
|
check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/f.out.json'))['stability_findings']))")" "0"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
f=[x for x in d['stability_deferrals']
|
|
if x['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0]
|
|
assert f['prior']['measurement_strength']=='measured', f['prior']
|
|
assert abs(f['prior']['share_of_audited_spend']-0.1194) < 1e-9, f['prior']
|
|
assert f['current']['measurement_strength']=='thin', f['current']
|
|
assert f['current']['share_of_audited_spend']==0.0, f['current']
|
|
assert f['prior']['run_id']=='2026-07-29T00-00-00Z', f['prior']
|
|
p=f['window_parity']
|
|
assert p['comparable'] is False, p
|
|
assert abs(p['current_days']-7.0) < 0.05, p
|
|
assert 25.9 < p['prior_days'] < 26.2, p
|
|
assert {x['field'] for x in f['suppressed_flips']} == {
|
|
'measurement_strength','share_of_audited_spend'}, f['suppressed_flips']
|
|
assert 'self_report' not in f, f
|
|
print(' ok deferral carries the real widths (7.0d vs 26.1d) and the suppressed flips')
|
|
" || fail=1
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f.out.json'))
|
|
assert d['summary']['to_file']==0, d['summary']
|
|
assert d['summary']['boundary_questions']==2, d['summary']
|
|
r=[c for c in d['candidates']
|
|
if c['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0]['reasons'][-1]
|
|
assert 'DEFERRED (FR-4.5)' in r and 'carried' in r, r
|
|
print(' ok not filed, not counted as a boundary question, reason cites the deferral')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (f2) real 60-day window against the real 27-day prior -> reports normally =="
|
|
# The other half of kotkan/claude-plugin-inference-arbitrage#15's evidence: the
|
|
# 60-day window (15 invocations, offload_value 4,953,520 of 35,722,973 weighted
|
|
# tokens = 13.87%) against the ~27-day `measured` 11.94% prior. Two adequately
|
|
# wide windows, same label, same side of the 2% threshold -> stable, files
|
|
# normally. The parity rule must not suppress this: the fix is about a narrow
|
|
# CURRENT window, not about refusing to ever conclude anything.
|
|
#
|
|
# --exclude-run drops the 7-day snapshot so the pairing under test really is
|
|
# 27d-vs-60d; the 60d-against-7d pairing is the reverse direction, and it still
|
|
# downgrades — see (k).
|
|
$BIN --classified calibration/anxious-60d.classified.json \
|
|
--target anxious --store "$store" \
|
|
--scan calibration/anxious-60d.scan.json \
|
|
--exclude-run 2026-07-29T01-00-00Z >"$tmp/f2.out.json"
|
|
check "verdict" "$(field "$tmp/f2.out.json" anxious/agent-wip/release-policy-derivation verdict)" \
|
|
"file"
|
|
check "downgraded" "$(gatefield "$tmp/f2.out.json" downgraded)" "0"
|
|
check "insufficient_window" "$(gatefield "$tmp/f2.out.json" insufficient_window)" "0"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/f2.out.json'))
|
|
c=d['candidates'][0]
|
|
assert c['stability'].startswith('stable against 2026-07-29T00-00-00Z'), c['stability']
|
|
assert abs(c['share_of_audited_spend']-0.1387) < 1e-9, c['share_of_audited_spend']
|
|
print(' ok the 60-day reading (13.87%) is stable against the 27-day one (11.94%)')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (g) empty store, no --wiki-checked -> hard error (kotkan/claude-plugin-inference-arbitrage#17) =="
|
|
# The bug: a cold local cache and a genuinely history-free wiki look identical
|
|
# to this script, which only ever reads the filesystem. Without an explicit
|
|
# assertion from the caller, refuse rather than silently degrade to unchecked.
|
|
classified "$tmp/g.json" measured 0.12
|
|
if $BIN --classified "$tmp/g.json" --target synth --store "$tmp/store-empty-g" \
|
|
>"$tmp/g.out.json" 2>"$tmp/g.err"; then
|
|
echo " FAIL expected non-zero exit, got success" >&2
|
|
fail=1
|
|
else
|
|
echo " ok exits non-zero"
|
|
fi
|
|
if grep -q "wiki-checked" "$tmp/g.err"; then
|
|
echo " ok error names --wiki-checked"
|
|
else
|
|
echo " FAIL error does not mention --wiki-checked: $(cat "$tmp/g.err")" >&2
|
|
fail=1
|
|
fi
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (h) empty store, --wiki-checked=empty -> unchecked, as today =="
|
|
# The caller checked the wiki and it really is history-free; behavior matches (c).
|
|
classified "$tmp/h.json" measured 0.12
|
|
$BIN --classified "$tmp/h.json" --target synth --store "$tmp/store-empty-h" \
|
|
--wiki-checked=empty >"$tmp/h.out.json"
|
|
check "verdict" "$(field "$tmp/h.out.json" synth/skill/step verdict)" "file"
|
|
check "stability" "$(field "$tmp/h.out.json" synth/skill/step stability)" \
|
|
"unchecked — no prior window to compare"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (i) empty store seeded via a real import, --wiki-checked=imported -> gate runs normally =="
|
|
# After `audit-snapshot import` seeds the store from a wiki-fetched
|
|
# snapshots.jsonl, the store is no longer empty, so the gate compares against
|
|
# the imported history exactly as if it had been local all along.
|
|
store=$tmp/store-import-i
|
|
mkdir -p "$store"
|
|
prior "$tmp/seed-i" thin 0.0
|
|
IMPORT_BIN=../bin/audit-snapshot
|
|
$IMPORT_BIN --store "$store" import --target synth \
|
|
--snapshots "$tmp/seed-i/Data/synth/snapshots.jsonl" >/dev/null
|
|
classified "$tmp/i.json" measured 0.12
|
|
$BIN --classified "$tmp/i.json" --target synth --store "$store" \
|
|
--wiki-checked=imported >"$tmp/i.out.json"
|
|
check "verdict" "$(field "$tmp/i.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/i.out.json" downgraded)" "1"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (j) narrow current window, measured wide prior -> insufficient-window =="
|
|
# The kotkan/claude-plugin-inference-arbitrage#15 shape, synthetically: a 30-day
|
|
# window measured the candidate at 12%, a 7-day window sees nothing. 7d is below
|
|
# the 27.0d parity floor, so the weak reading does not get to overturn the
|
|
# measured one — neither filed nor downgraded, and no self-report.
|
|
classified "$tmp/j.json" thin 0.0
|
|
prior "$tmp/store-j" measured 0.12
|
|
scanwin "$tmp/j.scan.json" 2026-07-23 2026-07-30
|
|
$BIN --classified "$tmp/j.json" --target synth --store "$tmp/store-j" \
|
|
--scan "$tmp/j.scan.json" >"$tmp/j.out.json"
|
|
check "verdict" "$(field "$tmp/j.out.json" synth/skill/step verdict)" "insufficient-window"
|
|
check "downgraded" "$(gatefield "$tmp/j.out.json" downgraded)" "0"
|
|
check "insufficient_window" "$(gatefield "$tmp/j.out.json" insufficient_window)" "1"
|
|
check "self-reports" "$(python3 -c "import json;print(len(json.load(open('$tmp/j.out.json'))['stability_findings']))")" "0"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/j.out.json'))
|
|
f=d['stability_deferrals'][0]
|
|
p=f['window_parity']
|
|
assert p['prior_days']==30.0 and p['current_days']==7.0 and p['required_days']==27.0, p
|
|
assert p['comparable'] is False and p['parity_ratio']==0.9, p
|
|
assert 'at least 27.0d' in f['carry_forward'], f['carry_forward']
|
|
assert f['prior']['run_id']=='2026-07-01T00-00-00Z', f['prior']
|
|
s=d['candidates'][0]['stability']
|
|
assert s.startswith('insufficient-window against 2026-07-01T00-00-00Z'), s
|
|
assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==0, d['summary']
|
|
assert d['summary']['stability_insufficient_window']==1, d['summary']
|
|
print(' ok deferral names both widths, the floor, and the window it defers against')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (k) reverse direction — thin wide prior, measured narrow current -> downgraded =="
|
|
# The narrowness carve-out is one-directional. Here the WIDE window is the one
|
|
# that saw nothing and the narrow one measures 12%: the weak reading is the
|
|
# earlier one, the contradiction is genuine, and the human would still be told
|
|
# two different things. Downgrade, exactly as before the fix.
|
|
classified "$tmp/k.json" measured 0.12
|
|
prior "$tmp/store-k" thin 0.0
|
|
scanwin "$tmp/k.scan.json" 2026-07-23 2026-07-30
|
|
$BIN --classified "$tmp/k.json" --target synth --store "$tmp/store-k" \
|
|
--scan "$tmp/k.scan.json" >"$tmp/k.out.json"
|
|
check "verdict" "$(field "$tmp/k.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/k.out.json" downgraded)" "1"
|
|
check "insufficient_window" "$(gatefield "$tmp/k.out.json" insufficient_window)" "0"
|
|
python3 -c "
|
|
import json
|
|
d=json.load(open('$tmp/k.out.json'))
|
|
f=d['stability_findings'][0]
|
|
assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage', f['self_report']
|
|
p=f['window_parity']
|
|
assert p['comparable'] is False, p
|
|
print(' ok still self-reported, and the parity block records the widths anyway')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (l) two comparably wide windows, genuine flip -> downgraded =="
|
|
# 28 days against 30 is at the parity floor (27.0d), so the windows are
|
|
# comparably sampled and a measured -> thin flip is real instability.
|
|
classified "$tmp/l.json" thin 0.0
|
|
prior "$tmp/store-l" measured 0.12
|
|
scanwin "$tmp/l.scan.json" 2026-07-02 2026-07-30
|
|
$BIN --classified "$tmp/l.json" --target synth --store "$tmp/store-l" \
|
|
--scan "$tmp/l.scan.json" >"$tmp/l.out.json"
|
|
check "verdict" "$(field "$tmp/l.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/l.out.json" downgraded)" "1"
|
|
check "insufficient_window" "$(gatefield "$tmp/l.out.json" insufficient_window)" "0"
|
|
python3 -c "
|
|
import json
|
|
p=json.load(open('$tmp/l.out.json'))['stability_findings'][0]['window_parity']
|
|
assert p['current_days']==28.0 and p['required_days']==27.0 and p['comparable'] is True, p
|
|
print(' ok 28.0d clears the 27.0d floor, so the flip stands')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (m) measured both windows, threshold crossed on a narrow window -> downgraded =="
|
|
# A window that still reached `measured` observed the candidate often enough; a
|
|
# share that then crosses the 2% filing threshold is a claim about relative
|
|
# spend, not a sampling failure, so narrowness does not excuse it.
|
|
classified "$tmp/m.json" measured 0.01
|
|
prior "$tmp/store-m" measured 0.12
|
|
scanwin "$tmp/m.scan.json" 2026-07-23 2026-07-30
|
|
$BIN --classified "$tmp/m.json" --target synth --store "$tmp/store-m" \
|
|
--scan "$tmp/m.scan.json" >"$tmp/m.out.json"
|
|
check "verdict" "$(field "$tmp/m.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/m.out.json" downgraded)" "1"
|
|
check "insufficient_window" "$(gatefield "$tmp/m.out.json" insufficient_window)" "0"
|
|
python3 -c "
|
|
import json
|
|
f=json.load(open('$tmp/m.out.json'))['stability_findings'][0]
|
|
assert {x['field'] for x in f['flips']} == {'share_of_audited_spend'}, f['flips']
|
|
print(' ok only the threshold flip is reported, and it is not excused')
|
|
" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (n) unreadable window widths -> downgraded, and said so explicitly =="
|
|
# No --scan means no current window bounds. Narrowness cannot be established, so
|
|
# the flip is NOT quietly excused — the finding records that the check could not
|
|
# run, rather than defaulting either way.
|
|
classified "$tmp/n.json" thin 0.0
|
|
prior "$tmp/store-n" measured 0.12
|
|
$BIN --classified "$tmp/n.json" --target synth --store "$tmp/store-n" >"$tmp/n.out.json"
|
|
check "verdict" "$(field "$tmp/n.out.json" synth/skill/step verdict)" "boundary-question"
|
|
check "downgraded" "$(gatefield "$tmp/n.out.json" downgraded)" "1"
|
|
python3 -c "
|
|
import json
|
|
p=json.load(open('$tmp/n.out.json'))['stability_findings'][0]['window_parity']
|
|
assert p['comparable'] is None and p['current_days'] is None, p
|
|
assert 'unreadable' in p['why'], p
|
|
print(' ok comparability is null with an explicit reason, not a silent exemption')
|
|
" || fail=1
|
|
|
|
exit "$fail"
|