Oleks
6b634fd9f8
decline_reason() only ever read a candidate's own grade, so a candidate boundary-classify graded high/complete-triple -- meaning it cleared FR-4.2 fine -- fell through to the generic 'did not clear the filing threshold' line when stability-classify downgraded it afterward. That's the opposite of what happened, and it directly contradicted the correct FR-4.5 section rendered a few lines below it in the same judgment-calls drawer, poisoning the next audit's step-2b recall with a false reason. Found live: real audit of anxious (2026-07-30) downgraded agent-wip/release-policy-derivation at 11.85% of audited spend -- the one gate it DID clear -- and the drawer said it didn't. Filed and now fixed as kotkan/claude-plugin-inference-arbitrage#16. Fix keeps FR-4.2's own check order first (a candidate can coincidentally be named in an unrelated stability finding without having actually cleared those checks) and only replaces the final fallback, since a candidate that reaches it cleared every earlier check on its own grade -- it cannot have also failed the filing threshold on its own merits. New regression case (h) in tests/memory.test.sh. Suite: all tests pass.
323 lines
16 KiB
Bash
Executable File
323 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/audit-snapshot memory-queries / memory-notes: the FR-9 mempalace bridge.
|
|
# Everything deterministic is asserted here — wing/room resolution, the query
|
|
# packing, which drawers exist for which run shape, the rendered content, and
|
|
# the redaction allowlist.
|
|
#
|
|
# NOT tested here, and not fakeable in this harness: the MCP calls themselves.
|
|
# `add_drawer`/`kg_add`/`search` run in the caller's session, not in a shell,
|
|
# so no assertion below proves a drawer landed. That is deliberate — the whole
|
|
# reason the payload is computed by a script is to make the untestable part one
|
|
# verbatim tool call with no judgment in it. The round trip was verified once,
|
|
# by hand, against the live palace (design/tasks.md 7.9.5).
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/audit-snapshot
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
py() { python3 -c "$@"; }
|
|
|
|
# drawer <notes.json> <kind> <field>
|
|
drawer() {
|
|
py "
|
|
import json,sys
|
|
d=json.load(open('$1'))
|
|
x=[k for k in d['drawers'] if k['kind']=='$2']
|
|
print(x[0]['$3'] if x else 'ABSENT')"
|
|
}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Fixtures — one fake target 'synth' with three candidates: one filed, one
|
|
# boundary question, one no-offload (whose reason prose reaches snapshot notes).
|
|
# --------------------------------------------------------------------------
|
|
|
|
# The strings that must never reach a drawer. Each stands for a different
|
|
# free-text path into the snapshot: an analyst reason, and a caller --note.
|
|
LEAK_REASON="the user said deploy the staging key AKIAIOSFODNN7EXAMPLE to prod"
|
|
LEAK_NOTE="filing: unavailable — grep -rn 'password=hunter2' /home/oleks/secrets"
|
|
|
|
inventory() {
|
|
py "
|
|
import json
|
|
json.dump({'plugin': {'name': 'synth', 'version': '1.2.0', 'source': 'oleks-local'},
|
|
'skills': [{'name': 'synth:alpha'}, {'name': 'synth:beta'}],
|
|
'agents': [{'name': 'synth:gamma'}]}, open('$1','w'))"
|
|
}
|
|
|
|
scan() {
|
|
py "
|
|
import json
|
|
json.dump({'window': {'since': '2026-07-01', 'until': '2026-07-29', 'days': 28},
|
|
'plugin': 'synth', 'tool_versions': {'cc_tokens_path': '/synthetic/cc-tokens'},
|
|
'coverage': {'attributed_turns': 400, 'candidate_turns': 20000, 'ratio': 0.02},
|
|
'totals': {'weighted_tokens': 10000000, 'invocations': 30,
|
|
'mechanical_share': 0.41},
|
|
'skills': [{'skill': 'synth:alpha', 'invocations': 20, 'offload_waste': 2000000,
|
|
'ngrams': [{'sig': ['list_issues', 'issue_read'], 'recurrences': 20,
|
|
'invocations': 20, 'waste': 2000000}]},
|
|
{'skill': 'synth:beta', 'invocations': 10, 'offload_waste': 400000,
|
|
'ngrams': [{'sig': ['glob', 'read'], 'recurrences': 10,
|
|
'invocations': 10, 'waste': 400000}]}],
|
|
'agents': []}, open('$1','w'))"
|
|
}
|
|
|
|
# classified <file> [--only-filed]
|
|
classified() {
|
|
py "
|
|
import json,sys
|
|
only = '${2:-}' == '--only-filed'
|
|
def cand(cid, skill, value, share, verdict, conf, overrule, t3=True):
|
|
return {'candidate_id': cid, 'skill': skill, 'position': 'llm-over-script-digest',
|
|
'determinism_tests': {t: True for t in ('T1','T2','T4','T5')} | {'T3': t3},
|
|
'falsifiability_triple': {'signature': True, 'examples': True,
|
|
'overrule_case': overrule},
|
|
'boundary_confidence': conf, 'measurement_strength': 'measured',
|
|
'share_of_audited_spend': share, 'offload_value': value,
|
|
'escalation_path': 'escalate', 'digest_schema': 'typed rows',
|
|
'verdict': verdict, 'reasons': ['synthetic']}
|
|
cands = [cand('synth/alpha/dispatch', 'synth:alpha', 2000000, 0.20, 'file', 'high', True)]
|
|
if not only:
|
|
cands.append(cand('synth/beta/triage', 'synth:beta', 400000, 0.04,
|
|
'boundary-question', 'medium', False))
|
|
nono = cand('synth/beta/prose', 'synth:beta', 0, 0.0, 'no-offload', 'low', False, t3=False)
|
|
nono['reasons'] = ['''$LEAK_REASON''']
|
|
cands.append(nono)
|
|
json.dump({'rubric_version': '1.0.0', 'target': 'synth', 'candidates': cands}, open('$1','w'))"
|
|
}
|
|
|
|
# stable <file> — one stability finding, the FR-4.5 downgrade shape
|
|
stable() {
|
|
py "
|
|
import json
|
|
json.dump({'stability_findings': [{
|
|
'gate': 'FR-4.5', 'candidate_id': 'synth/beta/triage', 'target': 'synth',
|
|
'skill': 'synth:beta', 'matched_by': 'slug',
|
|
'prior': {'run_id': '2026-07-01T00-00-00Z', 'window': {'since': '2026-06-01'},
|
|
'measurement_strength': 'thin', 'share_of_audited_spend': 0.0},
|
|
'current': {'window': {'since': '2026-07-01'}, 'measurement_strength': 'measured',
|
|
'share_of_audited_spend': 0.04},
|
|
'flips': [{'field': 'measurement_strength', 'prior': 'thin', 'current': 'measured'}],
|
|
'self_report': {'repo': 'kotkan/claude-plugin-inference-arbitrage',
|
|
'marker': '<!-- ia-stability: synth/beta/triage -->'}}]},
|
|
open('$1','w'))"
|
|
}
|
|
|
|
inventory "$tmp/inv.json"
|
|
scan "$tmp/scan.json"
|
|
classified "$tmp/cls.json"
|
|
stable "$tmp/stable.json"
|
|
store=$tmp/store
|
|
$BIN --store "$store" write --target synth --from "$tmp/inv.json" "$tmp/scan.json" \
|
|
--classified "$tmp/cls.json" --repo oleks/claude-plugin-synth \
|
|
--run-id 2026-07-29T12-00-00Z --note "$LEAK_NOTE" >/dev/null
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (a) memory-queries: wing/room resolution and query packing =="
|
|
$BIN --store "$store" memory-queries --target synth --inventory "$tmp/inv.json" \
|
|
>"$tmp/q.json"
|
|
check "target wing" "$(py "import json;print(json.load(open('$tmp/q.json'))['wings']['target'])")" \
|
|
"synth"
|
|
check "cross-target wing" \
|
|
"$(py "import json;print(json.load(open('$tmp/q.json'))['wings']['cross_target'])")" \
|
|
"claude-plugins"
|
|
check "target room" "$(py "import json;print(json.load(open('$tmp/q.json'))['rooms']['target'])")" \
|
|
"inference-arbitrage-audits"
|
|
check "cross-target room" \
|
|
"$(py "import json;print(json.load(open('$tmp/q.json'))['rooms']['cross_target'])")" \
|
|
"inference-arbitrage-lessons"
|
|
py "
|
|
import json
|
|
d=json.load(open('$tmp/q.json'))
|
|
qs=d['queries']
|
|
assert all(len(q['query'])<=250 for q in qs), [len(q['query']) for q in qs]
|
|
assert {q['wing'] for q in qs}=={'synth','claude-plugins'}, {q['wing'] for q in qs}
|
|
joined=' '.join(q['query'] for q in qs)
|
|
for n in ('synth:alpha','synth:beta','synth:gamma'):
|
|
assert n in joined, n
|
|
print(' ok every query fits 250 chars and names every skill and agent')" || fail=1
|
|
py "
|
|
import json
|
|
d=json.load(open('$tmp/q.json'))
|
|
a=d['advisory']
|
|
assert 'never override' in a and 'FR-4.2' in a and 'FR-4.5' in a, a
|
|
print(' ok advisory says results never override the mechanical gates')" || fail=1
|
|
|
|
# A cold store with no inventory still resolves wings — the read step must not
|
|
# depend on a pass that has not run yet.
|
|
$BIN --store "$tmp/cold" memory-queries --target other >"$tmp/q2.json"
|
|
check "cold-store target wing" \
|
|
"$(py "import json;print(json.load(open('$tmp/q2.json'))['wings']['target'])")" "other"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (b) memory-notes: three drawers, correct wings and rooms =="
|
|
$BIN --store "$store" memory-notes --target synth --stable "$tmp/stable.json" >"$tmp/n.json"
|
|
check "drawer count" "$(py "import json;print(len(json.load(open('$tmp/n.json'))['drawers']))")" "3"
|
|
check "run-summary wing" "$(drawer "$tmp/n.json" run-summary wing)" "synth"
|
|
check "run-summary room" "$(drawer "$tmp/n.json" run-summary room)" "inference-arbitrage-audits"
|
|
check "judgment-calls wing" "$(drawer "$tmp/n.json" judgment-calls wing)" "synth"
|
|
check "lessons wing" "$(drawer "$tmp/n.json" cross-target-lessons wing)" "claude-plugins"
|
|
check "lessons room" "$(drawer "$tmp/n.json" cross-target-lessons room)" \
|
|
"inference-arbitrage-lessons"
|
|
check "added_by" "$(drawer "$tmp/n.json" run-summary added_by)" "inference-arbitrage"
|
|
check "source_file" "$(drawer "$tmp/n.json" run-summary source_file)" \
|
|
"inference-arbitrage/Audits/synth/2026-07-29.md"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (c) rendered content carries the numbers, framed as the plugin frames them =="
|
|
py "
|
|
import json
|
|
c=[d for d in json.load(open('$tmp/n.json'))['drawers'] if d['kind']=='run-summary'][0]['content']
|
|
assert 'synth/alpha/dispatch' in c
|
|
assert 'oleks/claude-plugin-synth#' not in c # no issue filed in this fixture
|
|
assert '20.00%' in c, 'share of spend missing'
|
|
assert '41.0%' in c, 'mechanical share missing'
|
|
assert 'dilution' in c and 'not completeness' in c, 'coverage must be framed as dilution'
|
|
assert 'llm-over-script-digest' in c
|
|
print(' ok run summary carries share, mechanical share, position, dilution framing')" || fail=1
|
|
py "
|
|
import json
|
|
c=[d for d in json.load(open('$tmp/n.json'))['drawers'] if d['kind']=='judgment-calls'][0]['content']
|
|
assert 'synth/beta/triage' in c
|
|
assert 'no overrule case' in c, 'decline reason must come off the triple'
|
|
assert 'T3 FAIL' not in c, 'triage passes T3; a wrong test line would be worse than none'
|
|
assert 'Evidence-stability downgrades' in c
|
|
assert '<!-- ia-stability: synth/beta/triage -->' in c
|
|
assert 'kotkan/claude-plugin-inference-arbitrage' in c
|
|
print(' ok judgment drawer names the decline reason and the stability downgrade')" || fail=1
|
|
py "
|
|
import json
|
|
c=[d for d in json.load(open('$tmp/n.json'))['drawers']
|
|
if d['kind']=='cross-target-lessons'][0]['content']
|
|
assert 'llm-over-script-digest 2' in c, 'positions must be counted across both tiers'
|
|
assert 'measured 2' in c
|
|
assert '1 downgrade(s)' in c
|
|
assert 'not a failed scan' in c, '2% coverage must be explained, not reported bare'
|
|
print(' ok lessons drawer counts positions across tiers and explains low coverage')" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (d) redaction: free-text snapshot fields never reach a drawer (FR-3.5) =="
|
|
# `notes` holds a caller --note and analyst reason prose. Both are in the
|
|
# snapshot on disk; neither may cross into a wing shared with other projects.
|
|
py "
|
|
import json
|
|
snap=[json.loads(l) for l in open('$store/Data/synth/snapshots.jsonl')][-1]
|
|
joined=' '.join(snap['notes'])
|
|
assert '''$LEAK_NOTE''' in joined, 'fixture is not exercising the leak path'
|
|
assert 'AKIAIOSFODNN7EXAMPLE' in joined, 'fixture is not exercising the leak path'
|
|
print(' ok the snapshot on disk really does contain the free text')" || fail=1
|
|
py "
|
|
import json
|
|
d=json.load(open('$tmp/n.json'))
|
|
blob=json.dumps(d)
|
|
for needle in ('AKIAIOSFODNN7EXAMPLE', 'hunter2', 'grep -rn', 'the user said',
|
|
'filing: unavailable'):
|
|
assert needle not in blob, needle
|
|
print(' ok no snapshot note, reason, or command argument appears in any drawer or fact')" ||
|
|
fail=1
|
|
py "
|
|
import json
|
|
d=json.load(open('$tmp/n.json'))
|
|
blob=json.dumps(d)
|
|
assert 'FR-3.5' in blob, 'each drawer states the constraint it was rendered under'
|
|
print(' ok drawers carry the redaction note themselves')" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (e) knowledge-graph facts mirror the verdicts =="
|
|
py "
|
|
import json
|
|
facts=json.load(open('$tmp/n.json'))['facts']
|
|
t={(f['subject'],f['predicate'],f['object']) for f in facts}
|
|
assert ('synth','audited_by','inference-arbitrage') in t
|
|
assert ('synth/alpha/dispatch','classified_as','llm-over-script-digest') in t
|
|
assert ('synth/beta/triage','declined_as','boundary-question') in t
|
|
assert ('synth/beta/triage','downgraded_by','evidence-stability gate FR-4.5') in t
|
|
assert all(f['valid_from']=='2026-07-29' for f in facts), facts
|
|
assert not any(f['predicate']=='filed_as' for f in facts), 'nothing was filed in this fixture'
|
|
print(' ok facts cover audited/classified/declined/downgraded, dated by run')" || fail=1
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (f) a clean run: no judgment drawer, but the lessons drawer still writes =="
|
|
# The cross-target record accumulates on every run — a 'nothing to offload'
|
|
# result is exactly the calibration evidence worth keeping.
|
|
classified "$tmp/cls2.json" --only-filed
|
|
store2=$tmp/store2
|
|
$BIN --store "$store2" write --target synth --from "$tmp/inv.json" "$tmp/scan.json" \
|
|
--classified "$tmp/cls2.json" --run-id 2026-07-29T12-00-00Z >/dev/null
|
|
$BIN --store "$store2" memory-notes --target synth >"$tmp/n2.json"
|
|
check "drawer count" "$(py "import json;print(len(json.load(open('$tmp/n2.json'))['drawers']))")" "2"
|
|
check "judgment drawer" "$(drawer "$tmp/n2.json" judgment-calls wing)" "ABSENT"
|
|
check "lessons drawer still present" "$(drawer "$tmp/n2.json" cross-target-lessons wing)" \
|
|
"claude-plugins"
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (g) memory-notes refuses a target with no history, rather than writing nothing =="
|
|
if $BIN --store "$tmp/empty" memory-notes --target synth >/dev/null 2>&1; then
|
|
echo " FAIL memory-notes succeeded with no snapshots" >&2
|
|
fail=1
|
|
else
|
|
echo " ok memory-notes exits non-zero with no snapshots recorded"
|
|
fi
|
|
|
|
# --------------------------------------------------------------------------
|
|
echo "== (h) regression #16: a stability-downgraded candidate gets its real reason =="
|
|
# The bug: decline_reason() only ever looked at a candidate's own grade, so a
|
|
# candidate boundary-classify graded high/complete-triple (it cleared FR-4.2
|
|
# fine) but that stability-classify then downgraded fell through to the
|
|
# generic "did not clear the filing threshold" line -- the opposite of what
|
|
# happened, and a direct contradiction of the FR-4.5 section rendered a few
|
|
# lines below it in the same drawer.
|
|
py "
|
|
import json
|
|
cand = {'candidate_id': 'synth/gamma/flip', 'skill': 'synth:gamma',
|
|
'position': 'llm-over-script-digest',
|
|
'determinism_tests': {t: True for t in ('T1','T2','T3','T4','T5')},
|
|
'falsifiability_triple': {'signature': True, 'examples': True, 'overrule_case': True},
|
|
'boundary_confidence': 'high', 'measurement_strength': 'thin',
|
|
'share_of_audited_spend': 0.0, 'offload_value': 0,
|
|
'escalation_path': 'escalate', 'digest_schema': 'typed rows',
|
|
# rewritten by stability-classify in real use -- 'file' cleared, then downgraded
|
|
'verdict': 'boundary-question', 'reasons': ['downgraded by stability-classify']}
|
|
json.dump({'rubric_version': '1.0.0', 'target': 'synth', 'candidates': [cand]},
|
|
open('$tmp/cls3.json','w'))
|
|
json.dump({'stability_findings': [{
|
|
'gate': 'FR-4.5', 'candidate_id': 'synth/gamma/flip', 'target': 'synth',
|
|
'skill': 'synth:gamma', 'matched_by': 'slug',
|
|
'prior': {'run_id': '2026-07-01T00-00-00Z', 'window': {'since': '2026-06-01'},
|
|
'measurement_strength': 'thin', 'share_of_audited_spend': 0.0},
|
|
'current': {'window': {'since': '2026-07-01'}, 'measurement_strength': 'measured',
|
|
'share_of_audited_spend': 0.1185},
|
|
'flips': [{'field': 'measurement_strength', 'prior': 'thin', 'current': 'measured'}],
|
|
'self_report': {'repo': 'kotkan/claude-plugin-inference-arbitrage',
|
|
'marker': '<!-- ia-stability: synth/gamma/flip -->'}}]},
|
|
open('$tmp/stable3.json','w'))
|
|
"
|
|
store3=$tmp/store3
|
|
$BIN --store "$store3" write --target synth --from "$tmp/inv.json" "$tmp/scan.json" \
|
|
--classified "$tmp/cls3.json" --run-id 2026-07-29T12-00-00Z >/dev/null
|
|
$BIN --store "$store3" memory-notes --target synth --stable "$tmp/stable3.json" >"$tmp/n3.json"
|
|
py "
|
|
import json
|
|
c=[d for d in json.load(open('$tmp/n3.json'))['drawers'] if d['kind']=='judgment-calls'][0]['content']
|
|
assert 'did not clear the filing threshold' not in c, \
|
|
'regression: a candidate stability downgraded was never actually rejected by the threshold'
|
|
assert 'cleared boundary-classify' in c and 'FR-4.5' in c, \
|
|
'the decline reason must point at the real cause'
|
|
assert 'no overrule case' not in c, 'this candidate has a complete triple, that reason is wrong too'
|
|
print(' ok stability-downgraded candidate reports the real reason, not the filing-threshold line')" ||
|
|
fail=1
|
|
|
|
exit "$fail"
|