Oleks
c8943690e0
reconstruct_invocations() only sees already-attributed turns and splits on a wall-clock idle gap (10 min default); it has no signal for unrelated activity between two genuinely separate same-skill calls, so back-to-back invocations closer together than the gap collapse into one reported invocation. This inflates the headline per-invocation figures quoted in filed issue bodies without affecting group totals (weighted_tokens, offload_value, share_of_audited_spend). Sidechain-leakage was investigated as the suspected root cause and ruled out: turn attribution (the `mine` check) is correct, nothing session-wide enters `turns`. The real, narrower defect is invocation-count inflation from idle-gap merging. Adds `totals.invocation_caveat` to offload-scan's JSON output, threads it through audit-snapshot's snapshot totals, and renders it as a note in the issue-body template so downstream issue bodies don't overstate confidence in the per-invocation figure. Fixes kotkan/claude-plugin-inference-arbitrage#12
356 lines
16 KiB
Bash
Executable File
356 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/audit-snapshot: write / list / diff over two synthetic snapshots of one
|
|
# fake target, with hand-computable numbers. Every expectation below is
|
|
# arithmetic done by hand in the comments, not a golden file — a golden file
|
|
# would pass just as happily on a wrong-but-stable answer.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/audit-snapshot
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
store=$tmp/store
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# status <diff.json> <candidate_id>
|
|
status() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
r=[c for c in d['candidates'] if c['candidate_id']==sys.argv[2]]
|
|
print(r[0]['status'] if r else 'ABSENT')" "$@"
|
|
}
|
|
metric() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
r=[c for c in d['candidates'] if c['candidate_id']==sys.argv[2]][0]
|
|
print(r[sys.argv[3]][sys.argv[4]][sys.argv[5]])" "$@"
|
|
}
|
|
jq_path() {
|
|
python3 -c "
|
|
import json,sys
|
|
d=json.load(open(sys.argv[1]))
|
|
for k in sys.argv[2].split('.'):
|
|
d=d[int(k)] if k.isdigit() else d[k]
|
|
print(d)" "$@"
|
|
}
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Fixtures. Two runs of a fake plugin 'synth'.
|
|
#
|
|
# Run 1 (2026-07-01), audited spend 10,000,000 weighted tokens:
|
|
# steady 20 invocations, offload_value 2,000,000 -> 100,000/inv, 20% share
|
|
# growing 10 invocations, offload_value 400,000 -> 40,000/inv, 4% share
|
|
# fixed 10 invocations, offload_value 1,000,000 -> 100,000/inv, 10% share
|
|
# gone 10 invocations, offload_value 500,000 -> 50,000/inv, 5% share
|
|
# drifter 10 invocations, offload_value 600,000 -> 60,000/inv, 6% share
|
|
# quiet (boundary question) 10 inv, 300,000 -> 30,000/inv, 3% share
|
|
#
|
|
# Run 2 (2026-07-08) is a QUIETER window: audited spend 5,000,000, half of run
|
|
# 1. Every absolute number falls. Volume-normalized, only `growing` grew.
|
|
# --------------------------------------------------------------------------
|
|
|
|
# inventory <file> <version> <skill...>
|
|
inventory() {
|
|
local out=$1 version=$2
|
|
shift 2
|
|
python3 - "$out" "$version" "$@" <<'EOF'
|
|
import json, sys
|
|
out, version, *skills = sys.argv[1:]
|
|
json.dump({"plugin": {"name": "synth", "version": version, "source": "oleks-local"},
|
|
"skills": [{"name": s} for s in skills], "agents": []}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# scan <file> <total> then rows on stdin: skill invocations offload_value sig
|
|
scan() {
|
|
local out=$1 total=$2
|
|
shift 2
|
|
python3 - "$out" "$total" "$@" <<'EOF'
|
|
import json, sys
|
|
out, total, *rows = sys.argv[1:]
|
|
skills = []
|
|
for row in rows:
|
|
skill, inv, value, sig = row.split(":")
|
|
skills.append({"skill": skill, "invocations": int(inv),
|
|
"offload_waste": int(value), "offload_value": int(value),
|
|
"ngrams": [{"sig": sig.split(","), "recurrences": int(inv),
|
|
"invocations": int(inv), "waste": int(value)}]})
|
|
json.dump({"window": {"since": "2026-07-01", "until": "2026-07-08", "days": 7},
|
|
"plugin": "synth", "tool_versions": {"cc_tokens_path": "/synthetic/cc-tokens"},
|
|
"coverage": {"attributed_turns": 800, "candidate_turns": 1000, "ratio": 0.8},
|
|
"totals": {"weighted_tokens": int(total), "invocations": 60,
|
|
"mechanical_share": 0.61},
|
|
"skills": skills, "agents": []}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
# classified <file> <total> then rows: candidate_id:skill:value:share:verdict
|
|
classified() {
|
|
local out=$1
|
|
shift
|
|
python3 - "$out" "$@" <<'EOF'
|
|
import json, sys
|
|
out, *rows = sys.argv[1:]
|
|
cands = []
|
|
for row in rows:
|
|
cid, skill, value, share, verdict = row.split(":")
|
|
cands.append({"candidate_id": cid, "skill": skill, "position": "llm-over-script-digest",
|
|
"determinism_tests": {t: True for t in ("T1", "T2", "T3", "T4", "T5")},
|
|
"falsifiability_triple": {"signature": True, "examples": True,
|
|
"overrule_case": True},
|
|
"boundary_confidence": "high" if verdict == "file" else "medium",
|
|
"measurement_strength": "measured",
|
|
"share_of_audited_spend": float(share), "offload_value": int(value),
|
|
"escalation_path": "escalate", "digest_schema": "typed rows",
|
|
"verdict": verdict, "reasons": ["synthetic"]})
|
|
json.dump({"rubric_version": "1.0.0", "target": "synth", "candidates": cands}, open(out, "w"))
|
|
EOF
|
|
}
|
|
|
|
echo "== run 1: write =="
|
|
inventory "$tmp/inv1.json" 1.0.0 steady growing fixed gone drifter quiet
|
|
scan "$tmp/scan1.json" 10000000 \
|
|
"steady:20:2000000:list_issues,issue_read" \
|
|
"growing:10:400000:glob,read" \
|
|
"fixed:10:1000000:grep,read,read" \
|
|
"gone:10:500000:bash,bash" \
|
|
"drifter:10:600000:label_read,issue_read" \
|
|
"quiet:10:300000:read,read"
|
|
classified "$tmp/cls1.json" \
|
|
"synth/steady/a:steady:2000000:0.20:file" \
|
|
"synth/growing/a:growing:400000:0.04:file" \
|
|
"synth/fixed/a:fixed:1000000:0.10:file" \
|
|
"synth/gone/a:gone:500000:0.05:file" \
|
|
"synth/drifter/a:drifter:600000:0.06:file" \
|
|
"synth/quiet/a:quiet:300000:0.03:boundary-question"
|
|
cat >"$tmp/issues1.json" <<'EOF'
|
|
{"synth/fixed/a": "kotkan/claude-plugin-synth#1",
|
|
"synth/steady/a": "kotkan/claude-plugin-synth#2",
|
|
"synth/gone/a": "kotkan/claude-plugin-synth#3"}
|
|
EOF
|
|
$BIN --store "$store" write --target synth --run-id 2026-07-01T00:00:00Z \
|
|
--from "$tmp/inv1.json" "$tmp/scan1.json" --classified "$tmp/cls1.json" \
|
|
--issue-map "$tmp/issues1.json" --repo kotkan/claude-plugin-synth >"$tmp/w1.json"
|
|
check "run 1 candidates" "$(jq_path "$tmp/w1.json" candidates)" "5"
|
|
check "run 1 boundary questions" "$(jq_path "$tmp/w1.json" boundary_questions)" "1"
|
|
|
|
echo "== list after one run =="
|
|
$BIN --store "$store" list --target synth >"$tmp/l1.json"
|
|
check "runs" "$(jq_path "$tmp/l1.json" runs.0.run_id)" "2026-07-01T00:00:00Z"
|
|
check "weighted tokens" "$(jq_path "$tmp/l1.json" runs.0.weighted_tokens)" "10000000"
|
|
|
|
echo "== diff with one snapshot refuses =="
|
|
if $BIN --store "$store" diff --target synth >/dev/null 2>&1; then
|
|
echo " FAIL diff succeeded with a single snapshot" >&2
|
|
fail=1
|
|
else
|
|
echo " ok diff needs two snapshots"
|
|
fi
|
|
|
|
echo "== run 2: a quieter window =="
|
|
# `gone`'s skill is dropped from the target entirely -> stale.
|
|
# `drifter` keeps its candidate id but its tool sequence changes -> signature-drifted.
|
|
inventory "$tmp/inv2.json" 1.1.0 steady growing fixed drifter quiet newbie
|
|
scan "$tmp/scan2.json" 5000000 \
|
|
"steady:10:1000000:list_issues,issue_read" \
|
|
"growing:5:400000:glob,read" \
|
|
"fixed:5:50000:grep,read,read" \
|
|
"drifter:10:600000:label_read,issue_read,issue_write" \
|
|
"quiet:5:250000:read,read" \
|
|
"newbie:5:300000:web_fetch,read"
|
|
# steady : 1,000,000/10 = 100,000/inv (unchanged), share 20% -> persisting
|
|
# growing: 400,000/ 5 = 80,000/inv (+100%), share 8% -> grown
|
|
# fixed : dropped off the candidate list; issue #1 closed; remeasured 50,000
|
|
# over 5 invocations = 1% of 5,000,000, under the 2% threshold with
|
|
# >= 3 invocations -> resolved
|
|
# gone : skill absent from inventory 1.1.0 -> stale
|
|
# drifter: signature hash changes -> signature-drifted
|
|
# quiet : boundary question, 250,000/5 = 50,000/inv vs 30,000 (+67%) -> grown,
|
|
# so it is promoted
|
|
classified "$tmp/cls2.json" \
|
|
"synth/steady/a:steady:1000000:0.20:file" \
|
|
"synth/growing/a:growing:400000:0.08:file" \
|
|
"synth/drifter/a:drifter:600000:0.12:file" \
|
|
"synth/newbie/a:newbie:300000:0.06:file" \
|
|
"synth/quiet/a:quiet:250000:0.05:boundary-question"
|
|
cat >"$tmp/issues2.json" <<'EOF'
|
|
{"synth/steady/a": "kotkan/claude-plugin-synth#2",
|
|
"synth/drifter/a": "kotkan/claude-plugin-synth#4"}
|
|
EOF
|
|
$BIN --store "$store" write --target synth --run-id 2026-07-08T00:00:00Z \
|
|
--from "$tmp/inv2.json" "$tmp/scan2.json" --classified "$tmp/cls2.json" \
|
|
--issue-map "$tmp/issues2.json" --repo kotkan/claude-plugin-synth >/dev/null
|
|
|
|
cat >"$tmp/state.json" <<'EOF'
|
|
{"kotkan/claude-plugin-synth#1": "closed",
|
|
"kotkan/claude-plugin-synth#2": "open",
|
|
"kotkan/claude-plugin-synth#3": "closed"}
|
|
EOF
|
|
$BIN --store "$store" diff --target synth --issue-state "$tmp/state.json" >"$tmp/diff.json"
|
|
|
|
echo "== per-candidate statuses =="
|
|
check "steady" "$(status "$tmp/diff.json" synth/steady/a)" "persisting"
|
|
check "growing" "$(status "$tmp/diff.json" synth/growing/a)" "grown"
|
|
check "fixed" "$(status "$tmp/diff.json" synth/fixed/a)" "resolved"
|
|
check "gone" "$(status "$tmp/diff.json" synth/gone/a)" "stale"
|
|
check "drifter" "$(status "$tmp/diff.json" synth/drifter/a)" "signature-drifted"
|
|
check "newbie" "$(status "$tmp/diff.json" synth/newbie/a)" "new"
|
|
check "quiet" "$(status "$tmp/diff.json" synth/quiet/a)" "grown"
|
|
|
|
echo "== volume normalization: the window halved, nothing may read as progress =="
|
|
check "volume rel" "$(jq_path "$tmp/diff.json" volume.weighted_tokens.rel)" "-0.5"
|
|
# steady's absolute offload_value halved (2,000,000 -> 1,000,000) while its
|
|
# cost per invocation and share are flat. Absolute must not drive the verdict.
|
|
check "steady abs delta" "$(metric "$tmp/diff.json" synth/steady/a context_only offload_value delta)" "-1000000"
|
|
check "steady cost/inv rel" "$(metric "$tmp/diff.json" synth/steady/a primary cost_per_invocation rel)" "0.0"
|
|
check "growing cost/inv rel" "$(metric "$tmp/diff.json" synth/growing/a primary cost_per_invocation rel)" "1.0"
|
|
check "growing share rel" "$(metric "$tmp/diff.json" synth/growing/a primary share_of_plugin rel)" "1.0"
|
|
|
|
echo "== follow-through =="
|
|
check "unaddressed count" "$(python3 -c "import json;print(len(json.load(open('$tmp/diff.json'))['unaddressed']))")" "3"
|
|
check "unaddressed issues" "$(jq_path "$tmp/diff.json" unaddressed_issues.0)" "kotkan/claude-plugin-synth#2"
|
|
check "promotions" "$(jq_path "$tmp/diff.json" promotions.0)" "synth/quiet/a"
|
|
check "rubric comparable" "$(jq_path "$tmp/diff.json" rubric_comparable)" "True"
|
|
|
|
echo "== claimed-fixed-unconfirmed: closed issue, no remeasurement =="
|
|
# Same diff, but #1 closed with `fixed`'s signature absent from run 2's
|
|
# measurements. Rebuild run 2 without the `fixed` skill's scan row.
|
|
store2=$tmp/store2
|
|
scan "$tmp/scan2b.json" 5000000 \
|
|
"steady:10:1000000:list_issues,issue_read" \
|
|
"growing:5:400000:glob,read"
|
|
inventory "$tmp/inv2b.json" 1.1.0 steady growing fixed
|
|
classified "$tmp/cls2b.json" \
|
|
"synth/steady/a:steady:1000000:0.20:file" \
|
|
"synth/growing/a:growing:400000:0.08:file"
|
|
$BIN --store "$store2" write --target synth --run-id 2026-07-01T00:00:00Z \
|
|
--from "$tmp/inv1.json" "$tmp/scan1.json" --classified "$tmp/cls1.json" \
|
|
--issue-map "$tmp/issues1.json" >/dev/null
|
|
$BIN --store "$store2" write --target synth --run-id 2026-07-08T00:00:00Z \
|
|
--from "$tmp/inv2b.json" "$tmp/scan2b.json" --classified "$tmp/cls2b.json" >/dev/null
|
|
$BIN --store "$store2" diff --target synth --issue-state "$tmp/state.json" >"$tmp/diff2.json"
|
|
check "closed but unmeasured" "$(status "$tmp/diff2.json" synth/fixed/a)" "claimed-fixed-unconfirmed"
|
|
|
|
echo "== rendered pages =="
|
|
$BIN --store "$store" pages --target synth >"$tmp/pages.json"
|
|
check "jsonl page" "$(jq_path "$tmp/pages.json" pages.0.page)" "Data/synth/snapshots.jsonl"
|
|
check "dated page" "$(jq_path "$tmp/pages.json" pages.1.page)" "Audits/synth/2026-07-08"
|
|
check "latest page" "$(jq_path "$tmp/pages.json" pages.2.page)" "Audits/synth/Latest"
|
|
check "jsonl lines" "$(wc -l <"$store/Data/synth/snapshots.jsonl")" "2"
|
|
for want in "still unaddressed" "growing cost" "Open candidates" "Run history"; do
|
|
if grep -qF "$want" "$store/Audits/synth/Latest.md"; then
|
|
echo " ok Latest.md carries \"$want\""
|
|
else
|
|
echo " FAIL Latest.md missing \"$want\"" >&2
|
|
fail=1
|
|
fi
|
|
done
|
|
|
|
echo "== --note reaches the snapshot and the summary page (FR-6.4 seam) =="
|
|
store4=$tmp/store4
|
|
$BIN --store "$store4" write --target synth --run-id 2026-07-01T00:00:00Z \
|
|
--from "$tmp/inv1.json" "$tmp/scan1.json" --classified "$tmp/cls1.json" \
|
|
--note "filing: unavailable — no writable repo for synth" \
|
|
--note "second note" >/dev/null
|
|
check "note stored" "$(python3 -c "
|
|
import json
|
|
s=[json.loads(l) for l in open('$store4/Data/synth/snapshots.jsonl')][0]
|
|
print(s['notes'][-2])")" "filing: unavailable — no writable repo for synth"
|
|
if grep -qF "filing: unavailable" "$store4/Audits/synth/2026-07-01.md"; then
|
|
echo " ok note rendered under ## Notes"
|
|
else
|
|
echo " FAIL note missing from the summary page" >&2
|
|
fail=1
|
|
fi
|
|
|
|
echo "== import is idempotent =="
|
|
store3=$tmp/store3
|
|
$BIN --store "$store3" import --target synth --snapshots "$store/Data/synth/snapshots.jsonl" >"$tmp/i1.json"
|
|
check "imported" "$(jq_path "$tmp/i1.json" imported)" "2"
|
|
$BIN --store "$store3" import --target synth --snapshots "$store/Data/synth/snapshots.jsonl" >"$tmp/i2.json"
|
|
check "re-import skipped" "$(jq_path "$tmp/i2.json" skipped)" "2"
|
|
check "still two runs" "$(wc -l <"$store3/Data/synth/snapshots.jsonl")" "2"
|
|
|
|
echo "== a window the candidate sat out is 'unmeasured', never 'shrunk' (FR-5.4) =="
|
|
# The quiet-window artefact one level up from absolute tokens: `steady` runs 20
|
|
# times in run 1 and 0 times in run 5. Folding cost-per-invocation to 0 on a
|
|
# zero denominator would report -100% and read as a large improvement, when in
|
|
# fact nothing was observed at all.
|
|
store5=$tmp/store5
|
|
inventory "$tmp/inv5.json" 1.0.0 steady
|
|
scan "$tmp/scan5a.json" 10000000 "steady:20:2000000:list_issues,issue_read"
|
|
classified "$tmp/cls5a.json" "synth/steady/a:steady:2000000:0.20:file"
|
|
$BIN --store "$store5" write --target synth --run-id 2026-07-01T00:00:00Z \
|
|
--from "$tmp/inv5.json" "$tmp/scan5a.json" --classified "$tmp/cls5a.json" >/dev/null
|
|
# same signature, but zero invocations this window
|
|
scan "$tmp/scan5b.json" 8000000 "steady:0:0:list_issues,issue_read"
|
|
classified "$tmp/cls5b.json" "synth/steady/a:steady:0:0.0:file"
|
|
$BIN --store "$store5" write --target synth --run-id 2026-07-08T00:00:00Z \
|
|
--from "$tmp/inv5.json" "$tmp/scan5b.json" --classified "$tmp/cls5b.json" >/dev/null
|
|
$BIN --store "$store5" diff --target synth >"$tmp/d5.json"
|
|
check "status is unmeasured, not shrunk" "$(status "$tmp/d5.json" synth/steady/a)" "unmeasured"
|
|
check "cost/invocation is null, not 0" \
|
|
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation to)" "None"
|
|
check "no fabricated rel" \
|
|
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation rel)" "None"
|
|
check "flagged unmeasured" \
|
|
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation unmeasured)" "True"
|
|
check "counted in the summary" "$(jq_path "$tmp/d5.json" summary.unmeasured)" "1"
|
|
check "not counted as shrunk" "$(jq_path "$tmp/d5.json" summary.shrunk)" "0"
|
|
if grep -qF "not run" "$store5/Audits/synth/Latest.md" 2>/dev/null ||
|
|
grep -qF "not run" "$store5/Audits/synth/2026-07-08.md"; then
|
|
echo " ok page renders 'not run' rather than 0"
|
|
else
|
|
echo " FAIL page rendered a zero cost/invocation for a window with no invocations" >&2
|
|
fail=1
|
|
fi
|
|
|
|
echo "== kotkan/claude-plugin-inference-arbitrage#12: scan's invocation_caveat reaches the summary page =="
|
|
store6=$tmp/store6
|
|
inventory "$tmp/inv6.json" 1.0.0 steady
|
|
scan "$tmp/scan6.json" 10000000 "steady:20:2000000:list_issues,issue_read"
|
|
python3 -c "
|
|
import json
|
|
d = json.load(open('$tmp/scan6.json'))
|
|
d['totals']['invocation_caveat'] = ('invocation counts are a lower bound: two same-skill '
|
|
'calls within 10 minutes of each other on the wall clock are merged into one reported '
|
|
'invocation')
|
|
json.dump(d, open('$tmp/scan6.json', 'w'))
|
|
"
|
|
classified "$tmp/cls6.json" "synth/steady/a:steady:2000000:0.20:file"
|
|
$BIN --store "$store6" write --target synth --run-id 2026-07-01T00:00:00Z \
|
|
--from "$tmp/inv6.json" "$tmp/scan6.json" --classified "$tmp/cls6.json" >/dev/null
|
|
check "invocation_caveat stored in totals" "$(python3 -c "
|
|
import json
|
|
s=[json.loads(l) for l in open('$store6/Data/synth/snapshots.jsonl')][0]
|
|
print('lower bound' in s['totals']['invocation_caveat'])")" "True"
|
|
if grep -qF "lower bound" "$store6/Audits/synth/2026-07-01.md"; then
|
|
echo " ok invocation caveat rendered in the audit page"
|
|
else
|
|
echo " FAIL invocation caveat missing from the audit page" >&2
|
|
fail=1
|
|
fi
|
|
|
|
echo "== a re-run of the same run_id is refused =="
|
|
if $BIN --store "$store" write --target synth --run-id 2026-07-08T00:00:00Z \
|
|
--from "$tmp/inv2.json" "$tmp/scan2.json" --classified "$tmp/cls2.json" >/dev/null 2>&1; then
|
|
echo " FAIL duplicate run_id accepted" >&2
|
|
fail=1
|
|
else
|
|
echo " ok duplicate run_id refused"
|
|
fi
|
|
|
|
exit "$fail"
|