Oleks
c8943690e0
reconstruct_invocations() only sees already-attributed turns and splits on a wall-clock idle gap (10 min default); it has no signal for unrelated activity between two genuinely separate same-skill calls, so back-to-back invocations closer together than the gap collapse into one reported invocation. This inflates the headline per-invocation figures quoted in filed issue bodies without affecting group totals (weighted_tokens, offload_value, share_of_audited_spend). Sidechain-leakage was investigated as the suspected root cause and ruled out: turn attribution (the `mine` check) is correct, nothing session-wide enters `turns`. The real, narrower defect is invocation-count inflation from idle-gap merging. Adds `totals.invocation_caveat` to offload-scan's JSON output, threads it through audit-snapshot's snapshot totals, and renders it as a note in the issue-body template so downstream issue bodies don't overstate confidence in the per-invocation figure. Fixes kotkan/claude-plugin-inference-arbitrage#12
132 lines
6.1 KiB
Bash
Executable File
132 lines
6.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Asserts bin/offload-scan against tests/fixtures/transcripts, which are
|
|
# SYNTHETIC by construction (see fixtures/transcripts/generate.py — no real
|
|
# transcript content may ever enter this repo, per design/spec.md FR-3.5).
|
|
#
|
|
# The arithmetic each assertion checks, once, here:
|
|
# model claude-sonnet-5 -> tier 0.4 (input $2/Mtok / 5)
|
|
# inline turn = 0.4 * (100000 cache_read * 0.1 + out * 5)
|
|
# = 4200 at out=100, 8000 at out=2000
|
|
# sidechain turn = 0.4 * (200000 * 0.1 + 100 * 5) = 8200
|
|
# mine-me 9 turns @4200 = 37800 weighted, all mechanical
|
|
# think-hard 8000 + 4200 + 4200 = 16400 weighted, 0 mechanical
|
|
# fetcher 2 @8200 = 16400 weighted, both sidechain
|
|
# offload_value(mine-me) = 37800 * (1 - 0) * (1 + log2(3)) = 97711
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
|
|
|
OUT=$(python3 bin/offload-scan --plugin fixture-plugin \
|
|
--root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json)
|
|
fail=0
|
|
|
|
check() {
|
|
local desc="$1" expr="$2"
|
|
if ! python3 -c "
|
|
import json, sys
|
|
d = json.loads(sys.stdin.read())
|
|
assert ($expr), 'FAILED: $desc'
|
|
" <<<"$OUT"; then
|
|
echo "FAIL: $desc" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: $desc"
|
|
fi
|
|
}
|
|
|
|
skill() { echo "[s for s in d['skills'] if s['skill'] == 'fixture-plugin:$1'][0]"; }
|
|
|
|
# --- 3.1 attribution filter and de-dup ------------------------------------
|
|
check "only target-plugin turns are attributed" "d['coverage']['attributed_turns'] == 14"
|
|
check "two skills and one agent found" "len(d['skills']) == 2 and len(d['agents']) == 1"
|
|
check "agent attributed via attributionAgent" "d['agents'][0]['agent'] == 'fixture-plugin:fetcher-agent'"
|
|
|
|
# --- 3.2 invocation reconstruction ----------------------------------------
|
|
check "idle gap splits mine-me into 3 invocations" "$(skill mine-me)['invocations'] == 3"
|
|
check "think-hard is one contiguous invocation" "$(skill think-hard)['invocations'] == 1"
|
|
# kotkan/claude-plugin-inference-arbitrage#12: reconstruct_invocations() only
|
|
# sees already-attributed turns and splits on wall-clock idle gap alone, so
|
|
# it cannot tell two genuinely separate same-skill calls apart from one
|
|
# contiguous call if they land inside the gap. The emitted totals must say so
|
|
# rather than let the headline invocation/cost-per-invocation figures imply
|
|
# more confidence than the reconstruction can support.
|
|
check "totals carry a lower-bound caveat naming the idle gap threshold" "
|
|
'lower bound' in d['totals']['invocation_caveat']
|
|
and '10 minutes' in d['totals']['invocation_caveat']
|
|
"
|
|
|
|
# --- 3.3 turn classification ----------------------------------------------
|
|
check "mine-me is wholly mechanical" "$(skill mine-me)['mtr'] == 1.0"
|
|
check "a Write turn is never mechanical" "$(skill think-hard)['mtr'] == 0.0"
|
|
check "a no-tool 2000-token turn is judgment" "$(skill think-hard)['judgment_density'] == 0.333"
|
|
check "the turn after a tool error is a retry" "$(skill think-hard)['retry_density'] == 0.333"
|
|
|
|
# --- 3.4 signature normalization and masking ------------------------------
|
|
check "signatures carry shapes, not literals" "
|
|
$(skill mine-me)['ngrams'][0]['sig'] == ['Grep(path=<path>, pattern=<str>)',
|
|
'Read(file_path=<path>)',
|
|
'issue_read(index=<n>)']
|
|
"
|
|
check "MCP tool names are shortened to their last segment" "
|
|
all('mcp__' not in s for s in $(skill mine-me)['ngrams'][0]['sig'])
|
|
"
|
|
for literal in needle haystack_line "/fake/y.md" "out.md" "deliberation"; do
|
|
if grep -qF -- "$literal" <<<"$OUT"; then
|
|
echo "FAIL: raw fixture literal '$literal' leaked into output (FR-3.5)" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: '$literal' does not appear in output"
|
|
fi
|
|
done
|
|
|
|
# --- 3.5 n-gram mining ------------------------------------------------------
|
|
check "the recurring 3-gram is found once, maximal" "len($(skill mine-me)['ngrams']) == 1"
|
|
check "3-gram recurs in 3 distinct invocations" "
|
|
$(skill mine-me)['ngrams'][0]['recurrences'] == 3
|
|
and $(skill mine-me)['ngrams'][0]['invocations'] == 3
|
|
"
|
|
check "a one-invocation skill mines no n-grams" "$(skill think-hard)['ngrams'] == []"
|
|
|
|
# --- 3.6 metrics -------------------------------------------------------------
|
|
check "weighted tokens come from the cc-tokens tier weighting" "
|
|
$(skill mine-me)['weighted_tokens'] == 37800
|
|
and $(skill think-hard)['weighted_tokens'] == 16400
|
|
and d['agents'][0]['weighted_tokens'] == 16400
|
|
"
|
|
check "plugin total is the sum of its groups" "d['totals']['weighted_tokens'] == 70600"
|
|
check "offload_waste counts only mechanical turns" "
|
|
$(skill mine-me)['offload_waste'] == 37800 and $(skill think-hard)['offload_waste'] == 0
|
|
"
|
|
check "offload_value = waste * (1-judgment) * (1+log2(recurrences))" "
|
|
$(skill mine-me)['offload_value'] == 97711
|
|
"
|
|
check "read amplification is high when a haystack yields one word" "
|
|
$(skill mine-me)['read_amplification'] > 100
|
|
"
|
|
check "fanout multiplier is the sidechain context multiple" "d['agents'][0]['fanout_multiplier'] == 2.0"
|
|
check "no sidechain turns means no fanout" "$(skill mine-me)['fanout_multiplier'] == 1.0"
|
|
|
|
# --- 3.7 attribution coverage -------------------------------------------------
|
|
check "coverage denominator includes the unattributed turn in an active session" "
|
|
d['coverage']['candidate_turns'] == 15 and d['coverage']['ratio'] == 0.933
|
|
"
|
|
check "a session the target never touched is excluded entirely" "d['window']['sessions'] == 3"
|
|
|
|
# --- failure modes ------------------------------------------------------------
|
|
if python3 bin/offload-scan --plugin fixture-plugin --root /nonexistent-root >/dev/null 2>&1; then
|
|
echo "FAIL: missing transcript root should exit non-zero" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: missing transcript root exits non-zero"
|
|
fi
|
|
|
|
EMPTY=$(python3 bin/offload-scan --plugin no-such-plugin \
|
|
--root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json)
|
|
if [ "$(python3 -c "import json,sys; d=json.load(sys.stdin); print(d['coverage']['attributed_turns'], len(d['skills']))" <<<"$EMPTY")" = "0 0" ]; then
|
|
echo "ok: a plugin with no history yields an empty, non-crashing report"
|
|
else
|
|
echo "FAIL: unknown plugin should yield an empty report" >&2
|
|
fail=1
|
|
fi
|
|
|
|
exit "$fail"
|