Oleks
bdb62da898
Streams ~/.claude/projects/**/*.jsonl once, filters to turns attributed to the target plugin (attributionPlugin / attributionSkill / attributionAgent), de-duplicates by (message.id, requestId) taking max on every usage field, and computes the spec.md §5.2 dynamic signals: MTR, offload_waste, recurring tool-signature n-grams, read amplification, retry density, judgment density, fan-out multiplier, and the composite offload_value — plus the FR-3.4 attribution-coverage headline. Token and cost arithmetic is taken from token-budget's cc-tokens, never reimplemented. It is imported as a module rather than shelled out with --json, because cc-tokens' subcommands aggregate by day/session/project/model and none can answer "weighted tokens for this set of turns" — the attribution fields are not on its CLI surface. Its Rec.cost / Rec.weighted still compute every number here, so the pricing table and cache multipliers keep exactly one home. Two deviations from design/plan.md §4.1, both because the transcripts say otherwise: agents attribute via attributionAgent on the subagent sidechain files, not agentName (which is a separate `type: "agent-name"` line mapping a session to a display label); and attributionSkill is sometimes bare rather than plugin-qualified, so it is normalized. Tool arguments are masked to <path>/<n>/<sha>/<url>/<str> before entering any emitted structure, and Bash commands are reduced to bare executable words — FR-3.5, asserted in the suite. Fixtures are synthetic by construction; no real transcript content is in this repo. Real-history check: 30-day window over a 1.7 GB history, 94s wall, 29 MB peak RSS, one process.
122 lines
5.5 KiB
Bash
Executable File
122 lines
5.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Asserts bin/offload-scan against tests/fixtures/transcripts, which are
|
|
# SYNTHETIC by construction (see fixtures/transcripts/generate.py — no real
|
|
# transcript content may ever enter this repo, per design/spec.md FR-3.5).
|
|
#
|
|
# The arithmetic each assertion checks, once, here:
|
|
# model claude-sonnet-5 -> tier 0.4 (input $2/Mtok / 5)
|
|
# inline turn = 0.4 * (100000 cache_read * 0.1 + out * 5)
|
|
# = 4200 at out=100, 8000 at out=2000
|
|
# sidechain turn = 0.4 * (200000 * 0.1 + 100 * 5) = 8200
|
|
# mine-me 9 turns @4200 = 37800 weighted, all mechanical
|
|
# think-hard 8000 + 4200 + 4200 = 16400 weighted, 0 mechanical
|
|
# fetcher 2 @8200 = 16400 weighted, both sidechain
|
|
# offload_value(mine-me) = 37800 * (1 - 0) * (1 + log2(3)) = 97711
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
|
|
|
OUT=$(python3 bin/offload-scan --plugin fixture-plugin \
|
|
--root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json)
|
|
fail=0
|
|
|
|
check() {
|
|
local desc="$1" expr="$2"
|
|
if ! python3 -c "
|
|
import json, sys
|
|
d = json.loads(sys.stdin.read())
|
|
assert ($expr), 'FAILED: $desc'
|
|
" <<<"$OUT"; then
|
|
echo "FAIL: $desc" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: $desc"
|
|
fi
|
|
}
|
|
|
|
skill() { echo "[s for s in d['skills'] if s['skill'] == 'fixture-plugin:$1'][0]"; }
|
|
|
|
# --- 3.1 attribution filter and de-dup ------------------------------------
|
|
check "only target-plugin turns are attributed" "d['coverage']['attributed_turns'] == 14"
|
|
check "two skills and one agent found" "len(d['skills']) == 2 and len(d['agents']) == 1"
|
|
check "agent attributed via attributionAgent" "d['agents'][0]['agent'] == 'fixture-plugin:fetcher-agent'"
|
|
|
|
# --- 3.2 invocation reconstruction ----------------------------------------
|
|
check "idle gap splits mine-me into 3 invocations" "$(skill mine-me)['invocations'] == 3"
|
|
check "think-hard is one contiguous invocation" "$(skill think-hard)['invocations'] == 1"
|
|
|
|
# --- 3.3 turn classification ----------------------------------------------
|
|
check "mine-me is wholly mechanical" "$(skill mine-me)['mtr'] == 1.0"
|
|
check "a Write turn is never mechanical" "$(skill think-hard)['mtr'] == 0.0"
|
|
check "a no-tool 2000-token turn is judgment" "$(skill think-hard)['judgment_density'] == 0.333"
|
|
check "the turn after a tool error is a retry" "$(skill think-hard)['retry_density'] == 0.333"
|
|
|
|
# --- 3.4 signature normalization and masking ------------------------------
|
|
check "signatures carry shapes, not literals" "
|
|
$(skill mine-me)['ngrams'][0]['sig'] == ['Grep(path=<path>, pattern=<str>)',
|
|
'Read(file_path=<path>)',
|
|
'issue_read(index=<n>)']
|
|
"
|
|
check "MCP tool names are shortened to their last segment" "
|
|
all('mcp__' not in s for s in $(skill mine-me)['ngrams'][0]['sig'])
|
|
"
|
|
for literal in needle haystack_line "/fake/y.md" "out.md" "deliberation"; do
|
|
if grep -qF -- "$literal" <<<"$OUT"; then
|
|
echo "FAIL: raw fixture literal '$literal' leaked into output (FR-3.5)" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: '$literal' does not appear in output"
|
|
fi
|
|
done
|
|
|
|
# --- 3.5 n-gram mining ------------------------------------------------------
|
|
check "the recurring 3-gram is found once, maximal" "len($(skill mine-me)['ngrams']) == 1"
|
|
check "3-gram recurs in 3 distinct invocations" "
|
|
$(skill mine-me)['ngrams'][0]['recurrences'] == 3
|
|
and $(skill mine-me)['ngrams'][0]['invocations'] == 3
|
|
"
|
|
check "a one-invocation skill mines no n-grams" "$(skill think-hard)['ngrams'] == []"
|
|
|
|
# --- 3.6 metrics -------------------------------------------------------------
|
|
check "weighted tokens come from the cc-tokens tier weighting" "
|
|
$(skill mine-me)['weighted_tokens'] == 37800
|
|
and $(skill think-hard)['weighted_tokens'] == 16400
|
|
and d['agents'][0]['weighted_tokens'] == 16400
|
|
"
|
|
check "plugin total is the sum of its groups" "d['totals']['weighted_tokens'] == 70600"
|
|
check "offload_waste counts only mechanical turns" "
|
|
$(skill mine-me)['offload_waste'] == 37800 and $(skill think-hard)['offload_waste'] == 0
|
|
"
|
|
check "offload_value = waste * (1-judgment) * (1+log2(recurrences))" "
|
|
$(skill mine-me)['offload_value'] == 97711
|
|
"
|
|
check "read amplification is high when a haystack yields one word" "
|
|
$(skill mine-me)['read_amplification'] > 100
|
|
"
|
|
check "fanout multiplier is the sidechain context multiple" "d['agents'][0]['fanout_multiplier'] == 2.0"
|
|
check "no sidechain turns means no fanout" "$(skill mine-me)['fanout_multiplier'] == 1.0"
|
|
|
|
# --- 3.7 attribution coverage -------------------------------------------------
|
|
check "coverage denominator includes the unattributed turn in an active session" "
|
|
d['coverage']['candidate_turns'] == 15 and d['coverage']['ratio'] == 0.933
|
|
"
|
|
check "a session the target never touched is excluded entirely" "d['window']['sessions'] == 3"
|
|
|
|
# --- failure modes ------------------------------------------------------------
|
|
if python3 bin/offload-scan --plugin fixture-plugin --root /nonexistent-root >/dev/null 2>&1; then
|
|
echo "FAIL: missing transcript root should exit non-zero" >&2
|
|
fail=1
|
|
else
|
|
echo "ok: missing transcript root exits non-zero"
|
|
fi
|
|
|
|
EMPTY=$(python3 bin/offload-scan --plugin no-such-plugin \
|
|
--root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json)
|
|
if [ "$(python3 -c "import json,sys; d=json.load(sys.stdin); print(d['coverage']['attributed_turns'], len(d['skills']))" <<<"$EMPTY")" = "0 0" ]; then
|
|
echo "ok: a plugin with no history yields an empty, non-crashing report"
|
|
else
|
|
echo "FAIL: unknown plugin should yield an empty report" >&2
|
|
fail=1
|
|
fi
|
|
|
|
exit "$fail"
|