#!/usr/bin/env bash # Asserts bin/offload-scan against tests/fixtures/transcripts, which are # SYNTHETIC by construction (see fixtures/transcripts/generate.py — no real # transcript content may ever enter this repo, per design/spec.md FR-3.5). # # The arithmetic each assertion checks, once, here: # model claude-sonnet-5 -> tier 0.4 (input $2/Mtok / 5) # inline turn = 0.4 * (100000 cache_read * 0.1 + out * 5) # = 4200 at out=100, 8000 at out=2000 # sidechain turn = 0.4 * (200000 * 0.1 + 100 * 5) = 8200 # mine-me 9 turns @4200 = 37800 weighted, all mechanical # think-hard 8000 + 4200 + 4200 = 16400 weighted, 0 mechanical # fetcher 2 @8200 = 16400 weighted, both sidechain # offload_value(mine-me) = 37800 * (1 - 0) * (1 + log2(3)) = 97711 set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/.." OUT=$(python3 bin/offload-scan --plugin fixture-plugin \ --root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json) fail=0 check() { local desc="$1" expr="$2" if ! python3 -c " import json, sys d = json.loads(sys.stdin.read()) assert ($expr), 'FAILED: $desc' " <<<"$OUT"; then echo "FAIL: $desc" >&2 fail=1 else echo "ok: $desc" fi } skill() { echo "[s for s in d['skills'] if s['skill'] == 'fixture-plugin:$1'][0]"; } # --- 3.1 attribution filter and de-dup ------------------------------------ check "only target-plugin turns are attributed" "d['coverage']['attributed_turns'] == 14" check "two skills and one agent found" "len(d['skills']) == 2 and len(d['agents']) == 1" check "agent attributed via attributionAgent" "d['agents'][0]['agent'] == 'fixture-plugin:fetcher-agent'" # --- 3.2 invocation reconstruction ---------------------------------------- check "idle gap splits mine-me into 3 invocations" "$(skill mine-me)['invocations'] == 3" check "think-hard is one contiguous invocation" "$(skill think-hard)['invocations'] == 1" # kotkan/claude-plugin-inference-arbitrage#12: reconstruct_invocations() only # sees already-attributed turns and splits on wall-clock idle gap alone, so # it cannot tell two genuinely separate same-skill calls apart from one # contiguous call if they land inside the gap. The emitted totals must say so # rather than let the headline invocation/cost-per-invocation figures imply # more confidence than the reconstruction can support. check "totals carry a lower-bound caveat naming the idle gap threshold" " 'lower bound' in d['totals']['invocation_caveat'] and '10 minutes' in d['totals']['invocation_caveat'] " # --- 3.3 turn classification ---------------------------------------------- check "mine-me is wholly mechanical" "$(skill mine-me)['mtr'] == 1.0" check "a Write turn is never mechanical" "$(skill think-hard)['mtr'] == 0.0" check "a no-tool 2000-token turn is judgment" "$(skill think-hard)['judgment_density'] == 0.333" check "the turn after a tool error is a retry" "$(skill think-hard)['retry_density'] == 0.333" # --- 3.4 signature normalization and masking ------------------------------ check "signatures carry shapes, not literals" " $(skill mine-me)['ngrams'][0]['sig'] == ['Grep(path=, pattern=)', 'Read(file_path=)', 'issue_read(index=)'] " check "MCP tool names are shortened to their last segment" " all('mcp__' not in s for s in $(skill mine-me)['ngrams'][0]['sig']) " for literal in needle haystack_line "/fake/y.md" "out.md" "deliberation"; do if grep -qF -- "$literal" <<<"$OUT"; then echo "FAIL: raw fixture literal '$literal' leaked into output (FR-3.5)" >&2 fail=1 else echo "ok: '$literal' does not appear in output" fi done # --- 3.5 n-gram mining ------------------------------------------------------ check "the recurring 3-gram is found once, maximal" "len($(skill mine-me)['ngrams']) == 1" check "3-gram recurs in 3 distinct invocations" " $(skill mine-me)['ngrams'][0]['recurrences'] == 3 and $(skill mine-me)['ngrams'][0]['invocations'] == 3 " check "a one-invocation skill mines no n-grams" "$(skill think-hard)['ngrams'] == []" # --- 3.6 metrics ------------------------------------------------------------- check "weighted tokens come from the cc-tokens tier weighting" " $(skill mine-me)['weighted_tokens'] == 37800 and $(skill think-hard)['weighted_tokens'] == 16400 and d['agents'][0]['weighted_tokens'] == 16400 " check "plugin total is the sum of its groups" "d['totals']['weighted_tokens'] == 70600" check "offload_waste counts only mechanical turns" " $(skill mine-me)['offload_waste'] == 37800 and $(skill think-hard)['offload_waste'] == 0 " check "offload_value = waste * (1-judgment) * (1+log2(recurrences))" " $(skill mine-me)['offload_value'] == 97711 " check "read amplification is high when a haystack yields one word" " $(skill mine-me)['read_amplification'] > 100 " check "fanout multiplier is the sidechain context multiple" "d['agents'][0]['fanout_multiplier'] == 2.0" check "no sidechain turns means no fanout" "$(skill mine-me)['fanout_multiplier'] == 1.0" # --- 3.7 attribution coverage ------------------------------------------------- check "coverage denominator includes the unattributed turn in an active session" " d['coverage']['candidate_turns'] == 15 and d['coverage']['ratio'] == 0.933 " check "a session the target never touched is excluded entirely" "d['window']['sessions'] == 3" # --- failure modes ------------------------------------------------------------ if python3 bin/offload-scan --plugin fixture-plugin --root /nonexistent-root >/dev/null 2>&1; then echo "FAIL: missing transcript root should exit non-zero" >&2 fail=1 else echo "ok: missing transcript root exits non-zero" fi EMPTY=$(python3 bin/offload-scan --plugin no-such-plugin \ --root tests/fixtures/transcripts --since 2026-06-01 --until 2026-08-01 --json) if [ "$(python3 -c "import json,sys; d=json.load(sys.stdin); print(d['coverage']['attributed_turns'], len(d['skills']))" <<<"$EMPTY")" = "0 0" ]; then echo "ok: a plugin with no history yields an empty, non-crashing report" else echo "FAIL: unknown plugin should yield an empty report" >&2 fail=1 fi exit "$fail"