claude-plugin-inference-arbitrage/tests/candidate-digest.test.sh

#!/usr/bin/env bash
# bin/candidate-digest: the inv.json/scan.json join
# (kotkan/claude-plugin-inference-arbitrage#24). Synthetic fixtures for the
# three worked examples from the issue body, plus the has_bin_script suppression.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"

BIN=../bin/candidate-digest
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
fail=0

field() {
	python3 -c "import json,sys;print(json.load(open(sys.argv[1]))[int(sys.argv[2])][sys.argv[3]])" "$@"
}

check() {
	local what=$1 got=$2 want=$3
	if [ "$got" = "$want" ]; then
		echo "  ok   $what = $want"
	else
		echo "  FAIL $what = $got (want $want)" >&2
		fail=1
	fi
}

# ---------------------------------------------------------------------------
# Example 1: verb_ratio=0.71 skill backed by a bin/ script -> has_bin_script
# True, flags=[] regardless of its dynamic numbers (conjunction fails).
# Example 2: agent row, mtr=0.17 share=0.40 judgment_density=0.034 with a
# dominant recurring 8-gram -> flags=['high-share-low-judgment-recurring-ngram'].
# Example 3 (EDGE): same mtr/share/ngram but judgment_density=0.55 -> flags=[]
# (the judgment_density brake suppresses the false positive).
# ---------------------------------------------------------------------------

cat >"$tmp/inv.json" <<'EOF'
{
  "plugin": {"name": "synthetic"},
  "skills": [
    {
      "name": "syn:scripted-skill",
      "verb_ratio": 0.71,
      "command_blocks": [{"sig": "${CLAUDE_PLUGIN_ROOT}/bin/do-the-thing --json", "count": 1}]
    }
  ]
}
EOF

cat >"$tmp/scan.json" <<'EOF'
{
  "skills": [
    {
      "skill": "syn:scripted-skill",
      "invocations": 12,
      "turns": 90,
      "share_of_plugin": 0.5,
      "mtr": 0.8,
      "judgment_density": 0.05,
      "read_amplification": 10.0,
      "retry_density": 0.0,
      "fanout_multiplier": 1.0,
      "ngrams": [{"sig": ["a", "b", "c"], "recurrences": 9, "invocations": 5, "waste": 1000}]
    }
  ],
  "agents": [
    {
      "agent": "syn:flagged-agent",
      "invocations": 26,
      "turns": 208,
      "share_of_plugin": 0.40,
      "mtr": 0.17,
      "judgment_density": 0.034,
      "read_amplification": 4.0,
      "retry_density": 0.02,
      "fanout_multiplier": 1.0,
      "ngrams": [{"sig": ["x", "y"], "recurrences": 8, "invocations": 6, "waste": 800000}]
    },
    {
      "agent": "syn:healthy-agent",
      "invocations": 26,
      "turns": 208,
      "share_of_plugin": 0.40,
      "mtr": 0.17,
      "judgment_density": 0.55,
      "read_amplification": 4.0,
      "retry_density": 0.02,
      "fanout_multiplier": 1.0,
      "ngrams": [{"sig": ["x", "y"], "recurrences": 8, "invocations": 6, "waste": 800000}]
    }
  ]
}
EOF

$BIN "$tmp/inv.json" "$tmp/scan.json" >"$tmp/out.json"

echo "== example 1: scripted skill, high mtr/share, has_bin_script suppresses everything =="
row=$(python3 -c "
import json
rows = json.load(open('$tmp/out.json'))
print([i for i, r in enumerate(rows) if r['name'] == 'syn:scripted-skill'][0])
")
check "row_kind" "$(field "$tmp/out.json" "$row" row_kind)" "skill"
check "has_bin_script" "$(field "$tmp/out.json" "$row" has_bin_script)" "True"
check "flags" "$(field "$tmp/out.json" "$row" flags)" "[]"

echo "== example 2: agent, judgment_density=0.034 -> flagged =="
row=$(python3 -c "
import json
rows = json.load(open('$tmp/out.json'))
print([i for i, r in enumerate(rows) if r['name'] == 'syn:flagged-agent'][0])
")
check "row_kind" "$(field "$tmp/out.json" "$row" row_kind)" "agent"
check "has_bin_script" "$(field "$tmp/out.json" "$row" has_bin_script)" "False"
check "dominant_ngram_recurrences" "$(field "$tmp/out.json" "$row" dominant_ngram_recurrences)" "8"
check "flags" "$(field "$tmp/out.json" "$row" flags)" "['high-share-low-judgment-recurring-ngram']"

echo "== example 3 (EDGE): same mtr/share/ngram, judgment_density=0.55 -> brake suppresses =="
row=$(python3 -c "
import json
rows = json.load(open('$tmp/out.json'))
print([i for i, r in enumerate(rows) if r['name'] == 'syn:healthy-agent'][0])
")
check "flags" "$(field "$tmp/out.json" "$row" flags)" "[]"

echo "== below MIN_SHARE_OF_SPEND -> never flagged even with a low judgment_density =="
cat >"$tmp/scan-thin.json" <<'EOF'
{
  "skills": [],
  "agents": [
    {
      "agent": "syn:thin-agent",
      "invocations": 12,
      "turns": 90,
      "share_of_plugin": 0.005,
      "mtr": 0.9,
      "judgment_density": 0.01,
      "read_amplification": 3.0,
      "retry_density": 0.0,
      "fanout_multiplier": 1.0,
      "ngrams": [{"sig": ["p", "q"], "recurrences": 5, "invocations": 4, "waste": 100}]
    }
  ]
}
EOF
cat >"$tmp/inv-empty.json" <<'EOF'
{"plugin": {"name": "synthetic"}, "skills": []}
EOF
$BIN "$tmp/inv-empty.json" "$tmp/scan-thin.json" >"$tmp/out-thin.json"
check "flags" "$(field "$tmp/out-thin.json" 0 flags)" "[]"

echo "== schema: every CandidateRow key present =="
python3 -c "
import json
rows = json.load(open('$tmp/out.json'))
want = {'row_kind', 'name', 'share_of_spend', 'mtr', 'judgment_density',
        'read_amplification', 'retry_density', 'fanout_multiplier',
        'dominant_ngram_recurrences', 'has_bin_script', 'flags'}
for r in rows:
    assert set(r.keys()) == want, (r.keys(), want)
print('ok')
"

exit "$fail"