Oleks
38e1adbd48
references/boundary-rubric.md is design/rubric.md verbatim (4.1).
skills/boundary-rubric is usable standalone on a single step before any code
exists — the rubric is more valuable applied before the fact than after (4.2).
agents/offload-analyst classifies pre-computed inventory/scan JSON, leads with
coverage, treats hook/prose drift as its own category, and enforces the hard
gate: no overrule case -> boundary question, never filed (4.3).
bin/boundary-classify applies rubric §6's confidence table and the FR-4.2 filing
gate. The determinism-test outcomes and the falsifiability triple are inference;
grading them is a lookup, so it is a script rather than agent prose (FR-8.1).
Calibration gate (4.4) passes on real runs:
- token-budget -> 0 high-confidence, "nothing to offload here"
- worktree-discipline with bin/worktree-audit masked out
-> worktree-safety classification at high,
position llm-over-script-digest
tests/classify.test.sh asserts both gates plus synthetic table cases; 67
assertions green across the suite.
124 lines
5.1 KiB
Bash
Executable File
124 lines
5.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/boundary-classify: rubric §6 confidence table + the FR-4.2 filing gate.
|
|
# Synthetic cases with hand-computable expectations, then the two calibration
|
|
# gates from tasks.md 4.4.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/boundary-classify
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
# field <json-file> <candidate-index> <key>
|
|
field() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['candidates'][int(sys.argv[2])][sys.argv[3]])" "$@"
|
|
}
|
|
summary() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['summary'][sys.argv[2]])" "$@"
|
|
}
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# A complete, high-confidence position-3 candidate with a measured 10% share.
|
|
mk() {
|
|
cat >"$tmp/in.json" <<EOF
|
|
{
|
|
"target": "synthetic",
|
|
"audited_spend_weighted_tokens": 1000000,
|
|
"candidates": [{
|
|
"candidate_id": "syn/skill/step",
|
|
"skill": "syn:skill",
|
|
"position": ${1:-\"llm-over-script-digest\"},
|
|
"determinism_tests": {"T1": true, "T2": ${2:-true}, "T3": true, "T4": ${3:-true}, "T5": true},
|
|
"falsifiability": {
|
|
"signature": "f(x: int) -> str",
|
|
"examples": ["1 -> a", "2 -> b", "EDGE: 0 -> z"],
|
|
"edge_case_index": 2,
|
|
"overrule_case": ${4:-\"a human would say no when the tree is live\"}
|
|
},
|
|
"escalation_path": ${5:-\"unclassifiable rows escalate\"},
|
|
"digest_schema": "one typed row per item",
|
|
"measurement": {"offload_value": ${6:-100000}, "invocations": 12, "attributed_turns": 200}
|
|
}]
|
|
}
|
|
EOF
|
|
$BIN "$tmp/in.json" >"$tmp/out.json"
|
|
}
|
|
|
|
echo "== complete position-3 candidate, measured 10% share -> high / file =="
|
|
mk
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "high"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "file"
|
|
check "share" "$(field "$tmp/out.json" 0 share_of_audited_spend)" "0.1"
|
|
check "strength" "$(field "$tmp/out.json" 0 measurement_strength)" "measured"
|
|
|
|
echo "== FR-4.2 hard gate: no overrule case -> boundary question, never filed =="
|
|
mk '"llm-over-script-digest"' true true null
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "boundary-question"
|
|
|
|
echo "== T4 fails -> low (rubric §6 bottom row) =="
|
|
mk '"llm-over-script-digest"' true false
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "low"
|
|
|
|
echo "== T2 fails -> low =="
|
|
mk '"llm-over-script-digest"' false true
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "low"
|
|
|
|
echo "== no escalation path -> medium (positioning question P3) =="
|
|
mk '"llm-over-script-digest"' true true '"human overrules on live trees"' null
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
|
|
echo "== position 4 -> no-offload, a finding rather than a question =="
|
|
mk '"pure-inference"'
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "no-offload"
|
|
|
|
echo "== measured but 0.5% of spend -> below the 2% filing threshold =="
|
|
mk '"llm-over-script-digest"' true true '"human overrules"' '"escalates"' 5000
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "high"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "boundary-question"
|
|
|
|
echo "== position 2 without a staleness plan -> medium =="
|
|
mk '"script-with-compiled-judgment"'
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
|
|
echo "== CALIBRATION GATE 1 (tasks.md 4.4): token-budget -> zero high-confidence =="
|
|
$BIN calibration/token-budget.judgments.json >"$tmp/tb.json"
|
|
check "high_confidence" "$(summary "$tmp/tb.json" high_confidence)" "0"
|
|
check "to_file" "$(summary "$tmp/tb.json" to_file)" "0"
|
|
check "conclusion" "$(summary "$tmp/tb.json" conclusion)" "nothing to offload here"
|
|
|
|
echo "== CALIBRATION GATE 2 (tasks.md 4.4): masked worktree-discipline -> known cut is high =="
|
|
$BIN calibration/worktree-discipline-masked.judgments.json >"$tmp/wd.json"
|
|
check "high_confidence" "$(summary "$tmp/wd.json" high_confidence)" "1"
|
|
check "top candidate" "$(field "$tmp/wd.json" 0 candidate_id)" \
|
|
"worktree-discipline/sweep-worktrees/worktree-safety-classification"
|
|
check "position" "$(field "$tmp/wd.json" 0 position)" "llm-over-script-digest"
|
|
check "confidence" "$(field "$tmp/wd.json" 0 boundary_confidence)" "high"
|
|
check "drift is its own category" "$(summary "$tmp/wd.json" drift_notes)" "1"
|
|
|
|
echo "== the mask actually removes the script (S3 setup is real) =="
|
|
if [ ! -f ../../worktree-discipline/.claude-plugin/plugin.json ]; then
|
|
echo " skip (no sibling worktree-discipline checkout)"
|
|
exit "$fail"
|
|
fi
|
|
../bin/plugin-inventory ../../worktree-discipline --json |
|
|
python3 calibration/mask-worktree-audit.py >"$tmp/masked.json"
|
|
check "worktree-audit absent" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(any('worktree-audit' in s['file'] for s in d['scripts']))")" \
|
|
"False"
|
|
check "its command block is gone too" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(len(d['_mask']['removed_command_blocks']))")" \
|
|
"1"
|
|
|
|
exit "$fail"
|