Oleks
105e9f5bae
S1-S3 and S5-S8 pass; S4 is prepared but deliberately unfiled pending user sign-off (tasks.md 7.4 stays unchecked). Two bugs surfaced by running the composed pipeline against real data rather than synthetic fixtures, both in load-bearing places: audit-snapshot: cost_per_invocation folded to 0 when a candidate had zero invocations in a window, so a step that simply did not run reported -100% and direction() called it `shrunk`. That is the quiet-window artefact FR-5.4 exists to prevent, one level up from absolute tokens, and it contradicted direction()'s own docstring. It is now None, metric_delta marks the pair `unmeasured`, direction returns a new `unmeasured` status, and the pages render "not run". tests/calibration/mask-worktree-audit.py: recomputed script_loc from scripts alone, silently dropping all 1051 LOC of worktree-discipline's hooks. Masking one 272-LOC script appeared to remove 1323, taking script_coverage 0.353 -> 0.083 instead of 0.297, which let S3 pass partly on an artefact of the mask. The mask now matches build_aggregate. S3 still flags the known cut `high` / llm-over-script-digest on the corrected, harder input. Both fixes carry regression tests; suite is 156 assertions, exit 0.
134 lines
5.8 KiB
Bash
Executable File
134 lines
5.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/boundary-classify: rubric §6 confidence table + the FR-4.2 filing gate.
|
|
# Synthetic cases with hand-computable expectations, then the two calibration
|
|
# gates from tasks.md 4.4.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")"
|
|
|
|
BIN=../bin/boundary-classify
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
# field <json-file> <candidate-index> <key>
|
|
field() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['candidates'][int(sys.argv[2])][sys.argv[3]])" "$@"
|
|
}
|
|
summary() {
|
|
python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['summary'][sys.argv[2]])" "$@"
|
|
}
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
|
|
# A complete, high-confidence position-3 candidate with a measured 10% share.
|
|
mk() {
|
|
cat >"$tmp/in.json" <<EOF
|
|
{
|
|
"target": "synthetic",
|
|
"audited_spend_weighted_tokens": 1000000,
|
|
"candidates": [{
|
|
"candidate_id": "syn/skill/step",
|
|
"skill": "syn:skill",
|
|
"position": ${1:-\"llm-over-script-digest\"},
|
|
"determinism_tests": {"T1": true, "T2": ${2:-true}, "T3": true, "T4": ${3:-true}, "T5": true},
|
|
"falsifiability": {
|
|
"signature": "f(x: int) -> str",
|
|
"examples": ["1 -> a", "2 -> b", "EDGE: 0 -> z"],
|
|
"edge_case_index": 2,
|
|
"overrule_case": ${4:-\"a human would say no when the tree is live\"}
|
|
},
|
|
"escalation_path": ${5:-\"unclassifiable rows escalate\"},
|
|
"digest_schema": "one typed row per item",
|
|
"measurement": {"offload_value": ${6:-100000}, "invocations": 12, "attributed_turns": 200}
|
|
}]
|
|
}
|
|
EOF
|
|
$BIN "$tmp/in.json" >"$tmp/out.json"
|
|
}
|
|
|
|
echo "== complete position-3 candidate, measured 10% share -> high / file =="
|
|
mk
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "high"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "file"
|
|
check "share" "$(field "$tmp/out.json" 0 share_of_audited_spend)" "0.1"
|
|
check "strength" "$(field "$tmp/out.json" 0 measurement_strength)" "measured"
|
|
|
|
echo "== FR-4.2 hard gate: no overrule case -> boundary question, never filed =="
|
|
mk '"llm-over-script-digest"' true true null
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "boundary-question"
|
|
|
|
echo "== T4 fails -> low (rubric §6 bottom row) =="
|
|
mk '"llm-over-script-digest"' true false
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "low"
|
|
|
|
echo "== T2 fails -> low =="
|
|
mk '"llm-over-script-digest"' false true
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "low"
|
|
|
|
echo "== no escalation path -> medium (positioning question P3) =="
|
|
mk '"llm-over-script-digest"' true true '"human overrules on live trees"' null
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
|
|
echo "== position 4 -> no-offload, a finding rather than a question =="
|
|
mk '"pure-inference"'
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "no-offload"
|
|
|
|
echo "== measured but 0.5% of spend -> below the 2% filing threshold =="
|
|
mk '"llm-over-script-digest"' true true '"human overrules"' '"escalates"' 5000
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "high"
|
|
check "verdict" "$(field "$tmp/out.json" 0 verdict)" "boundary-question"
|
|
|
|
echo "== position 2 without a staleness plan -> medium =="
|
|
mk '"script-with-compiled-judgment"'
|
|
check "confidence" "$(field "$tmp/out.json" 0 boundary_confidence)" "medium"
|
|
|
|
echo "== CALIBRATION GATE 1 (tasks.md 4.4): token-budget -> zero high-confidence =="
|
|
$BIN calibration/token-budget.judgments.json >"$tmp/tb.json"
|
|
check "high_confidence" "$(summary "$tmp/tb.json" high_confidence)" "0"
|
|
check "to_file" "$(summary "$tmp/tb.json" to_file)" "0"
|
|
check "conclusion" "$(summary "$tmp/tb.json" conclusion)" "nothing to offload here"
|
|
|
|
echo "== CALIBRATION GATE 2 (tasks.md 4.4): masked worktree-discipline -> known cut is high =="
|
|
$BIN calibration/worktree-discipline-masked.judgments.json >"$tmp/wd.json"
|
|
check "high_confidence" "$(summary "$tmp/wd.json" high_confidence)" "1"
|
|
check "top candidate" "$(field "$tmp/wd.json" 0 candidate_id)" \
|
|
"worktree-discipline/sweep-worktrees/worktree-safety-classification"
|
|
check "position" "$(field "$tmp/wd.json" 0 position)" "llm-over-script-digest"
|
|
check "confidence" "$(field "$tmp/wd.json" 0 boundary_confidence)" "high"
|
|
check "drift is its own category" "$(summary "$tmp/wd.json" drift_notes)" "1"
|
|
|
|
echo "== the mask actually removes the script (S3 setup is real) =="
|
|
if [ ! -f ../../worktree-discipline/.claude-plugin/plugin.json ]; then
|
|
echo " skip (no sibling worktree-discipline checkout)"
|
|
exit "$fail"
|
|
fi
|
|
../bin/plugin-inventory ../../worktree-discipline --json |
|
|
python3 calibration/mask-worktree-audit.py >"$tmp/masked.json"
|
|
check "worktree-audit absent" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(any('worktree-audit' in s['file'] for s in d['scripts']))")" \
|
|
"False"
|
|
check "its command block is gone too" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(len(d['_mask']['removed_command_blocks']))")" \
|
|
"1"
|
|
# The mask must recompute script_loc the same way build_aggregate does, hooks
|
|
# included; recomputing from scripts alone deletes every hook and makes the
|
|
# masked plugin look far more script-starved than it is, which would let S3
|
|
# pass on an artefact of the mask instead of on the missing script.
|
|
check "script_loc still counts hooks" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(d['aggregate']['script_loc'] == sum(s['loc'] for s in d['scripts']) + sum(h['loc'] for h in d['hooks']))")" \
|
|
"True"
|
|
check "only worktree-audit's LOC was removed" \
|
|
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));m=d['_mask'];print(m['script_loc_before'] - m['script_loc_after'] < 500)")" \
|
|
"True"
|
|
|
|
exit "$fail"
|