Oleks
832b07b248
kotkan/claude-plugin-inference-arbitrage#30 reported "FAIL repo = kotkan/worktree-discipline (want kotkan/claude-plugin-worktree-discipline)". Investigation found no stale reference in this repo's source or fixtures — the expected value already matches the live repo (verified: only kotkan/claude-plugin-worktree-discipline exists on Gitea; the pre-rename bare-name slug is a redirect only). The failure came entirely from the live git remote of the sibling ../worktree-discipline checkout on disk, which is outside this repo and outside version control, so it can drift independently (e.g. an old clone never repointed after the 2026-07-22 kotkan rename). That also meant the assertion silently never ran in CI, since the bare CI container has no sibling checkout and the test just printed "skip". Replace the dependency on that external checkout with a throwaway git repo created inside the test's own tmpdir, with its remote set explicitly to the current repo slug. Same code path (resolve_repo/parse_remote in bin/filing-plan) is exercised deterministically, and the assertion now always runs instead of skipping outside a workspace with that sibling checkout present. Verified the new test both passes against the correct remote and correctly reproduces the exact original failure signature when pointed at the stale pre-rename slug.
229 lines
11 KiB
Bash
Executable File
229 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# bin/filing-plan: output path A — the marker-based idempotency contract
|
|
# (FR-6.3), template rendering (FR-6.1), and repo-less degradation (FR-6.4).
|
|
#
|
|
# The load-bearing case is "run 1 files, run 2 comments". It is proved here the
|
|
# only way that means anything: run 1's ACTUAL rendered body is fed back as the
|
|
# tracker's state for run 2. Nothing is asserted about the marker by hand.
|
|
#
|
|
# tests/fixtures/gitea-issues.json is copied verbatim from real Gitea API
|
|
# responses (probe issue kotkan/claude-plugin-inference-arbitrage#9, since
|
|
# closed), so the shape this parses is the shape the API returns.
|
|
set -euo pipefail
|
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
|
|
|
BIN=bin/filing-plan
|
|
JUD=tests/fixtures/filing-judgments.json
|
|
tmp=$(mktemp -d)
|
|
trap 'rm -rf "$tmp"' EXIT
|
|
fail=0
|
|
|
|
check() {
|
|
local what=$1 got=$2 want=$3
|
|
if [ "$got" = "$want" ]; then
|
|
echo " ok $what = $want"
|
|
else
|
|
echo " FAIL $what = $got (want $want)" >&2
|
|
fail=1
|
|
fi
|
|
}
|
|
# q <file> <python-expr over `d`>
|
|
q() { python3 -c "import json,sys;d=json.load(open(sys.argv[1]));print($2)" "$1"; }
|
|
|
|
bin/boundary-classify "$JUD" >"$tmp/cls.json"
|
|
|
|
plan() { # plan <existing.json> <out.json> [extra args...]
|
|
local existing=$1 out=$2
|
|
shift 2
|
|
$BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
|
--target-repo oleks/claude-plugin-fixture --existing "$existing" \
|
|
--audit-page Audits/fixture-plugin/2026-07-29 \
|
|
--window "2026-07-22 -> 2026-07-29" --run-date 2026-07-29 "$@" >"$out"
|
|
}
|
|
|
|
echo "== the fixture's two candidates grade as expected (guards the rest) =="
|
|
check "A verdict" "$(q "$tmp/cls.json" "d['candidates'][0]['verdict']")" "file"
|
|
check "B verdict" "$(q "$tmp/cls.json" "d['candidates'][1]['verdict']")" "no-offload"
|
|
|
|
echo "== RUN 1: empty tracker -> files, with the marker as line 1 =="
|
|
echo '[]' >"$tmp/empty.json"
|
|
plan "$tmp/empty.json" "$tmp/plan1.json"
|
|
check "to_file" "$(q "$tmp/plan1.json" "d['summary']['to_file']")" "1"
|
|
check "to_comment" "$(q "$tmp/plan1.json" "d['summary']['to_comment']")" "0"
|
|
check "action" "$(q "$tmp/plan1.json" "d['actions'][0]['action']")" "file"
|
|
check "marker is line 1" \
|
|
"$(q "$tmp/plan1.json" "d['actions'][0]['body'].splitlines()[0]")" \
|
|
"<!-- ia-candidate: fixture-plugin/skill-a/label-derivation -->"
|
|
check "token-offload label required" \
|
|
"$(q "$tmp/plan1.json" "d['actions'][0]['required_labels']")" "['token-offload']"
|
|
check "non-file verdict skipped" "$(q "$tmp/plan1.json" "d['actions'][1]['action']")" "skip"
|
|
|
|
echo "== the body carries every element of the FR-6 contract =="
|
|
body=$(q "$tmp/plan1.json" "d['actions'][0]['body']")
|
|
for needle in \
|
|
"**Measured cost.**" "**Evidence strength.**" "**Boundary position.**" \
|
|
"Determinism tests: T1 ✓ T2 ✓ T3 ✓ T4 ✓ T5 ✓" \
|
|
"**Proposed signature.**" "derive_labels(" "**Examples.**" \
|
|
"**When a human would overrule this.**" "**Escalation path.**" \
|
|
"**What crosses the boundary.**" \
|
|
"Audits/fixture-plugin/2026-07-29" "Rubric v1.0.0"; do
|
|
if grep -qF -- "$needle" <<<"$body"; then
|
|
echo " ok body contains: $needle"
|
|
else
|
|
echo " FAIL body missing: $needle" >&2
|
|
fail=1
|
|
fi
|
|
done
|
|
check "measurement is quoted, not invented" \
|
|
"$(grep -cF 'Recurring sequence observed 17x' <<<"$body")" "1"
|
|
check "3 examples incl. the edge case" "$(grep -c '^- ' <<<"$body")" "3"
|
|
|
|
echo "== RUN 2 (FR-6.3, S5): run 1's own body is now on the tracker -> comment =="
|
|
python3 - "$tmp/plan1.json" "$tmp/tracker.json" <<'PY'
|
|
import json, sys
|
|
plan = json.load(open(sys.argv[1]))
|
|
filed = next(a for a in plan["actions"] if a["action"] == "file")
|
|
json.dump([{"number": 41, "state": "open", "title": filed["proposed_title"],
|
|
"body": filed["body"], "comments": []}], open(sys.argv[2], "w"))
|
|
PY
|
|
plan "$tmp/tracker.json" "$tmp/plan2.json" --status grown
|
|
check "files nothing on re-run" "$(q "$tmp/plan2.json" "d['summary']['to_file']")" "0"
|
|
check "comments instead" "$(q "$tmp/plan2.json" "d['summary']['to_comment']")" "1"
|
|
check "on the right issue" \
|
|
"$(q "$tmp/plan2.json" "d['actions'][0]['issue']")" "oleks/claude-plugin-fixture#41"
|
|
check "comment re-carries the marker" \
|
|
"$(q "$tmp/plan2.json" "d['actions'][0]['body'].splitlines()[0]")" \
|
|
"<!-- ia-candidate: fixture-plugin/skill-a/label-derivation -->"
|
|
check "comment reports the diff status" \
|
|
"$(q "$tmp/plan2.json" "'Status: grown' in d['actions'][0]['body']")" "True"
|
|
|
|
echo "== RUN 3: the comment from run 2 is the only marker left (body rewritten) =="
|
|
python3 - "$tmp/plan2.json" "$tmp/tracker3.json" <<'PY'
|
|
import json, sys
|
|
plan = json.load(open(sys.argv[1]))
|
|
c = next(a for a in plan["actions"] if a["action"] == "comment")
|
|
json.dump([{"number": 41, "state": "closed", "title": "a human retitled this",
|
|
"body": "and rewrote the body, dropping the marker",
|
|
"comments": [{"body": c["body"]}]}], open(sys.argv[2], "w"))
|
|
PY
|
|
plan "$tmp/tracker3.json" "$tmp/plan3.json"
|
|
check "still no duplicate" "$(q "$tmp/plan3.json" "d['summary']['to_file']")" "0"
|
|
check "found via the comment" "$(q "$tmp/plan3.json" "d['actions'][0]['action']")" "comment"
|
|
check "closed state surfaced for issuer" \
|
|
"$(q "$tmp/plan3.json" "d['actions'][0]['issue_state']")" "closed"
|
|
|
|
echo "== kotkan/claude-plugin-inference-arbitrage#23: a marker that lost its"
|
|
echo " HTML-comment wrapper on write (oleks/claude-plugin-cluster#56, #57)"
|
|
echo " is still found — backward compat, not a second accepted format =="
|
|
python3 - "$tmp/plan1.json" "$tmp/bare.json" <<'PY'
|
|
import json, sys
|
|
plan = json.load(open(sys.argv[1]))
|
|
filed = next(a for a in plan["actions"] if a["action"] == "file")
|
|
# Same body, but the first line has lost its "<!-- " / " -->" wrapper — the
|
|
# exact shape observed on the live tracker.
|
|
bare_body = filed["body"].replace(
|
|
"<!-- ia-candidate: fixture-plugin/skill-a/label-derivation -->",
|
|
"ia-candidate: fixture-plugin/skill-a/label-derivation",
|
|
)
|
|
json.dump([{"number": 56, "state": "closed", "title": filed["proposed_title"],
|
|
"body": bare_body, "comments": []}], open(sys.argv[2], "w"))
|
|
PY
|
|
plan "$tmp/bare.json" "$tmp/plan-bare.json"
|
|
check "bare marker still found, no duplicate filed" \
|
|
"$(q "$tmp/plan-bare.json" "d['summary']['to_file']")" "0"
|
|
check "recognized as a comment on the existing issue" \
|
|
"$(q "$tmp/plan-bare.json" "d['actions'][0]['action']")" "comment"
|
|
check "on issue 56, matching the observed failure" \
|
|
"$(q "$tmp/plan-bare.json" "d['actions'][0]['issue_number']")" "56"
|
|
|
|
echo "== a bare marker for a DIFFERENT id still does not false-match =="
|
|
python3 - "$tmp/bare.json" "$tmp/bare-other.json" <<'PY'
|
|
import json, sys
|
|
issues = json.load(open(sys.argv[1]))
|
|
issues[0]["body"] = issues[0]["body"].replace(
|
|
"fixture-plugin/skill-a/label-derivation", "fixture-plugin/skill-a/label-derivation-longer",
|
|
)
|
|
json.dump(issues, open(sys.argv[2], "w"))
|
|
PY
|
|
plan "$tmp/bare-other.json" "$tmp/plan-bare-other.json"
|
|
check "different id (even as a prefix) still files fresh" \
|
|
"$(q "$tmp/plan-bare-other.json" "d['summary']['to_file']")" "1"
|
|
|
|
echo "== real Gitea API shapes parse, and a DIFFERENT id does not false-match =="
|
|
plan tests/fixtures/gitea-issues.json "$tmp/plan4.json"
|
|
check "unrelated markers -> files" "$(q "$tmp/plan4.json" "d['summary']['to_file']")" "1"
|
|
check "searched the fixture" "$(q "$tmp/plan4.json" "d['summary']['searched_issues']")" "1"
|
|
|
|
echo "== a marker on two issues warns and takes the lowest number =="
|
|
python3 - "$tmp/plan1.json" "$tmp/dupes.json" <<'PY'
|
|
import json, sys
|
|
filed = next(a for a in json.load(open(sys.argv[1]))["actions"] if a["action"] == "file")
|
|
json.dump([{"number": 77, "state": "open", "title": "later dupe", "body": filed["body"], "comments": []},
|
|
{"number": 41, "state": "open", "title": "original", "body": filed["body"], "comments": []}],
|
|
open(sys.argv[2], "w"))
|
|
PY
|
|
plan "$tmp/dupes.json" "$tmp/plan5.json"
|
|
check "lowest issue wins" "$(q "$tmp/plan5.json" "d['actions'][0]['issue_number']")" "41"
|
|
check "warns a human" "$(q "$tmp/plan5.json" "len(d['warnings'])")" "1"
|
|
|
|
echo "== --existing is mandatory: a skipped search must never mean 'file it all' =="
|
|
if $BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
|
--target-repo oleks/claude-plugin-fixture --audit-page A --window w \
|
|
--run-date 2026-07-29 >/dev/null 2>&1; then
|
|
echo " FAIL plan ran without --existing" >&2
|
|
fail=1
|
|
else
|
|
echo " ok refuses to plan without a search result"
|
|
fi
|
|
|
|
echo "== FR-6.4: a target with no writable repo degrades to summary-only =="
|
|
# Real, not synthetic: third-party/baoyu-design is a vendored plugin whose only
|
|
# remote is github.com, i.e. nothing this environment can file to.
|
|
THIRD=../third-party/baoyu-design
|
|
if [ -d "$THIRD" ]; then
|
|
bin/plugin-inventory "$THIRD" --json >"$tmp/tp-inv.json"
|
|
check "inventory still resolves it" "$(q "$tmp/tp-inv.json" "d['plugin']['name']")" "baoyu-design"
|
|
$BIN queries --classified "$tmp/cls.json" --judgments "$JUD" \
|
|
--target-path "$THIRD" --workspace /nonexistent >"$tmp/tp-q.json"
|
|
check "filing" "$(q "$tmp/tp-q.json" "d['filing']")" "unavailable"
|
|
check "reason names the remote" \
|
|
"$(q "$tmp/tp-q.json" "'github.com' in d['filing_unavailable_reason']")" "True"
|
|
check "no queries emitted" "$(q "$tmp/tp-q.json" "'queries' in d")" "False"
|
|
|
|
$BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
|
--target-path "$THIRD" --workspace /nonexistent --existing "$tmp/empty.json" \
|
|
--audit-page A --window w --run-date 2026-07-29 >"$tmp/tp-plan.json"
|
|
check "nothing filed" "$(q "$tmp/tp-plan.json" "d['summary']['to_file']")" "0"
|
|
check "candidate marked unavailable" \
|
|
"$(q "$tmp/tp-plan.json" "d['actions'][0]['filing']")" "unavailable"
|
|
else
|
|
echo " skip (no third-party/baoyu-design checkout)"
|
|
fi
|
|
|
|
echo "== a writable target resolves through its own checkout's remote =="
|
|
# Was: read the live remote of a sibling ../worktree-discipline checkout on
|
|
# disk. That checkout is outside this repo and outside version control, so
|
|
# its remote can drift independently of anything here — e.g. a stale clone
|
|
# still pointing at the pre-rename slug `kotkan/worktree-discipline` instead
|
|
# of `kotkan/claude-plugin-worktree-discipline` (kotkan org repos were
|
|
# renamed to the claude-plugin-<name> convention on 2026-07-22). That drift
|
|
# produced a false "repo" mismatch (kotkan/claude-plugin-inference-arbitrage#30)
|
|
# with nothing to fix in this repo's source or fixtures — both already agreed
|
|
# with the current live repo name. It also meant this assertion silently
|
|
# never ran in CI, since the bare CI container has no sibling checkout at all
|
|
# and the test just printed "skip". A synthetic git repo with a fixed remote
|
|
# tests the same resolve_repo()/parse_remote() code path deterministically,
|
|
# with no dependency on what else happens to be checked out next door.
|
|
wd_git="$tmp/wd-fixture"
|
|
mkdir -p "$wd_git"
|
|
git -C "$wd_git" init -q
|
|
git -C "$wd_git" remote add origin git@git.oleks.space:kotkan/claude-plugin-worktree-discipline.git
|
|
$BIN queries --classified "$tmp/cls.json" --judgments "$JUD" \
|
|
--target-path "$wd_git" >"$tmp/wd-q.json"
|
|
check "repo" "$(q "$tmp/wd-q.json" "d['repo']")" "kotkan/claude-plugin-worktree-discipline"
|
|
check "filing" "$(q "$tmp/wd-q.json" "d['filing']")" "available"
|
|
check "label scan is the authoritative query" \
|
|
"$(q "$tmp/wd-q.json" "d['queries']['label_scan']['labels']")" "['token-offload']"
|
|
|
|
exit "$fail"
|