Fix markdown lint in the Phase 5 docs
This commit is contained in:
Oleks
@@ -184,9 +184,23 @@ what you looked at, and the conclusion that the boundary is already in the right
|
||||
place — with the `pure-inference` findings named, because they are the evidence
|
||||
that you looked rather than shrugged.
|
||||
|
||||
### 7. Filing (Phase 6 — not yet built)
|
||||
### 7. Filing, and the run summary
|
||||
|
||||
Filing goes `offload-analyst → Agent(anxious:issuer-agent) → cluster:gitea-agent`.
|
||||
This plugin holds no Gitea credentials and calls no Gitea tool directly. Until
|
||||
Phase 6 lands, **report candidates to the user and file nothing** — including
|
||||
candidates that `boundary-classify` returns with `"verdict": "file"`.
|
||||
Both output paths are procedure, not judgment, and the `offload-audit` skill
|
||||
carries them step by step. Read it rather than improvising: `skills/offload-audit/SKILL.md`.
|
||||
|
||||
The four things that are yours to hold:
|
||||
|
||||
- **The chain is fixed.** `offload-analyst → Agent(anxious:issuer-agent) →
|
||||
cluster:gitea-agent → Gitea`. You hold no Gitea credentials and call no Gitea
|
||||
tool directly. `issuer` decides repo, title, labels and milestone; you own the
|
||||
issue body.
|
||||
- **Filing is implicit** for a candidate that cleared the gate — do not ask
|
||||
permission. Equally, never file one that did not, and never re-grade a verdict
|
||||
`boundary-classify` already returned.
|
||||
- **Never file without the marker search.** `bin/filing-plan` decides
|
||||
file-vs-comment from the search results and renders both bodies; execute its
|
||||
plan verbatim. A re-run that duplicates issues is the worst failure this
|
||||
plugin has.
|
||||
- **Path B always runs**, including when filing was unavailable (FR-6.4) and
|
||||
when the answer was "nothing to offload here".
|
||||
|
||||
Executable
+450
@@ -0,0 +1,450 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Decides, deterministically, what output path A does with each candidate:
|
||||
open a new issue, comment on the one that already exists, or skip it — and
|
||||
renders the exact body text for each (FR-6.1..FR-6.4, design/plan.md §10).
|
||||
|
||||
Everything here is a `pure-script` step by the plugin's own rubric: the verdicts
|
||||
were already graded by bin/boundary-classify, the marker match is a literal
|
||||
string comparison, and rendering a template is a total function of its inputs.
|
||||
None of it may be done by inference — an audit that decides duplicates by
|
||||
"looking at" the tracker is exactly the shape this plugin reports on, and
|
||||
getting it wrong spams a real human's tracker.
|
||||
|
||||
This script never touches Gitea. It consumes the results of a search that
|
||||
`anxious:issuer-agent` -> `cluster:gitea-agent` performed, and emits a plan that
|
||||
the same chain executes. Two subcommands, in order:
|
||||
|
||||
filing-plan queries --classified c.json --judgments j.json [--target-path P]
|
||||
filing-plan plan --classified c.json --judgments j.json --existing e.json ...
|
||||
|
||||
`queries` resolves the target repo (or reports that filing is unavailable) and
|
||||
emits the marker searches to run. `plan` consumes their merged results.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
PLUGIN_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DEFAULT_TEMPLATE = os.path.join(PLUGIN_ROOT, "references", "issue-template.md")
|
||||
DEFAULT_WORKSPACE = os.path.expanduser("~/projects/claude-plugins")
|
||||
DEFAULT_GITEA_HOST = "git.oleks.space"
|
||||
DEFAULT_WIKI_REPO = "kotkan/claude-plugin-inference-arbitrage"
|
||||
OFFLOAD_LABEL = "token-offload"
|
||||
|
||||
TEMPLATE_SECTIONS = {"issue-title", "issue-body", "update-comment"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# template loading and rendering
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def load_templates(path):
|
||||
"""Same contract as signals-catalog.md: one fenced block per recognized
|
||||
`## <section>` heading, everything else prose."""
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
text = fh.read()
|
||||
|
||||
out = {}
|
||||
for m in re.finditer(r"^##\s+([a-z-]+)\s*$", text, re.MULTILINE):
|
||||
name = m.group(1)
|
||||
if name not in TEMPLATE_SECTIONS:
|
||||
continue
|
||||
block = re.search(r"```\S*\n(.*?)```", text[m.end():], re.DOTALL)
|
||||
if block:
|
||||
out[name] = block.group(1).strip("\n")
|
||||
missing = TEMPLATE_SECTIONS - out.keys()
|
||||
if missing:
|
||||
sys.exit(f"error: {path} is missing section(s): {', '.join(sorted(missing))}")
|
||||
return out
|
||||
|
||||
|
||||
PLACEHOLDER = re.compile(r"\{\{([a-z_]+)\}\}")
|
||||
|
||||
|
||||
def render(template, values):
|
||||
"""Strict substitution. An unsupplied placeholder is an error, never an
|
||||
empty string — a half-filled issue body is worse than a crashed audit."""
|
||||
missing = sorted(
|
||||
{k for k in PLACEHOLDER.findall(template) if values.get(k) in (None, "")}
|
||||
)
|
||||
if missing:
|
||||
sys.exit(f"error: template placeholder(s) unsupplied: {', '.join(missing)}")
|
||||
return PLACEHOLDER.sub(lambda m: str(values[m.group(1)]), template)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# the marker (FR-6.3)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def marker(candidate_id):
|
||||
return f"<!-- ia-candidate: {candidate_id} -->"
|
||||
|
||||
|
||||
def marker_re(candidate_id):
|
||||
"""Tolerates whitespace drift inside the comment, nothing else. The id
|
||||
itself must match exactly — a fuzzy id match would let two genuinely
|
||||
different candidates collapse onto one issue."""
|
||||
return re.compile(r"<!--\s*ia-candidate:\s*" + re.escape(candidate_id) + r"\s*-->")
|
||||
|
||||
|
||||
def find_existing(candidate_id, issues):
|
||||
"""Returns (issue_or_None, duplicates). Matches the body and every comment,
|
||||
so an issue keeps its identity even if a human rewrites the body."""
|
||||
pat = marker_re(candidate_id)
|
||||
hits = []
|
||||
for issue in issues:
|
||||
haystacks = [issue.get("body") or ""]
|
||||
haystacks += [c if isinstance(c, str) else (c.get("body") or "")
|
||||
for c in issue.get("comments") or []]
|
||||
if any(pat.search(h) for h in haystacks):
|
||||
hits.append(issue)
|
||||
if not hits:
|
||||
return None, []
|
||||
hits.sort(key=lambda i: i.get("number", 0))
|
||||
return hits[0], hits[1:]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# target repo resolution (FR-6.4)
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def git_remote(path):
|
||||
if not os.path.isdir(os.path.join(path, ".git")):
|
||||
return None
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", "-C", path, "remote", "get-url", "origin"],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
return out.stdout.strip() if out.returncode == 0 else None
|
||||
|
||||
|
||||
def parse_remote(url, host):
|
||||
"""owner/repo if the remote points at our Gitea, else None."""
|
||||
m = re.match(r"^(?:git@|ssh://git@|https://)([^/:]+)[:/](.+?)(?:\.git)?/?$", url)
|
||||
if not m or m.group(1) != host:
|
||||
return None
|
||||
parts = m.group(2).split("/")
|
||||
return "/".join(parts[-2:]) if len(parts) >= 2 else None
|
||||
|
||||
|
||||
def resolve_repo(args, target_name):
|
||||
"""(repo, filing, reason). `filing` is 'available' or 'unavailable'.
|
||||
|
||||
Order: explicit flag, the target checkout's own remote, then the workspace
|
||||
sibling checkout — because an audit normally runs against
|
||||
~/.claude/plugins/cache/... which carries no .git at all.
|
||||
"""
|
||||
if args.target_repo:
|
||||
return args.target_repo, "available", None
|
||||
|
||||
tried = []
|
||||
candidates = []
|
||||
if args.target_path:
|
||||
candidates.append(os.path.abspath(args.target_path))
|
||||
if target_name and args.workspace:
|
||||
sibling = os.path.join(args.workspace, target_name)
|
||||
if sibling not in candidates:
|
||||
candidates.append(sibling)
|
||||
|
||||
for path in candidates:
|
||||
if not os.path.isdir(path):
|
||||
continue
|
||||
url = git_remote(path)
|
||||
if url is None:
|
||||
tried.append(f"{path}: no git remote")
|
||||
continue
|
||||
repo = parse_remote(url, args.gitea_host)
|
||||
if repo:
|
||||
return repo, "available", None
|
||||
tried.append(f"{path}: remote {url} is not on {args.gitea_host}")
|
||||
|
||||
if not candidates:
|
||||
detail = "no --target-repo, --target-path, or workspace checkout to resolve from"
|
||||
else:
|
||||
detail = "; ".join(tried) or "no checkout found at " + ", ".join(candidates)
|
||||
return None, "unavailable", (
|
||||
f"no writable repo for target {target_name!r} ({detail}). "
|
||||
"Path A is skipped; the candidate is reported in the run summary only (FR-6.4)."
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# rendering values
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
TICKS = {True: "✓", False: "✗"}
|
||||
|
||||
|
||||
def fmt_tests(tests):
|
||||
return " ".join(f"{t} {TICKS[tests.get(t) is True]}" for t in ("T1", "T2", "T3", "T4", "T5"))
|
||||
|
||||
|
||||
def fmt_tokens(n):
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M"
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.1f}k"
|
||||
return str(n)
|
||||
|
||||
|
||||
def fmt_examples(examples):
|
||||
return "\n".join(f"- {e}" for e in examples)
|
||||
|
||||
|
||||
def fmt_evidence(strength, m):
|
||||
inv, turns = m.get("invocations", 0), m.get("attributed_turns", 0)
|
||||
if strength == "measured":
|
||||
return f"`measured` — {inv} invocations, {turns} attributed turns."
|
||||
if strength == "thin":
|
||||
return (
|
||||
f"`thin` — only {inv} invocation(s) and {turns} attributed turns in this "
|
||||
"window. **The cost figure above is arithmetic, not evidence.** The "
|
||||
"correctness case stands on the rubric; the value claim is unproven."
|
||||
)
|
||||
return (
|
||||
"`unmeasured` — no transcript history for this target in the window. This "
|
||||
"candidate rests on the plugin definition alone (FR-2.4)."
|
||||
)
|
||||
|
||||
|
||||
def fmt_cost(cand, cls, window):
|
||||
"""Only states what was actually measured. Absent signals are omitted, never
|
||||
guessed at — a fabricated MTR in a filed issue is unrecoverable."""
|
||||
m = cand.get("measurement") or {}
|
||||
bits = []
|
||||
value = cls.get("offload_value") or 0
|
||||
share = cls.get("share_of_audited_spend") or 0
|
||||
inv = m.get("invocations", 0)
|
||||
head = f"{fmt_tokens(value)} weighted tokens"
|
||||
if inv:
|
||||
head += f" over {inv} invocation{'s' if inv != 1 else ''}"
|
||||
if window:
|
||||
head += f" ({window})"
|
||||
if share:
|
||||
head += f", {share:.0%} of this plugin's audited spend"
|
||||
bits.append(head + ".")
|
||||
|
||||
stats = []
|
||||
if m.get("mtr") is not None:
|
||||
stats.append(f"MTR {m['mtr']:.2f}")
|
||||
if m.get("judgment_density") is not None:
|
||||
stats.append(f"judgment density {m['judgment_density']:.2f}")
|
||||
if stats:
|
||||
bits.append(" ".join(stats) + ".")
|
||||
|
||||
ngram = m.get("ngram") or {}
|
||||
if ngram.get("sig") and ngram.get("recurrences"):
|
||||
seq = " -> ".join(ngram["sig"])
|
||||
bits.append(f"Recurring sequence observed {ngram['recurrences']}x: {seq}.")
|
||||
return " ".join(bits)
|
||||
|
||||
|
||||
def headline(cand):
|
||||
if cand.get("headline"):
|
||||
return cand["headline"]
|
||||
# Last path segment of the candidate id is its slug by construction
|
||||
# (design/plan.md §6: "<plugin>/<skill>/<slug>").
|
||||
return cand["candidate_id"].rsplit("/", 1)[-1].replace("-", " ")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# the plan
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
def index_judgments(doc):
|
||||
return {c["candidate_id"]: c for c in doc.get("candidates", [])}
|
||||
|
||||
|
||||
def build(args, cls_doc, jud_doc, existing, templates):
|
||||
target = cls_doc.get("target") or jud_doc.get("target")
|
||||
repo, filing, reason = resolve_repo(args, target)
|
||||
judgments = index_judgments(jud_doc)
|
||||
|
||||
actions, warnings = [], []
|
||||
for cls in cls_doc.get("candidates", []):
|
||||
cid = cls["candidate_id"]
|
||||
verdict = cls.get("verdict")
|
||||
if verdict != "file":
|
||||
actions.append({
|
||||
"candidate_id": cid,
|
||||
"action": "skip",
|
||||
"reason": f"verdict {verdict!r} — path A files only 'file' verdicts "
|
||||
"(FR-4.2 gate, enforced by boundary-classify)",
|
||||
})
|
||||
continue
|
||||
|
||||
if filing == "unavailable":
|
||||
actions.append({
|
||||
"candidate_id": cid,
|
||||
"action": "skip",
|
||||
"filing": "unavailable",
|
||||
"reason": reason,
|
||||
})
|
||||
continue
|
||||
|
||||
cand = judgments.get(cid)
|
||||
if cand is None:
|
||||
sys.exit(f"error: candidate {cid!r} has verdict 'file' but no judgments entry")
|
||||
|
||||
f = cand.get("falsifiability") or {}
|
||||
values = {
|
||||
"candidate_id": cid,
|
||||
"headline": headline(cand),
|
||||
"skill": cand.get("skill") or cls.get("skill"),
|
||||
"measured_cost": fmt_cost(cand, cls, args.window),
|
||||
"evidence": fmt_evidence(cls.get("measurement_strength"), cand.get("measurement") or {}),
|
||||
"position": cls.get("position"),
|
||||
"boundary_confidence": cls.get("boundary_confidence"),
|
||||
"determinism_tests": fmt_tests(cls.get("determinism_tests") or {}),
|
||||
"signature": f.get("signature"),
|
||||
"examples": fmt_examples(f.get("examples") or []),
|
||||
"overrule_case": f.get("overrule_case"),
|
||||
"escalation_path": cand.get("escalation_path"),
|
||||
"digest_schema": cand.get("digest_schema") or "n/a — position 1, the whole step moves.",
|
||||
"audit_page": args.audit_page,
|
||||
"wiki_repo": args.wiki_repo,
|
||||
"rubric_version": cls_doc.get("rubric_version", "1.0.0"),
|
||||
"run_date": args.run_date,
|
||||
"window": args.window,
|
||||
"status": args.status,
|
||||
}
|
||||
|
||||
found, dupes = find_existing(cid, existing)
|
||||
if dupes:
|
||||
numbers = ", ".join("#%s" % d.get("number") for d in [found] + dupes)
|
||||
warnings.append(
|
||||
f"{cid}: marker found on {1 + len(dupes)} issues ({numbers}) — "
|
||||
"commenting on the lowest-numbered one; the rest want a human merge"
|
||||
)
|
||||
|
||||
if found:
|
||||
actions.append({
|
||||
"candidate_id": cid,
|
||||
"action": "comment",
|
||||
"issue": f"{repo}#{found['number']}",
|
||||
"issue_number": found["number"],
|
||||
"issue_state": found.get("state", "unknown"),
|
||||
"marker": marker(cid),
|
||||
"body": render(templates["update-comment"], values),
|
||||
"note": "existing issue is closed; issuer decides whether to reopen"
|
||||
if found.get("state") == "closed" else None,
|
||||
})
|
||||
else:
|
||||
actions.append({
|
||||
"candidate_id": cid,
|
||||
"action": "file",
|
||||
"marker": marker(cid),
|
||||
"proposed_title": render(templates["issue-title"], values),
|
||||
"body": render(templates["issue-body"], values),
|
||||
"required_labels": [OFFLOAD_LABEL],
|
||||
"labels_note": "the four-axis anxious taxonomy is issuer's call; "
|
||||
f"{OFFLOAD_LABEL} is required and issuer creates it if absent",
|
||||
})
|
||||
|
||||
counts = {a: sum(1 for x in actions if x["action"] == a) for a in ("file", "comment", "skip")}
|
||||
return {
|
||||
"target": target,
|
||||
"repo": repo,
|
||||
"filing": filing,
|
||||
"filing_unavailable_reason": reason,
|
||||
"run_date": args.run_date,
|
||||
"window": args.window,
|
||||
"audit_page": args.audit_page,
|
||||
"delegation": "offload-analyst -> Agent(anxious:issuer-agent) -> cluster:gitea-agent -> Gitea",
|
||||
"summary": {
|
||||
"to_file": counts["file"],
|
||||
"to_comment": counts["comment"],
|
||||
"skipped": counts["skip"],
|
||||
"searched_issues": len(existing),
|
||||
},
|
||||
"warnings": warnings,
|
||||
"actions": actions,
|
||||
}
|
||||
|
||||
|
||||
def build_queries(args, cls_doc, jud_doc):
|
||||
target = cls_doc.get("target") or jud_doc.get("target")
|
||||
repo, filing, reason = resolve_repo(args, target)
|
||||
ids = [c["candidate_id"] for c in cls_doc.get("candidates", []) if c.get("verdict") == "file"]
|
||||
doc = {
|
||||
"target": target,
|
||||
"repo": repo,
|
||||
"filing": filing,
|
||||
"filing_unavailable_reason": reason,
|
||||
"candidates_to_search": ids,
|
||||
}
|
||||
if filing == "available" and ids:
|
||||
doc["queries"] = {
|
||||
"full_text": [
|
||||
{"tool": "search_issues", "repo": repo, "state": "all",
|
||||
"q": f"ia-candidate: {cid}"} for cid in ids
|
||||
],
|
||||
"label_scan": {"tool": "list_issues", "repo": repo, "state": "all",
|
||||
"labels": [OFFLOAD_LABEL]},
|
||||
}
|
||||
doc["note"] = (
|
||||
"Run BOTH and merge into one array of "
|
||||
"{number, state, title, body, comments[]} for `filing-plan plan --existing`. "
|
||||
"The label scan is authoritative; the full-text query is the fast path and "
|
||||
"may miss on indexer tokenization. See references/issue-template.md."
|
||||
)
|
||||
return doc
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(prog="filing-plan")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
def common(p):
|
||||
p.add_argument("--classified", required=True, help="bin/boundary-classify output")
|
||||
p.add_argument("--judgments", required=True, help="the analyst's judgments JSON")
|
||||
p.add_argument("--target-repo", help="owner/repo, if already known")
|
||||
p.add_argument("--target-path", help="the audited plugin's directory")
|
||||
p.add_argument("--workspace", default=DEFAULT_WORKSPACE)
|
||||
p.add_argument("--gitea-host", default=DEFAULT_GITEA_HOST)
|
||||
p.add_argument("--json", action="store_true", help="compact single-line JSON")
|
||||
|
||||
q = sub.add_parser("queries", help="resolve the repo and emit the marker searches")
|
||||
common(q)
|
||||
|
||||
p = sub.add_parser("plan", help="decide file/comment/skip per candidate")
|
||||
common(p)
|
||||
p.add_argument("--existing", required=True,
|
||||
help="JSON array of issues from the marker search. Required even when "
|
||||
"empty ([]): defaulting to 'nothing exists' would duplicate on re-run.")
|
||||
p.add_argument("--audit-page", required=True, help="e.g. Audits/anxious/2026-07-29")
|
||||
p.add_argument("--window", required=True, help="e.g. 2026-07-22 -> 2026-07-29")
|
||||
p.add_argument("--run-date", required=True)
|
||||
p.add_argument("--wiki-repo", default=DEFAULT_WIKI_REPO)
|
||||
p.add_argument("--status", default="persisting",
|
||||
help="snapshot-diff status for the update comment (audit-snapshot diff)")
|
||||
p.add_argument("--template", default=DEFAULT_TEMPLATE)
|
||||
|
||||
args = ap.parse_args()
|
||||
with open(args.classified) as fh:
|
||||
cls_doc = json.load(fh)
|
||||
with open(args.judgments) as fh:
|
||||
jud_doc = json.load(fh)
|
||||
|
||||
if args.cmd == "queries":
|
||||
out = build_queries(args, cls_doc, jud_doc)
|
||||
else:
|
||||
with open(args.existing) as fh:
|
||||
existing = json.load(fh)
|
||||
if not isinstance(existing, list):
|
||||
sys.exit("error: --existing must be a JSON array of issues")
|
||||
out = build(args, cls_doc, jud_doc, existing, load_templates(args.template))
|
||||
|
||||
json.dump(out, sys.stdout, indent=None if args.json else 2, ensure_ascii=False)
|
||||
sys.stdout.write("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
---
|
||||
description: Audit a Claude Code plugin for steps that should be a deterministic script instead of raw inference — measure, classify, file the well-evidenced candidates, and write the run summary. Usage — /offload-audit <plugin|path> [--days N]
|
||||
argument-hint: <plugin name or path> [--days N]
|
||||
allowed-tools: ["Bash", "Read", "Grep", "Glob", "Skill", "Agent"]
|
||||
---
|
||||
|
||||
# /offload-audit
|
||||
|
||||
Run the full audit on **$ARGUMENTS**.
|
||||
|
||||
1. Parse `$ARGUMENTS` into a target (a plugin name or a directory path) and an
|
||||
optional `--days N` window, defaulting to 30.
|
||||
2. If no target was given, ask which plugin to audit. Do not guess one, and do
|
||||
not default to the current directory — auditing the wrong plugin wastes a
|
||||
full transcript scan.
|
||||
3. Invoke the `offload-audit` skill and follow it exactly, including the filing
|
||||
gate and the marker search. For a heavy target, or when the report should not
|
||||
consume this session's context, hand the work to
|
||||
`Agent(subagent_type="inference-arbitrage:offload-analyst")` instead — one
|
||||
agent for the whole audit, never one per skill (spec §7).
|
||||
|
||||
Filing is implicit: a candidate that clears the rubric gate and the value
|
||||
threshold is filed without asking. Everything below the gate is reported as a
|
||||
boundary question and filed nowhere.
|
||||
+6
-6
@@ -122,16 +122,16 @@ These block design, not just code. Answers go into `Methodology/Calibration.md`.
|
||||
|
||||
## Phase 6 — Output paths
|
||||
|
||||
- [ ] **6.1** `references/issue-template.md` per PLAN §10.
|
||||
- [ ] **6.2** `anxious:issuer` delegation path; create the `token-offload` label
|
||||
- [x] **6.1** `references/issue-template.md` per PLAN §10.
|
||||
- [x] **6.2** `anxious:issuer` delegation path; create the `token-offload` label
|
||||
on a target repo on first audit (implicitly, via `issuer`).
|
||||
- [ ] **6.3** **Idempotency (FR-6.3)** — `ia-candidate` marker search before
|
||||
- [x] **6.3** **Idempotency (FR-6.3)** — `ia-candidate` marker search before
|
||||
filing; comment on the existing issue instead of duplicating. Verify by
|
||||
running the same audit twice (S5). This is the highest-consequence
|
||||
correctness detail in the whole plugin.
|
||||
- [ ] **6.4** Summary page generation → `Audits/<target>/<date>` + `Latest`.
|
||||
- [ ] **6.5** Graceful `filing: unavailable` for repo-less targets (FR-6.4).
|
||||
- [ ] **6.6** `skills/offload-audit/SKILL.md` and `commands/offload-audit.md`
|
||||
- [x] **6.4** Summary page generation → `Audits/<target>/<date>` + `Latest`.
|
||||
- [x] **6.5** Graceful `filing: unavailable` for repo-less targets (FR-6.4).
|
||||
- [x] **6.6** `skills/offload-audit/SKILL.md` and `commands/offload-audit.md`
|
||||
tying the whole procedure together.
|
||||
|
||||
## Phase 7 — Acceptance
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
# issue-template.md — the issue body contract
|
||||
|
||||
The shape of every issue this plugin causes to be filed on a **target plugin's
|
||||
own repo** (output path A, `design/plan.md` §10, FR-6). Loaded at runtime by
|
||||
`bin/filing-plan`, which renders it; it is kept here rather than in the script
|
||||
body so that a change to what an issue says is a reviewable diff.
|
||||
|
||||
Format is the same as `signals-catalog.md`: each `## <section>` heading below is
|
||||
followed by **exactly one fenced block**, and the fenced block is the only part
|
||||
the parser reads. Prose outside the fences is for humans.
|
||||
|
||||
Placeholders are `{{name}}`. Every placeholder in a template must be supplied —
|
||||
`filing-plan` fails loudly on an unknown or missing one rather than rendering a
|
||||
half-filled issue.
|
||||
|
||||
## Who decides what
|
||||
|
||||
`filing-plan` renders the **body**, because the body is the contract: the marker,
|
||||
the measurement, the rubric verdict, and the falsifiability triple are all
|
||||
already-computed values and rendering them is a `pure-script` step.
|
||||
|
||||
It does **not** decide where the issue goes. Repo, final title, labels beyond
|
||||
`token-offload`, and milestone are `anxious:issuer-agent`'s call, per its own
|
||||
taxonomy. The `issue-title` section below is a *proposal* passed to `issuer`,
|
||||
not an instruction.
|
||||
|
||||
## The marker is load-bearing
|
||||
|
||||
The first line of every body — and of every update comment — is
|
||||
|
||||
```text
|
||||
<!-- ia-candidate: <candidate_id> -->
|
||||
```
|
||||
|
||||
It is how a re-run recognizes its own prior work (FR-6.3). It renders invisible
|
||||
in Gitea, is stable across runs because `candidate_id` is, and must never be
|
||||
edited by hand on a filed issue. Removing it from an issue causes the next audit
|
||||
to file a duplicate — which the spec names as the single most likely way for
|
||||
this plugin to become hated.
|
||||
|
||||
## issue-title
|
||||
|
||||
A proposal. `issuer` may rewrite it.
|
||||
|
||||
```text
|
||||
token-offload: {{headline}} ({{skill}})
|
||||
```
|
||||
|
||||
## issue-body
|
||||
|
||||
`{{examples}}` renders as a markdown list, one input→output pair per line, with
|
||||
the edge case marked. `{{determinism_tests}}` renders as `T1 ✓ T2 ✓ …`, using ✗
|
||||
for a failed test. `{{evidence}}` always renders — it states whether the cost
|
||||
claim is `measured`, `thin`, or `unmeasured`, so a reader never has to guess
|
||||
whether the number is evidence or arithmetic.
|
||||
|
||||
```markdown
|
||||
<!-- ia-candidate: {{candidate_id}} -->
|
||||
|
||||
**Measured cost.** {{measured_cost}}
|
||||
|
||||
**Evidence strength.** {{evidence}}
|
||||
|
||||
**Boundary position.** {{position}} — confidence {{boundary_confidence}}.
|
||||
Determinism tests: {{determinism_tests}}
|
||||
|
||||
**Proposed signature.**
|
||||
`{{signature}}`
|
||||
|
||||
**Examples.**
|
||||
|
||||
{{examples}}
|
||||
|
||||
**When a human would overrule this.** {{overrule_case}}
|
||||
|
||||
**Escalation path.** {{escalation_path}}
|
||||
|
||||
**What crosses the boundary.** {{digest_schema}}
|
||||
|
||||
---
|
||||
|
||||
This is a proposal from an audit, not an accepted design. It is deliberately
|
||||
falsifiable: if the overrule case above is *common*, the cut belongs earlier in
|
||||
the pipeline than proposed, and this issue should be reshaped rather than
|
||||
implemented. Closing this as `wontfix` is a useful outcome and is tracked as a
|
||||
calibration signal.
|
||||
|
||||
Audit: `{{audit_page}}` on the {{wiki_repo}} wiki. Rubric v{{rubric_version}}.
|
||||
```
|
||||
|
||||
## update-comment
|
||||
|
||||
Posted instead of a new issue when the marker is already present on the repo
|
||||
(FR-6.3). It carries the marker again so that the search finds this issue
|
||||
whether the marker lives in the body or only in a comment.
|
||||
|
||||
```markdown
|
||||
<!-- ia-candidate: {{candidate_id}} -->
|
||||
|
||||
**Re-measured {{run_date}}** — window {{window}}. Status: {{status}}.
|
||||
|
||||
{{measured_cost}}
|
||||
|
||||
Evidence strength: {{evidence}}
|
||||
|
||||
Boundary position {{position}}, confidence {{boundary_confidence}} (rubric
|
||||
v{{rubric_version}}).
|
||||
|
||||
Audit: `{{audit_page}}` on the {{wiki_repo}} wiki.
|
||||
```
|
||||
|
||||
## marker-search
|
||||
|
||||
How the pre-filing search is actually performed (FR-6.3). This is not
|
||||
hand-waved: both queries below run against the **target** repo, through
|
||||
`anxious:issuer-agent` → `cluster:gitea-agent`, and their merged results are
|
||||
what `filing-plan plan --existing` consumes.
|
||||
|
||||
**Three steps, not one.** The obvious one-query design does not work, and the
|
||||
reason was established empirically against the live Gitea (2026-07-29, probe
|
||||
issue `kotkan/claude-plugin-inference-arbitrage#9`, since closed):
|
||||
|
||||
> **Neither `search_issues` nor `list_issues` returns the issue `body`.** Both
|
||||
> return metadata only — number, title, state, labels, timestamps. A marker
|
||||
> match therefore *cannot* be made from a search result. The body must be
|
||||
> fetched per issue with `issue_read(method="get")`, and comments with
|
||||
> `issue_read(method="get_comments")`.
|
||||
|
||||
Also established by the same probe: `search_issues` **does** full-text match on
|
||||
body content, including text inside an HTML comment (a query for a string that
|
||||
appeared only in the body, never in the title, returned the issue). So the
|
||||
full-text path works — it just cannot be trusted to *confirm* a match, only to
|
||||
suggest one.
|
||||
|
||||
```text
|
||||
1. candidates list_issues(owner, repo, labels=["token-offload"], state="all")
|
||||
search_issues(query="ia-candidate: <candidate_id>", owner, state="all")
|
||||
-> union of issue numbers; bodies are NOT in these responses
|
||||
2. fetch issue_read(method="get", index=N) -> body
|
||||
issue_read(method="get_comments", index=N) -> comment bodies
|
||||
(comments only needed when the body has no marker)
|
||||
3. match filing-plan plan --existing <merged.json> -> local literal match
|
||||
```
|
||||
|
||||
Step 1's label scan is the authoritative half: every issue this plugin files
|
||||
carries `token-offload`, so it is a bounded, exact superset of the issues that
|
||||
can carry a marker. The full-text query is the safety net for an issue whose
|
||||
label a human removed. Step 3's match is a literal string comparison and is the
|
||||
only thing that decides identity — the search engine never does.
|
||||
|
||||
If step 1 finds no `token-offload` label on the target repo, that is the
|
||||
first-audit case: there are no prior issues, `issuer` creates the label with the
|
||||
first filing, and the empty result is correct.
|
||||
|
||||
The merged shape handed to `--existing` is a JSON array of
|
||||
`{number, state, title, body, comments: [...]}`. `filing-plan` matches
|
||||
`<!--\s*ia-candidate:\s*<id>\s*-->` against the body **and each comment**, so an
|
||||
issue keeps its identity even if a human rewrites the body entirely, as long as
|
||||
one update comment survives — also verified against the live probe issue.
|
||||
|
||||
**`--existing` is mandatory, even when empty.** `filing-plan plan` refuses to
|
||||
run without it rather than defaulting to "nothing exists", because that default
|
||||
turns a failed search into a duplicate-filing spree — the exact outcome FR-6.3
|
||||
exists to prevent.
|
||||
@@ -16,19 +16,22 @@ plan §6 is the source, with the two additions noted at the bottom.
|
||||
},
|
||||
"window": {"since": "...", "until": "...", "days": 7},
|
||||
"coverage": {"ratio": 0.735, "attributed_turns": 812},
|
||||
"totals": {"weighted_tokens": 41200000, "invocations": 60, "share_mechanical": 0.61},
|
||||
"totals": {"weighted_tokens": 41200000, "invocations": 60,
|
||||
"share_mechanical": 0.61},
|
||||
"candidates": [{
|
||||
"candidate_id": "anxious/steward/label-derivation",
|
||||
"signature": ["list_issues(<str>)", "issue_read(<n>)"],
|
||||
"signature_source": "explicit | scan-ngram | slug",
|
||||
"signature_hash": "b41f...", // sha256 of the joined signature, 16 hex
|
||||
"signature_hash": "b41f...", // sha256 of the signature, 16 hex
|
||||
"skill": "anxious:steward-agent",
|
||||
"position": "llm-over-script-digest",
|
||||
"boundary_confidence": "high",
|
||||
"determinism_tests": {"T1": true, "...": true},
|
||||
"falsifiability": {"signature": true, "examples": true, "overrule_case": true},
|
||||
"measurement": {"invocations": 22, "offload_waste": 6900000, "offload_value": 8400000,
|
||||
"share_of_plugin": 0.22, "cost_per_invocation": 381818},
|
||||
"falsifiability": {"signature": true, "examples": true,
|
||||
"overrule_case": true},
|
||||
"measurement": {"invocations": 22, "offload_waste": 6900000,
|
||||
"offload_value": 8400000, "share_of_plugin": 0.22,
|
||||
"cost_per_invocation": 381818},
|
||||
"measurement_strength": "measured | thin | unmeasured",
|
||||
"digest_schema": "...", "escalation_path": "...",
|
||||
"issue": "oleks/claude-plugin-anxious#41",
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
---
|
||||
name: offload-audit
|
||||
description: Run a full offload audit on a Claude Code plugin — measure what its skills and agents actually cost, classify each candidate against the boundary rubric, file the well-evidenced ones as issues on the target's own repo, and write the run summary to this plugin's wiki. Trigger on "audit this plugin for token optimization", "can any of this be a script", "offload analysis", "where is this plugin wasting inference", "check <plugin> for scripting opportunities", "what is this plugin paying inference for", "run an offload audit". Read-only with respect to the audited plugin.
|
||||
allowed-tools: Bash, Read, Grep, Glob, Skill, Agent
|
||||
---
|
||||
|
||||
# Audit a plugin for work that should be a script
|
||||
|
||||
Six steps, in order. Steps 1–3 are scripts, step 4 is the only judgment, steps
|
||||
5–6 are delegation. **Do not do by hand anything a `bin/` tool does** — this
|
||||
plugin is subject to its own thesis (FR-8), and an audit that counts things by
|
||||
inference is the finding, not the method.
|
||||
|
||||
Full rubric: the `boundary-rubric` skill. Deeper reporting discipline:
|
||||
`agents/offload-analyst.md`. Issue-body contract and the marker-search
|
||||
mechanics: `${CLAUDE_PLUGIN_ROOT}/references/issue-template.md`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Resolve and measure
|
||||
|
||||
```bash
|
||||
IA=${CLAUDE_PLUGIN_ROOT}
|
||||
$IA/bin/plugin-inventory <path|name> --json > inv.json # static pass
|
||||
$IA/bin/offload-scan --plugin <name> --days 30 --json > scan.json # dynamic pass
|
||||
```
|
||||
|
||||
`plugin-inventory` refuses a directory that is not a plugin rather than guessing
|
||||
(FR-1.3). A target with no transcript history is still auditable — the static
|
||||
pass stands alone and every candidate is marked `unmeasured` (FR-2.4).
|
||||
|
||||
## 2. Read the measurement honestly, before looking for candidates
|
||||
|
||||
Open the report with window, **absolute** evidence counts, the named structural
|
||||
blind spots (hooks, agents, direct `commands/`), and only then the coverage
|
||||
ratio — **stated as dilution, not completeness.**
|
||||
|
||||
> Coverage answers "how much of these sessions was about something else", not
|
||||
> "how much of this plugin's work did we see."
|
||||
|
||||
A real run against `anxious` measured 0.021. That is not a failed scan; it is
|
||||
long multi-topic sessions. Reporting it as completeness is the easiest way to
|
||||
mislead the reader. Then give the mechanical-share headline — the number that
|
||||
says whether there is anything here at all.
|
||||
|
||||
## 3. Gather candidates
|
||||
|
||||
From `plugin-inventory`: high verb ratio **with no `bin/` script behind it**
|
||||
(a conjunction — never the ratio alone), duplicated command blocks, rule tables,
|
||||
low script coverage. From `offload-scan`: high MTR with real spend, recurring
|
||||
n-grams, read amplification, retry density, fan-out. `judgment_density` is a
|
||||
**brake** — high judgment density with high spend means the plugin is healthy
|
||||
and must be reported that way.
|
||||
|
||||
`aggregate.hook_prose_drift` entries are **not offload candidates.** Pass them
|
||||
with `"category": "hook-prose-drift"` so they come back as `drift-note`.
|
||||
|
||||
## 4. Judge, then grade mechanically
|
||||
|
||||
This is the only step that is inference. Invoke the `boundary-rubric` skill per
|
||||
candidate; write the five determinism tests, the falsifiability triple, the
|
||||
escalation path and digest schema into a judgments JSON (shape:
|
||||
`tests/calibration/*.judgments.json`), then:
|
||||
|
||||
```bash
|
||||
$IA/bin/boundary-classify judgments.json > classified.json
|
||||
```
|
||||
|
||||
**Never grade by hand and never override the verdict it returns.** The filing
|
||||
threshold (`high` confidence **and** ≥2% of audited spend **and** the triple
|
||||
complete including the overrule case) lives in that script.
|
||||
|
||||
> **No overrule case → boundary question, reported, never filed.** Hard gate,
|
||||
> FR-4.2. When genuinely torn, do not file.
|
||||
|
||||
## 5. Output path A — file the issues
|
||||
|
||||
**Filing is implicit.** Do not ask permission to file a candidate that cleared
|
||||
the gate — this matches the environment's standing bug-handling convention. What
|
||||
you must not do is file anything the gate rejected.
|
||||
|
||||
**This plugin holds no Gitea credentials and calls no Gitea MCP tool.** The chain
|
||||
is fixed:
|
||||
|
||||
```text
|
||||
offload-audit → Agent(anxious:issuer-agent) → cluster:gitea-agent → Gitea
|
||||
```
|
||||
|
||||
### 5a. Resolve the repo and get the searches
|
||||
|
||||
```bash
|
||||
$IA/bin/filing-plan queries --classified classified.json --judgments judgments.json \
|
||||
--target-path <resolved plugin path> > queries.json
|
||||
```
|
||||
|
||||
If `filing` is `unavailable` (FR-6.4 — a third-party or marketplace plugin with
|
||||
no repo this environment can write to), **skip the rest of path A entirely**,
|
||||
carry the reason into the report and into the run summary, and go to step 6.
|
||||
Path B still runs. This is a normal outcome, not an error.
|
||||
|
||||
### 5b. Run the marker search — the step that prevents duplicates
|
||||
|
||||
Delegate the searches in `queries.json`. Both queries, merged; **the label scan
|
||||
is the authoritative one.** Bodies are not in listing responses, so each
|
||||
candidate issue needs an `issue_read`:
|
||||
|
||||
```text
|
||||
Agent(subagent_type="anxious:issuer-agent", prompt=...)
|
||||
"Read-only lookup on <owner/repo>, no writes. For an inference-arbitrage
|
||||
audit I need to know whether these candidates already have issues.
|
||||
1. list_issues(owner, repo, labels=['token-offload'], state='all')
|
||||
2. search_issues(query='ia-candidate: <id>', owner, state='all') [per id]
|
||||
3. For each issue number found: issue_read(method='get') → body
|
||||
and issue_read(method='get_comments') → comment bodies
|
||||
Return a JSON array of {number, state, title, body, comments:[{body}]}.
|
||||
Do not create, edit, or label anything."
|
||||
```
|
||||
|
||||
Write the returned array to `existing.json`. **If the search fails or you cannot
|
||||
get bodies, stop path A and report it** — do not proceed with an empty array.
|
||||
`filing-plan` requires `--existing` precisely so that a failed search cannot
|
||||
silently become a duplicate-filing spree (FR-6.3).
|
||||
|
||||
### 5c. Build the plan
|
||||
|
||||
```bash
|
||||
$IA/bin/filing-plan plan --classified classified.json --judgments judgments.json \
|
||||
--target-path <path> --existing existing.json \
|
||||
--audit-page "Audits/<target>/<YYYY-MM-DD>" \
|
||||
--window "<since> -> <until>" --run-date <YYYY-MM-DD> \
|
||||
--status <snapshot-diff status, if known> > filing.json
|
||||
```
|
||||
|
||||
Each action is `file`, `comment`, or `skip`, with the exact body already
|
||||
rendered. **Execute the plan verbatim — do not rewrite a body, and never turn a
|
||||
`comment` into a `file`.** Surface any `warnings` to the user.
|
||||
|
||||
### 5d. Execute through `issuer`
|
||||
|
||||
One `Agent` call per action (or one call carrying all of them):
|
||||
|
||||
```text
|
||||
Agent(subagent_type="anxious:issuer-agent", prompt=...)
|
||||
# for action == "file"
|
||||
"File this on <owner/repo>. It is an inference-arbitrage audit finding.
|
||||
Proposed title: <proposed_title> (yours to rewrite per your taxonomy)
|
||||
Body — file VERBATIM, the first line is an idempotency marker that a later
|
||||
audit searches for and MUST NOT be altered or reformatted:
|
||||
<body>
|
||||
Labels: 'token-offload' is required — create it on this repo if absent
|
||||
(colour #8b5cf6, 'A step paying for inference where a deterministic script
|
||||
would do'). Add the four-axis taxonomy labels and any milestone per your own
|
||||
rules; typically kind/chore or kind/capability-gap, area/agent-behavior,
|
||||
domain/agents, activity/automate.
|
||||
Return the issue reference in full owner/repo#num form."
|
||||
|
||||
# for action == "comment"
|
||||
"Add this comment to <issue>, an existing inference-arbitrage finding being
|
||||
re-measured. Do not open a new issue. Post the body VERBATIM (the first line
|
||||
is an idempotency marker). If the issue is closed, use your judgement about
|
||||
reopening; the audit does not require it.
|
||||
<body>"
|
||||
```
|
||||
|
||||
`issuer` owns repo, final title, labels beyond `token-offload`, and milestone.
|
||||
You own the body.
|
||||
|
||||
Collect the returned refs into `issue-map.json` — `{"<candidate_id>": "owner/repo#N"}`.
|
||||
|
||||
## 6. Output path B — the run summary
|
||||
|
||||
Path B runs **after** path A so the summary records real issue numbers, and it
|
||||
runs **even when path A was skipped**.
|
||||
|
||||
```bash
|
||||
$IA/bin/audit-snapshot write --target <name> --from inv.json scan.json \
|
||||
--classified classified.json --issue-map issue-map.json --repo <owner/repo>
|
||||
$IA/bin/audit-snapshot pages --target <name> > pages.json
|
||||
```
|
||||
|
||||
`audit-snapshot` renders `Audits/<target>/<YYYY-MM-DD>`, regenerates
|
||||
`Audits/<target>/Latest`, and appends the machine-readable record to
|
||||
`Data/<target>/snapshots.jsonl`. It does not write to Gitea either — delegate
|
||||
each page in `pages.json` to `cluster:gitea-agent` for the wiki of
|
||||
`kotkan/claude-plugin-inference-arbitrage`, respecting `append_only` on the
|
||||
JSONL.
|
||||
|
||||
When filing was unavailable, say so on the page: pass the reason through
|
||||
`audit-snapshot write --note "<reason>"` if that flag exists, and state it in
|
||||
the report regardless.
|
||||
|
||||
## 7. Report to the user
|
||||
|
||||
In this order (FR-7.3): coverage and window → mechanical-share headline →
|
||||
ranked candidate table with position, confidence and value → diff against the
|
||||
previous snapshot → boundary questions, as questions → drift notes, with their
|
||||
documentation fix → issues filed or updated, in full `owner/repo#num` form →
|
||||
the wiki page path.
|
||||
|
||||
Rank by measured `offload_value` and say that you are doing so. Name skills and
|
||||
agents, **never UUIDs or session ids**. Emit **only shapes and counts** — no
|
||||
transcript content, file contents, command arguments, or user prose ever enters
|
||||
a report, an issue, or a wiki page (FR-3.5).
|
||||
|
||||
### "Nothing to offload here" is a complete answer
|
||||
|
||||
If nothing clears the bar, say so plainly and stop. Do not pad the report to
|
||||
look thorough. `token-budget` is expected to produce exactly this result, and a
|
||||
run against it that yields confident candidates is **a bug in this plugin**, not
|
||||
a finding about `token-budget`. Report the `pure-inference` findings by name —
|
||||
they are the evidence that you looked rather than shrugged.
|
||||
|
||||
### Never
|
||||
|
||||
- Never edit the plugin you audit, and never implement your own recommendation.
|
||||
- Never call a Gitea tool directly from this skill.
|
||||
- Never file a candidate the gate rejected, or one whose repo is unavailable.
|
||||
- Never re-run an audit "to be safe" without the marker search. That is the one
|
||||
mistake that makes this plugin hated.
|
||||
@@ -73,7 +73,7 @@ list and the unaddressed list are all in the output.
|
||||
### The statuses
|
||||
|
||||
| Status | Means |
|
||||
|---|---|
|
||||
| --- | --- |
|
||||
| `new` | Not present in the compared snapshot. |
|
||||
| `persisting` | Both volume-normalized metrics inside the ±10% noise band. |
|
||||
| `grown` / `shrunk` | Cost per invocation moved beyond the band; share of spend breaks ties. |
|
||||
|
||||
Executable
+178
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
# bin/filing-plan: output path A — the marker-based idempotency contract
|
||||
# (FR-6.3), template rendering (FR-6.1), and repo-less degradation (FR-6.4).
|
||||
#
|
||||
# The load-bearing case is "run 1 files, run 2 comments". It is proved here the
|
||||
# only way that means anything: run 1's ACTUAL rendered body is fed back as the
|
||||
# tracker's state for run 2. Nothing is asserted about the marker by hand.
|
||||
#
|
||||
# tests/fixtures/gitea-issues.json is copied verbatim from real Gitea API
|
||||
# responses (probe issue kotkan/claude-plugin-inference-arbitrage#9, since
|
||||
# closed), so the shape this parses is the shape the API returns.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||
|
||||
BIN=bin/filing-plan
|
||||
JUD=tests/fixtures/filing-judgments.json
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
fail=0
|
||||
|
||||
check() {
|
||||
local what=$1 got=$2 want=$3
|
||||
if [ "$got" = "$want" ]; then
|
||||
echo " ok $what = $want"
|
||||
else
|
||||
echo " FAIL $what = $got (want $want)" >&2
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
# q <file> <python-expr over `d`>
|
||||
q() { python3 -c "import json,sys;d=json.load(open(sys.argv[1]));print($2)" "$1"; }
|
||||
|
||||
bin/boundary-classify "$JUD" >"$tmp/cls.json"
|
||||
|
||||
plan() { # plan <existing.json> <out.json> [extra args...]
|
||||
local existing=$1 out=$2
|
||||
shift 2
|
||||
$BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
||||
--target-repo oleks/claude-plugin-fixture --existing "$existing" \
|
||||
--audit-page Audits/fixture-plugin/2026-07-29 \
|
||||
--window "2026-07-22 -> 2026-07-29" --run-date 2026-07-29 "$@" >"$out"
|
||||
}
|
||||
|
||||
echo "== the fixture's two candidates grade as expected (guards the rest) =="
|
||||
check "A verdict" "$(q "$tmp/cls.json" "d['candidates'][0]['verdict']")" "file"
|
||||
check "B verdict" "$(q "$tmp/cls.json" "d['candidates'][1]['verdict']")" "no-offload"
|
||||
|
||||
echo "== RUN 1: empty tracker -> files, with the marker as line 1 =="
|
||||
echo '[]' >"$tmp/empty.json"
|
||||
plan "$tmp/empty.json" "$tmp/plan1.json"
|
||||
check "to_file" "$(q "$tmp/plan1.json" "d['summary']['to_file']")" "1"
|
||||
check "to_comment" "$(q "$tmp/plan1.json" "d['summary']['to_comment']")" "0"
|
||||
check "action" "$(q "$tmp/plan1.json" "d['actions'][0]['action']")" "file"
|
||||
check "marker is line 1" \
|
||||
"$(q "$tmp/plan1.json" "d['actions'][0]['body'].splitlines()[0]")" \
|
||||
"<!-- ia-candidate: fixture-plugin/skill-a/label-derivation -->"
|
||||
check "token-offload label required" \
|
||||
"$(q "$tmp/plan1.json" "d['actions'][0]['required_labels']")" "['token-offload']"
|
||||
check "non-file verdict skipped" "$(q "$tmp/plan1.json" "d['actions'][1]['action']")" "skip"
|
||||
|
||||
echo "== the body carries every element of the FR-6 contract =="
|
||||
body=$(q "$tmp/plan1.json" "d['actions'][0]['body']")
|
||||
for needle in \
|
||||
"**Measured cost.**" "**Evidence strength.**" "**Boundary position.**" \
|
||||
"Determinism tests: T1 ✓ T2 ✓ T3 ✓ T4 ✓ T5 ✓" \
|
||||
"**Proposed signature.**" "derive_labels(" "**Examples.**" \
|
||||
"**When a human would overrule this.**" "**Escalation path.**" \
|
||||
"**What crosses the boundary.**" \
|
||||
"Audits/fixture-plugin/2026-07-29" "Rubric v1.0.0"; do
|
||||
if grep -qF -- "$needle" <<<"$body"; then
|
||||
echo " ok body contains: $needle"
|
||||
else
|
||||
echo " FAIL body missing: $needle" >&2
|
||||
fail=1
|
||||
fi
|
||||
done
|
||||
check "measurement is quoted, not invented" \
|
||||
"$(grep -cF 'Recurring sequence observed 17x' <<<"$body")" "1"
|
||||
check "3 examples incl. the edge case" "$(grep -c '^- ' <<<"$body")" "3"
|
||||
|
||||
echo "== RUN 2 (FR-6.3, S5): run 1's own body is now on the tracker -> comment =="
|
||||
python3 - "$tmp/plan1.json" "$tmp/tracker.json" <<'PY'
|
||||
import json, sys
|
||||
plan = json.load(open(sys.argv[1]))
|
||||
filed = next(a for a in plan["actions"] if a["action"] == "file")
|
||||
json.dump([{"number": 41, "state": "open", "title": filed["proposed_title"],
|
||||
"body": filed["body"], "comments": []}], open(sys.argv[2], "w"))
|
||||
PY
|
||||
plan "$tmp/tracker.json" "$tmp/plan2.json" --status grown
|
||||
check "files nothing on re-run" "$(q "$tmp/plan2.json" "d['summary']['to_file']")" "0"
|
||||
check "comments instead" "$(q "$tmp/plan2.json" "d['summary']['to_comment']")" "1"
|
||||
check "on the right issue" \
|
||||
"$(q "$tmp/plan2.json" "d['actions'][0]['issue']")" "oleks/claude-plugin-fixture#41"
|
||||
check "comment re-carries the marker" \
|
||||
"$(q "$tmp/plan2.json" "d['actions'][0]['body'].splitlines()[0]")" \
|
||||
"<!-- ia-candidate: fixture-plugin/skill-a/label-derivation -->"
|
||||
check "comment reports the diff status" \
|
||||
"$(q "$tmp/plan2.json" "'Status: grown' in d['actions'][0]['body']")" "True"
|
||||
|
||||
echo "== RUN 3: the comment from run 2 is the only marker left (body rewritten) =="
|
||||
python3 - "$tmp/plan2.json" "$tmp/tracker3.json" <<'PY'
|
||||
import json, sys
|
||||
plan = json.load(open(sys.argv[1]))
|
||||
c = next(a for a in plan["actions"] if a["action"] == "comment")
|
||||
json.dump([{"number": 41, "state": "closed", "title": "a human retitled this",
|
||||
"body": "and rewrote the body, dropping the marker",
|
||||
"comments": [{"body": c["body"]}]}], open(sys.argv[2], "w"))
|
||||
PY
|
||||
plan "$tmp/tracker3.json" "$tmp/plan3.json"
|
||||
check "still no duplicate" "$(q "$tmp/plan3.json" "d['summary']['to_file']")" "0"
|
||||
check "found via the comment" "$(q "$tmp/plan3.json" "d['actions'][0]['action']")" "comment"
|
||||
check "closed state surfaced for issuer" \
|
||||
"$(q "$tmp/plan3.json" "d['actions'][0]['issue_state']")" "closed"
|
||||
|
||||
echo "== real Gitea API shapes parse, and a DIFFERENT id does not false-match =="
|
||||
plan tests/fixtures/gitea-issues.json "$tmp/plan4.json"
|
||||
check "unrelated markers -> files" "$(q "$tmp/plan4.json" "d['summary']['to_file']")" "1"
|
||||
check "searched the fixture" "$(q "$tmp/plan4.json" "d['summary']['searched_issues']")" "1"
|
||||
|
||||
echo "== a marker on two issues warns and takes the lowest number =="
|
||||
python3 - "$tmp/plan1.json" "$tmp/dupes.json" <<'PY'
|
||||
import json, sys
|
||||
filed = next(a for a in json.load(open(sys.argv[1]))["actions"] if a["action"] == "file")
|
||||
json.dump([{"number": 77, "state": "open", "title": "later dupe", "body": filed["body"], "comments": []},
|
||||
{"number": 41, "state": "open", "title": "original", "body": filed["body"], "comments": []}],
|
||||
open(sys.argv[2], "w"))
|
||||
PY
|
||||
plan "$tmp/dupes.json" "$tmp/plan5.json"
|
||||
check "lowest issue wins" "$(q "$tmp/plan5.json" "d['actions'][0]['issue_number']")" "41"
|
||||
check "warns a human" "$(q "$tmp/plan5.json" "len(d['warnings'])")" "1"
|
||||
|
||||
echo "== --existing is mandatory: a skipped search must never mean 'file it all' =="
|
||||
if $BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
||||
--target-repo oleks/claude-plugin-fixture --audit-page A --window w \
|
||||
--run-date 2026-07-29 >/dev/null 2>&1; then
|
||||
echo " FAIL plan ran without --existing" >&2
|
||||
fail=1
|
||||
else
|
||||
echo " ok refuses to plan without a search result"
|
||||
fi
|
||||
|
||||
echo "== FR-6.4: a target with no writable repo degrades to summary-only =="
|
||||
# Real, not synthetic: third-party/baoyu-design is a vendored plugin whose only
|
||||
# remote is github.com, i.e. nothing this environment can file to.
|
||||
THIRD=../third-party/baoyu-design
|
||||
if [ -d "$THIRD" ]; then
|
||||
bin/plugin-inventory "$THIRD" --json >"$tmp/tp-inv.json"
|
||||
check "inventory still resolves it" "$(q "$tmp/tp-inv.json" "d['plugin']['name']")" "baoyu-design"
|
||||
$BIN queries --classified "$tmp/cls.json" --judgments "$JUD" \
|
||||
--target-path "$THIRD" --workspace /nonexistent >"$tmp/tp-q.json"
|
||||
check "filing" "$(q "$tmp/tp-q.json" "d['filing']")" "unavailable"
|
||||
check "reason names the remote" \
|
||||
"$(q "$tmp/tp-q.json" "'github.com' in d['filing_unavailable_reason']")" "True"
|
||||
check "no queries emitted" "$(q "$tmp/tp-q.json" "'queries' in d")" "False"
|
||||
|
||||
$BIN plan --classified "$tmp/cls.json" --judgments "$JUD" \
|
||||
--target-path "$THIRD" --workspace /nonexistent --existing "$tmp/empty.json" \
|
||||
--audit-page A --window w --run-date 2026-07-29 >"$tmp/tp-plan.json"
|
||||
check "nothing filed" "$(q "$tmp/tp-plan.json" "d['summary']['to_file']")" "0"
|
||||
check "candidate marked unavailable" \
|
||||
"$(q "$tmp/tp-plan.json" "d['actions'][0]['filing']")" "unavailable"
|
||||
else
|
||||
echo " skip (no third-party/baoyu-design checkout)"
|
||||
fi
|
||||
|
||||
echo "== a writable target resolves through its own checkout's remote =="
|
||||
if [ -d ../worktree-discipline/.git ]; then
|
||||
$BIN queries --classified "$tmp/cls.json" --judgments "$JUD" \
|
||||
--target-path ../worktree-discipline >"$tmp/wd-q.json"
|
||||
check "repo" "$(q "$tmp/wd-q.json" "d['repo']")" "kotkan/claude-plugin-worktree-discipline"
|
||||
check "filing" "$(q "$tmp/wd-q.json" "d['filing']")" "available"
|
||||
check "label scan is the authoritative query" \
|
||||
"$(q "$tmp/wd-q.json" "d['queries']['label_scan']['labels']")" "['token-offload']"
|
||||
else
|
||||
echo " skip (no sibling worktree-discipline checkout)"
|
||||
fi
|
||||
|
||||
exit "$fail"
|
||||
Vendored
+69
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"target": "fixture-plugin",
|
||||
"note": "Synthetic. Two candidates: one that clears the FR-4.2 gate and the 2% filing threshold, one that is correctly left to inference. Shapes match tests/calibration/*.judgments.json.",
|
||||
"audited_spend_weighted_tokens": 1000000,
|
||||
"candidates": [
|
||||
{
|
||||
"candidate_id": "fixture-plugin/skill-a/label-derivation",
|
||||
"skill": "fixture-plugin:skill-a",
|
||||
"headline": "label derivation is a documented lookup table applied by inference",
|
||||
"category": "usage",
|
||||
"position": "llm-over-script-digest",
|
||||
"determinism_tests": {
|
||||
"T1": true,
|
||||
"T2": true,
|
||||
"T3": true,
|
||||
"T4": true,
|
||||
"T5": true
|
||||
},
|
||||
"falsifiability": {
|
||||
"signature": "derive_labels(repo: str, issue: Issue) -> tuple[Labels, list[Ambiguity]]",
|
||||
"examples": [
|
||||
"repo='x', issue.title='fix crash' -> (['kind/bug'], [])",
|
||||
"repo='x', issue.body has a checklist -> (['kind/epic'], [])",
|
||||
"EDGE: repo='x', no signal at all -> ([], [Ambiguity('no axis-1 signal, defer to human')])"
|
||||
],
|
||||
"edge_case_index": 2,
|
||||
"overrule_case": "An issue whose title reads as a bug but which the maintainer is deliberately tracking as a capability gap. The script returns kind/bug; a human says no. It cannot see that intent — it is not in the issue text."
|
||||
},
|
||||
"escalation_path": "Rows the table cannot resolve are emitted as Ambiguity and go to the model, never defaulted. The script proposes labels and never applies them; application stays behind the judgment gate (P4).",
|
||||
"digest_schema": "One typed row per issue: (number, title, has_checklist, proposed_labels, ambiguities). Replaces reading every issue whole.",
|
||||
"measurement": {
|
||||
"offload_value": 200000,
|
||||
"invocations": 12,
|
||||
"attributed_turns": 200,
|
||||
"mtr": 0.68,
|
||||
"judgment_density": 0.11,
|
||||
"ngram": {
|
||||
"sig": ["list_issues(<str>)", "issue_read(<n>)", "label_read(<n>)"],
|
||||
"recurrences": 17
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"candidate_id": "fixture-plugin/skill-b/naming",
|
||||
"skill": "fixture-plugin:skill-b",
|
||||
"category": "usage",
|
||||
"position": "pure-inference",
|
||||
"determinism_tests": {
|
||||
"T1": false,
|
||||
"T2": false,
|
||||
"T3": true,
|
||||
"T4": false,
|
||||
"T5": false
|
||||
},
|
||||
"falsifiability": {
|
||||
"signature": null,
|
||||
"examples": [],
|
||||
"overrule_case": null
|
||||
},
|
||||
"escalation_path": null,
|
||||
"digest_schema": null,
|
||||
"measurement": {
|
||||
"offload_value": 90000,
|
||||
"invocations": 8,
|
||||
"attributed_turns": 120
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
Vendored
+14
@@ -0,0 +1,14 @@
|
||||
[
|
||||
{
|
||||
"number": 9,
|
||||
"state": "closed",
|
||||
"title": "[test artifact] Phase 6 idempotency probe — ia-candidate marker search",
|
||||
"body": "<!-- ia-candidate: probe/phase6/marker-search -->\n\nScratch issue created to verify, for real, that the FR-6.3 marker search finds an\n`ia-candidate` marker through the Gitea API. Not a real candidate; will be closed\nimmediately. See kotkan/claude-plugin-inference-arbitrage#6.",
|
||||
"comments": [
|
||||
{
|
||||
"id": 37761,
|
||||
"body": "<!-- ia-candidate: probe/phase6/comment-only -->\n\nSecond marker, in a comment only, to verify the comment-carried identity path\n(an issue whose body a human rewrote must still be found)."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in New Issue
Block a user