claude-plugin-inference-arbitrage/bin/stability-classify

#!/usr/bin/env python3
"""The evidence-stability gate (FR-4.5, rubric §5b): refuses to file a candidate
whose evidence label is an artifact of which measurement window happened to be
audited.

WHY THIS IS A GATE AND NOT A CAVEAT
-----------------------------------
Phase 7 filed `anxious/agent-wip/release-policy-derivation` on a 30-day window
as `measurement_strength: measured`, 11.94% of audited spend — a confident,
concrete cost claim. The identical candidate, classified over a 7-day window,
came back `thin`, 0.00% of spend. Nothing about the plugin changed between the
two; only the date range did.

A human reading either issue body alone has no way to know the other exists.
That is the `rubric.md` §4a failure — fast, confident, and wrong, with nobody
watching — reached by the plugin's own headline number. It was caught once, by
a human running a second skeptical review, which is not a repeatable control:
the next candidate gets filed only if someone happens to be suspicious that day.

So stability becomes mechanical, in the same register as FR-4.2's overrule-case
gate: a candidate whose evidence flips across windows is downgraded to a
boundary question and never filed, and the flip is emitted as a finding against
this plugin's own repo (FR-8.2), because a confidence label that moves with the
calendar is a bug in the auditor, not a fact about the target.

WHAT COUNTS AS A FLIP
---------------------
Two things, either of which disqualifies:

  1. `measurement_strength` differs between the windows (`measured` vs `thin`
     vs `unmeasured`). This is the label the issue body speaks in, so a change
     here changes what a human is told.
  2. `share_of_audited_spend` lands on opposite sides of MIN_SHARE_OF_SPEND.
     That threshold is the filing gate itself (spec §5.2.7): if one window would
     have filed on value and the other would not, the value claim is not
     load-bearing evidence, it is a window artifact.

Ordinary movement within a band is NOT a flip. A candidate that measures 8% one
window and 5% the next is stable — same label, same side of the threshold — and
that variation is what `audit-snapshot diff` exists to report as a trend. This
gate fires only when the *description a human receives* changes.

NO PRIOR WINDOW MEANS UNCHECKED, NOT UNSTABLE
---------------------------------------------
A first-ever audit of a target, or a candidate seen for the first time, has
nothing to compare against. It files normally, marked
`stability: unchecked`. Manufacturing an instability finding out of an absence
of history would contradict FR-2.4 — a target with no history stays auditable on
its own terms — and would make the first audit of every plugin file nothing.

The prior window is the most recent snapshot that contains this candidate by
FR-5.2 identity (issue > signature hash > slug), not merely the most recent
snapshot: a candidate that sat out one window is still comparable against the
window before it, and using the latest snapshot unconditionally would report
every such candidate as unmatched and let it through.

Runs after `boundary-classify` and before `filing-plan`; emits the same document
shape it consumes, with verdicts revised, so it drops into the pipeline in place
of `classified.json`.
"""

import argparse
import json
import sys

from ia_store import (DEFAULT_STORE, MIN_SHARE_OF_SPEND, build_index,
                      candidate_signature, find_match, load_snapshots,
                      signature_hash, tiered)

GATE = "FR-4.5"

UNCHECKED_NO_HISTORY = "unchecked — no prior window to compare"
UNCHECKED_NO_MATCH = "unchecked — no prior window recorded this candidate"


def keyed(cand, scan, explicit, issue_map):
    """The identity fields FR-5.2 matches on, computed for a candidate that is
    only classified and not yet snapshotted. Derived with the same
    `candidate_signature` the snapshot writer uses — a divergence here would
    make the gate silently match nothing and pass every unstable candidate."""
    sig, source = candidate_signature(cand, scan, explicit)
    return {
        "candidate_id": cand.get("candidate_id"),
        "issue": issue_map.get(cand.get("candidate_id")) or cand.get("issue"),
        "signature_hash": signature_hash(sig),
        "signature_source": source,
    }


def find_prior(key, history):
    """Most recent snapshot holding this candidate, newest first."""
    for snap in reversed(history):
        idx = build_index([c for c, _ in tiered(snap)])
        match, by = find_match(key, idx)
        if match:
            return snap, match, by
    return None, None, None


def crosses_threshold(a, b):
    return (a >= MIN_SHARE_OF_SPEND) != (b >= MIN_SHARE_OF_SPEND)


def flips(prior, curr_strength, curr_share):
    """Named, so the finding says exactly what moved rather than 'unstable'."""
    out = []
    prior_strength = prior.get("measurement_strength")
    prior_share = (prior.get("measurement") or {}).get("share_of_plugin", 0.0) or 0.0

    if prior_strength != curr_strength:
        out.append({
            "field": "measurement_strength",
            "prior": prior_strength,
            "current": curr_strength,
            "why": ("the evidence label the issue body speaks in changed with the window, "
                    "so the same candidate would be described differently to a human "
                    "depending only on when the audit ran"),
        })
    if crosses_threshold(prior_share, curr_share):
        out.append({
            "field": "share_of_audited_spend",
            "prior": prior_share,
            "current": curr_share,
            "threshold": MIN_SHARE_OF_SPEND,
            "why": (f"the windows land on opposite sides of the {MIN_SHARE_OF_SPEND:.0%} "
                    "filing threshold — one window would file on value and the other "
                    "would not, so the value claim is a window artifact"),
        })
    return out


def gate(doc, history, scan, explicit, issue_map):
    target = doc.get("target")
    window = scan.get("window") or {}
    findings = []
    checked = 0

    for cand in doc.get("candidates", []):
        if cand.get("verdict") != "file":
            continue

        if not history:
            cand["stability"] = UNCHECKED_NO_HISTORY
            continue

        key = keyed(cand, scan, explicit, issue_map)
        snap, prior, by = find_prior(key, history)
        if prior is None:
            cand["stability"] = UNCHECKED_NO_MATCH
            continue

        checked += 1
        curr_strength = cand.get("measurement_strength")
        curr_share = cand.get("share_of_audited_spend", 0.0) or 0.0
        moved = flips(prior, curr_strength, curr_share)

        if not moved:
            cand["stability"] = (
                f"stable against {snap.get('run_id')} (matched by {by}): "
                f"{curr_strength}, {curr_share:.2%} of audited spend either side")
            continue

        cand["verdict"] = "boundary-question"
        cand["stability"] = f"unstable against {snap.get('run_id')} (matched by {by})"
        cand.setdefault("reasons", []).append(
            f"HARD GATE ({GATE}): evidence is not stable across measurement windows — "
            + "; ".join(f"{f['field']} {f['prior']!r} -> {f['current']!r}" for f in moved)
            + ". Downgraded to a boundary question, never filed."
        )

        findings.append({
            "gate": GATE,
            "candidate_id": cand.get("candidate_id"),
            "target": target,
            "skill": cand.get("skill"),
            "matched_by": by,
            "prior": {
                "run_id": snap.get("run_id"),
                "window": snap.get("window"),
                "measurement_strength": prior.get("measurement_strength"),
                "share_of_audited_spend": (prior.get("measurement") or {}).get("share_of_plugin"),
            },
            "current": {
                "window": window,
                "measurement_strength": curr_strength,
                "share_of_audited_spend": curr_share,
            },
            "flips": moved,
            "headline": (
                f"{cand.get('candidate_id')} is {prior.get('measurement_strength')} at "
                f"{((prior.get('measurement') or {}).get('share_of_plugin') or 0):.2%} of spend in "
                f"{snap.get('run_id')} and {curr_strength} at {curr_share:.2%} in this window — "
                "the confidence label depends on the measurement window, not on the candidate"),
            # FR-8.2: the instability is a finding about THIS plugin, filed on
            # its own repo. The marker mirrors FR-6.3's `ia-candidate` so a
            # re-run comments instead of duplicating.
            "self_report": {
                "repo": "kotkan/claude-plugin-inference-arbitrage",
                "marker": f"<!-- ia-stability: {cand.get('candidate_id')} -->",
            },
        })

    rows = doc.get("candidates", [])
    summary = doc.setdefault("summary", {})
    summary["to_file"] = sum(1 for c in rows if c.get("verdict") == "file")
    summary["boundary_questions"] = sum(1 for c in rows if c.get("verdict") == "boundary-question")
    summary["stability_checked"] = checked
    summary["stability_downgraded"] = len(findings)

    doc["stability_gate"] = {
        "gate": GATE,
        "snapshots_available": len(history),
        "checked": checked,
        "downgraded": len(findings),
    }
    doc["stability_findings"] = findings
    return doc


def load_json(path):
    with open(path) as fh:
        return json.load(fh)


def main():
    ap = argparse.ArgumentParser(prog="stability-classify")
    ap.add_argument("--classified", required=True, help="bin/boundary-classify output")
    ap.add_argument("--target", required=True, help="target plugin name, as snapshotted")
    ap.add_argument("--store", default=DEFAULT_STORE,
                    help=f"local mirror of the wiki store (default {DEFAULT_STORE})")
    ap.add_argument("--scan", help="bin/offload-scan output, for signature identity")
    ap.add_argument("--signatures", help="JSON {candidate_id: [tool signature, ...]}")
    ap.add_argument("--issue-map", help='JSON {candidate_id: "owner/repo#N"}')
    ap.add_argument("--exclude-run", help="run_id to treat as not-yet-recorded (tests, re-runs)")
    ap.add_argument("--json", action="store_true", help="compact single-line JSON")
    args = ap.parse_args()

    doc = load_json(args.classified)
    history = [s for s in load_snapshots(args.store, args.target)
               if s.get("run_id") != args.exclude_run]

    result = gate(doc, history,
                  load_json(args.scan) if args.scan else {},
                  load_json(args.signatures) if args.signatures else {},
                  load_json(args.issue_map) if args.issue_map else {})

    json.dump(result, sys.stdout, indent=None if args.json else 2)
    sys.stdout.write("\n")


if __name__ == "__main__":
    main()