oleks
dcec9cabfb
Phase 7 filed anxious/agent-wip/release-policy-derivation as `measured` at 11.94% of audited spend on a 30-day window. The same candidate over 7 days came back `thin` at 0.00%. Only the date range differed. A human reading either issue body alone cannot know the other exists, which is rubric.md §4a's failure mode reached through the plugin's own headline number. That was caught by a human running a second skeptical review — luck, not a control. It is now mechanical, in the same register as FR-4.2's overrule-case gate: - bin/stability-classify matches each `file` candidate against the most recent prior snapshot recording it (FR-5.2 identity) and downgrades it to a boundary question if `measurement_strength` changed or the share crossed the 2% filing threshold between windows. Movement within a label is a trend, not an instability; no prior window is `unchecked`, not unstable (FR-2.4). - The downgrade is a finding against THIS repo, not the target's — the target did not change, the auditor described it two ways. Carries an `ia-stability` marker so a re-run comments rather than duplicating. - bin/ia_store.py factors the store and identity resolution out of audit-snapshot so both scripts hash signatures identically. A divergence there would make the gate match nothing and fail open. Documented as spec FR-4.5 and rubric §5b (shipped verbatim in references/). Wired into skills/offload-audit as step 4b, before filing-plan; steps 5-6 now consume its output rather than classified.json. S4 is retired in its old form — a human triage catching a bad candidate is not a repeatable test — and re-passed as: the gate catches the real release-policy-derivation case with no human in the loop. Asserted in tests/stability.test.sh case (f) against the verbatim two-window Phase 7 output. Closes kotkan/claude-plugin-inference-arbitrage#11 Suite: 178 assertions, exit 0.
144 lines
4.9 KiB
Python
144 lines
4.9 KiB
Python
"""Snapshot store access and candidate-identity resolution (FR-5.2), shared by
|
|
`bin/audit-snapshot` (which writes and diffs snapshots) and
|
|
`bin/stability-classify` (which reads them to gate filing, FR-4.5).
|
|
|
|
It exists so that identity is computed exactly once. The gate's whole job is to
|
|
find the *same* candidate in an earlier window; if it hashed a signature even
|
|
slightly differently from the writer, it would silently find nothing and pass
|
|
every unstable candidate through — a gate that fails open and looks like it
|
|
worked. Copy-pasting `signature_hash` between the two scripts is therefore not
|
|
a style problem, it is the failure mode.
|
|
|
|
Imported by name from `bin/`, which is `sys.path[0]` for a script run as
|
|
`bin/<tool>`.
|
|
"""
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
DEFAULT_STORE = "~/.cache/inference-arbitrage/wiki"
|
|
|
|
# spec §5.2.7's filing threshold, mirroring bin/boundary-classify — a candidate
|
|
# must clear the same bar to be called fixed, or stable, that it had to clear to
|
|
# be filed. Changing one without the others would make those words mean
|
|
# something the filing gate never meant.
|
|
MIN_SHARE_OF_SPEND = 0.02
|
|
MIN_INVOCATIONS = 3
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# store
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def store_root(store):
|
|
return os.path.expanduser(store)
|
|
|
|
|
|
def snapshots_path(store, target):
|
|
return os.path.join(store_root(store), "Data", target, "snapshots.jsonl")
|
|
|
|
|
|
def audits_dir(store, target):
|
|
return os.path.join(store_root(store), "Audits", target)
|
|
|
|
|
|
def load_snapshots(store, target):
|
|
path = snapshots_path(store, target)
|
|
if not os.path.exists(path):
|
|
return []
|
|
out = []
|
|
with open(path) as fh:
|
|
for lineno, line in enumerate(fh, 1):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
out.append(json.loads(line))
|
|
except json.JSONDecodeError as exc:
|
|
sys.exit(f"error: {path}:{lineno}: corrupt snapshot record: {exc}")
|
|
out.sort(key=lambda s: s.get("run_id", ""))
|
|
return out
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# identity (FR-5.2)
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def signature_hash(sig):
|
|
"""Hash of a normalized tool-sequence signature. `sig` is the list of
|
|
masked signatures offload-scan already emits, so normalization happened
|
|
upstream; joining them is all that is left."""
|
|
return hashlib.sha256("\n".join(sig).encode()).hexdigest()[:16]
|
|
|
|
|
|
def identity(cand):
|
|
"""FR-5.2 resolution order. Issue number first: it is the only identifier
|
|
a human also uses, and it survives a refactor that moves the code."""
|
|
if cand.get("issue"):
|
|
return ("issue", cand["issue"])
|
|
if cand.get("signature_hash"):
|
|
return ("signature", cand["signature_hash"])
|
|
return ("slug", cand.get("candidate_id"))
|
|
|
|
|
|
def build_index(cands):
|
|
idx = {"issue": {}, "signature": {}, "slug": {}}
|
|
for c in cands:
|
|
if c.get("issue"):
|
|
idx["issue"].setdefault(c["issue"], c)
|
|
if c.get("signature_hash"):
|
|
idx["signature"].setdefault(c["signature_hash"], c)
|
|
idx["slug"].setdefault(c.get("candidate_id"), c)
|
|
return idx
|
|
|
|
|
|
def find_match(cand, idx):
|
|
for kind, key in (
|
|
("issue", cand.get("issue")),
|
|
("signature", cand.get("signature_hash")),
|
|
("slug", cand.get("candidate_id")),
|
|
):
|
|
if key and key in idx[kind]:
|
|
return idx[kind][key], kind
|
|
return None, None
|
|
|
|
|
|
def tiered(snap):
|
|
"""Both tiers of a snapshot. A candidate that was a boundary question last
|
|
window and a filing candidate this one is the same candidate, so identity
|
|
matching must see both piles."""
|
|
for c in snap.get("candidates", []):
|
|
yield c, "candidate"
|
|
for c in snap.get("boundary_questions", []):
|
|
yield c, "boundary-question"
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# signatures derived from a scan
|
|
# --------------------------------------------------------------------------
|
|
|
|
|
|
def scan_rows(scan):
|
|
for row in scan.get("skills", []):
|
|
yield row.get("skill"), row
|
|
for row in scan.get("agents", []):
|
|
yield row.get("agent"), row
|
|
|
|
|
|
def candidate_signature(cand, scan, explicit):
|
|
"""Signature resolution, most trustworthy first: an explicitly supplied
|
|
tool sequence, then the dominant recurring n-gram of the candidate's own
|
|
skill, then the slug. The source is recorded because a slug-derived
|
|
identity cannot detect drift — it IS the thing drift is measured against."""
|
|
cid = cand.get("candidate_id")
|
|
if explicit.get(cid):
|
|
return list(explicit[cid]), "explicit"
|
|
for name, row in scan_rows(scan):
|
|
if name == cand.get("skill") and row.get("ngrams"):
|
|
return list(row["ngrams"][0]["sig"]), "scan-ngram"
|
|
return [f"slug:{cid}"], "slug"
|