diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index d3aee35..05f43a1 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "inference-arbitrage", - "version": "0.6.3", + "version": "0.7.0", "description": "Audits a Claude Code plugin's definitions and usage transcripts to find steps that should be a deterministic script instead of raw LLM inference, and files the well-evidenced ones as issues on the target repo.", "author": { "name": "oleks" diff --git a/bin/audit-snapshot b/bin/audit-snapshot index 6634ecb..6eac8b4 100755 --- a/bin/audit-snapshot +++ b/bin/audit-snapshot @@ -180,9 +180,14 @@ def build_snapshot(target, inventory, scan, classified, issue_map, explicit, run # the filing pass — `filing: unavailable` and why (FR-6.4) — which this # script cannot see and which have to reach the wiki page, not just the # user's terminal. + # `insufficient-window` is in this list, not in `candidates` or + # `boundary_questions`: an FR-4.5 deferral is neither a filed candidate nor + # a question put to a human, and the wider snapshot it was deferred against + # remains the standing comparison point. Recording it as a note keeps the + # deferral visible on the wiki page instead of silently dropping the row. notes = [ f"{c['candidate_id']}: {c['verdict']} — {c['reasons'][-1] if c.get('reasons') else ''}" - for c in rows if c.get("verdict") in ("no-offload", "drift-note") + for c in rows if c.get("verdict") in ("no-offload", "drift-note", "insufficient-window") ] + list(extra_notes) return { diff --git a/bin/stability-classify b/bin/stability-classify index 1a5482f..492b2ab 100755 --- a/bin/stability-classify +++ b/bin/stability-classify @@ -40,6 +40,66 @@ window and 5% the next is stable — same label, same side of the threshold — that variation is what `audit-snapshot diff` exists to report as a trend. This gate fires only when the *description a human receives* changes. +A NARROW WINDOW IS NOT EVIDENCE OF INSTABILITY EITHER +----------------------------------------------------- +The gate as first written treated every flip as symmetric, and that produced a +false downgrade on the very candidate that motivated it +(kotkan/claude-plugin-inference-arbitrage#15). Re-measuring +`anxious/agent-wip/release-policy-derivation` across four real windows with the +same script showed a monotonic accumulation, not a flip: + + 7 days -> 0 invocations, 0.00% of spend -> thin + 14 days -> 2 invocations, 0.72% -> thin + 27 days -> 12 invocations, 11.73% -> measured + 60 days -> 15 invocations, 13.87% -> measured + +The candidate runs about once every two days. A 7-day window did not observe an +unstable candidate; it failed to observe a real one. The wider the window, the +higher the reading — the 60-day pass had every opportunity to dilute the share +back down and instead raised it. Calling that "the confidence label moves with +the calendar" gets the direction of the artifact backwards: the *narrow* window +was the artifact, and the gate punished the candidate for it. + +So a weak reading only gets to overturn a `measured` one when it was taken over +a comparably wide window. Concretely, all three must hold: + + * the prior reading is `measured`, + * the current reading is `thin` or `unmeasured`, + * the current window is narrower than WINDOW_PARITY_RATIO of the prior + window's observed width (`until - since`), + +and then the comparison is not read as instability at all. The candidate takes +a third verdict, `insufficient-window`: it is neither filed nor downgraded, no +self-report is emitted, and it is carried forward — the wider prior snapshot +stays the standing comparison point, so the next audit run over an adequate +window compares against it rather than against this window's under-sample. + +The parity is relative, not an absolute day count, on purpose. "At least 21 +days" would be a guess about candidate frequency dressed up as a threshold — +this candidate needs ~27 days, a once-a-week step would need more, and a +once-an-hour step is fully measured in one. What the gate can know without +guessing is that a window materially narrower than the one that produced the +`measured` reading cannot fairly contradict it. WINDOW_PARITY_RATIO is 0.9 +rather than 1.0 because window bounds are observed session spans, not requested +ranges, so two nominally identical windows differ by hours. + +Three things stay downgrades, unchanged: + + 1. the reverse direction (`thin` prior, `measured` current) — a wider or + equally wide current window that now sees the candidate genuinely + contradicts the earlier reading, and the earlier reading is the weak one; + 2. any flip between comparably wide windows; + 3. a threshold-side flip in which BOTH windows read `measured`. If the + current window still reached `measured`, it observed the candidate often + enough; a share that then crosses the filing threshold is a claim about + relative spend, not a sampling failure, and narrowness does not excuse it. + +Width is read off the snapshot's own `window.since`/`window.until`. When either +window does not carry both bounds the widths are not comparable, and the gate +does NOT quietly excuse the flip: it downgrades as before and says so in the +finding's `window_parity` block, so an unreadable window is visible rather than +a silent exemption. + NO PRIOR WINDOW MEANS UNCHECKED, NOT UNSTABLE --------------------------------------------- A first-ever audit of a target, or a candidate seen for the first time, has @@ -80,6 +140,7 @@ of `classified.json`. """ import argparse +import datetime import json import sys @@ -92,6 +153,21 @@ GATE = "FR-4.5" UNCHECKED_NO_HISTORY = "unchecked — no prior window to compare" UNCHECKED_NO_MATCH = "unchecked — no prior window recorded this candidate" +# The verdict a comparison takes when it is too lopsided to read either way. +# Distinct from `boundary-question` on purpose: a boundary question is something +# a human is asked to judge, and this is the auditor declining to make a claim. +INSUFFICIENT_WINDOW = "insufficient-window" + +# Readings that mean "this window did not observe the candidate enough", as +# opposed to `measured`, which means it did. +WEAK_STRENGTHS = ("thin", "unmeasured") + +# How wide the current window must be, as a fraction of the prior window that +# produced the `measured` reading, before a weak reading may overturn it. Not +# 1.0: window bounds are observed session spans, so two nominally equal windows +# differ by hours and an exact-parity rule would fire on clock noise. +WINDOW_PARITY_RATIO = 0.9 + def keyed(cand, scan, explicit, issue_map): """The identity fields FR-5.2 matches on, computed for a candidate that is @@ -149,10 +225,78 @@ def flips(prior, curr_strength, curr_share): return out +def window_days(window): + """Observed width of a window in days, or None when it cannot be read. + + None is a real answer, not a default: `window_parity` reports it as + "not comparable" and lets the flip stand, rather than treating an + unreadable window as narrow (which would excuse every flip) or as wide + (which would hide that the check could not run).""" + window = window or {} + bounds = [] + for key in ("since", "until"): + raw = window.get(key) + if not isinstance(raw, str): + return None + try: + bounds.append(datetime.datetime.fromisoformat(raw)) + except ValueError: + return None + since, until = bounds + # One bound tz-aware and the other naive cannot be subtracted, and guessing + # a zone for the naive one would invent a width. + if (since.tzinfo is None) != (until.tzinfo is None): + return None + return (until - since).total_seconds() / 86400.0 + + +def window_parity(prior_window, curr_window): + """Whether the current window is wide enough to contradict the prior one. + + Always returned, whether or not it changes the outcome, so the finding + records the widths it was decided on.""" + prior_days = window_days(prior_window) + curr_days = window_days(curr_window) + out = { + "parity_ratio": WINDOW_PARITY_RATIO, + "prior_days": round(prior_days, 2) if prior_days is not None else None, + "current_days": round(curr_days, 2) if curr_days is not None else None, + "required_days": (round(prior_days * WINDOW_PARITY_RATIO, 2) + if prior_days is not None else None), + } + if prior_days is None or curr_days is None: + out["comparable"] = None + out["why"] = ("window width is unreadable — one of the two windows does not carry " + "both `since` and `until` — so narrowness cannot be established and " + "the flip is not excused; this is recorded rather than defaulted") + return out + out["comparable"] = curr_days >= out["required_days"] + out["why"] = ( + f"this window spans {out['current_days']}d against the prior window's " + f"{out['prior_days']}d; a weak reading needs at least " + f"{out['required_days']}d ({WINDOW_PARITY_RATIO:.0%} of the prior width) to " + "contradict a `measured` one" + if not out["comparable"] else + f"this window spans {out['current_days']}d against the prior window's " + f"{out['prior_days']}d, at or above the {out['required_days']}d parity floor, so " + "the two readings are comparably sampled and the flip is real") + return out + + +def deferrable(prior_strength, curr_strength): + """The only direction narrowness can excuse: a wide `measured` reading that a + narrow window failed to observe. The reverse (`thin` prior, `measured` now) + is a genuine contradiction in which the *earlier* reading is the weak one, + and a `measured` -> `measured` threshold crossing means both windows saw the + candidate, so neither is excused here.""" + return prior_strength == "measured" and curr_strength in WEAK_STRENGTHS + + def gate(doc, history, scan, explicit, issue_map): target = doc.get("target") window = scan.get("window") or {} findings = [] + deferrals = [] checked = 0 for cand in doc.get("candidates", []): @@ -180,6 +324,54 @@ def gate(doc, history, scan, explicit, issue_map): f"{curr_strength}, {curr_share:.2%} of audited spend either side") continue + parity = window_parity(snap.get("window"), window) + if deferrable(prior.get("measurement_strength"), curr_strength) \ + and parity["comparable"] is False: + # Not filed, not downgraded, not self-reported: this comparison + # cannot be made fairly, and saying so is the honest third answer. + cand["verdict"] = INSUFFICIENT_WINDOW + cand["stability"] = ( + f"insufficient-window against {snap.get('run_id')} (matched by {by}): " + f"{parity['current_days']}d window read {curr_strength} against a " + f"{parity['prior_days']}d window's measured " + f"{((prior.get('measurement') or {}).get('share_of_plugin') or 0):.2%} — " + "too narrow to compare fairly") + cand.setdefault("reasons", []).append( + f"DEFERRED ({GATE}): the current window ({parity['current_days']}d) is " + f"narrower than the {parity['required_days']}d parity floor set by the " + f"{parity['prior_days']}d window that produced the prior `measured` " + "reading, so this window's weak reading cannot fairly contradict it. " + "Neither filed nor downgraded to a boundary question this run; carried " + "forward for a comparison over an adequately wide window.") + deferrals.append({ + "gate": GATE, + "candidate_id": cand.get("candidate_id"), + "target": target, + "skill": cand.get("skill"), + "matched_by": by, + "prior": { + "run_id": snap.get("run_id"), + "window": snap.get("window"), + "measurement_strength": prior.get("measurement_strength"), + "share_of_audited_spend": (prior.get("measurement") or {}).get("share_of_plugin"), + }, + "current": { + "window": window, + "measurement_strength": curr_strength, + "share_of_audited_spend": curr_share, + }, + # What WOULD have been reported as instability, kept so the + # deferral is auditable rather than an unexplained absence. + "suppressed_flips": moved, + "window_parity": parity, + "carry_forward": ( + f"re-audit over a window of at least {parity['required_days']}d and " + f"compare against {snap.get('run_id')}, which remains the standing " + "comparison point — this window contributes no reading that could " + "overturn it"), + }) + continue + cand["verdict"] = "boundary-question" cand["stability"] = f"unstable against {snap.get('run_id')} (matched by {by})" cand.setdefault("reasons", []).append( @@ -206,6 +398,10 @@ def gate(doc, history, scan, explicit, issue_map): "share_of_audited_spend": curr_share, }, "flips": moved, + # Recorded on downgrades too, so a reader can see the widths this + # was decided on and that the narrow-window carve-out was + # considered and did not apply. + "window_parity": parity, "headline": ( f"{cand.get('candidate_id')} is {prior.get('measurement_strength')} at " f"{((prior.get('measurement') or {}).get('share_of_plugin') or 0):.2%} of spend in " @@ -226,14 +422,21 @@ def gate(doc, history, scan, explicit, issue_map): summary["boundary_questions"] = sum(1 for c in rows if c.get("verdict") == "boundary-question") summary["stability_checked"] = checked summary["stability_downgraded"] = len(findings) + summary["stability_insufficient_window"] = len(deferrals) doc["stability_gate"] = { "gate": GATE, "snapshots_available": len(history), "checked": checked, "downgraded": len(findings), + "insufficient_window": len(deferrals), + "window_parity_ratio": WINDOW_PARITY_RATIO, } doc["stability_findings"] = findings + # Kept out of `stability_findings` deliberately: those are FR-6.5 + # self-reports the caller files against this plugin's repo, and a deferral + # is not a defect to report — it is a comparison that was not made. + doc["stability_deferrals"] = deferrals return doc diff --git a/design/rubric.md b/design/rubric.md index 107c099..e9f861d 100644 --- a/design/rubric.md +++ b/design/rubric.md @@ -340,7 +340,7 @@ Hence the same shape of rule as §5: > windows land on opposite sides of the filing threshold, the candidate is not > filed. -Two boundaries make this a usable rule rather than a blanket refusal. +Three boundaries make this a usable rule rather than a blanket refusal. **Movement is not instability.** A candidate at 12% one window and 8% the next is stable: the human is told the same thing, and the difference is a trend, which @@ -356,6 +356,34 @@ finding out of missing history would mean no plugin could ever be audited once, which contradicts the principle that a plugin with no measured history stays auditable on its definition alone. +**A narrow window is not instability either — and this one was learned the hard +way.** The rule above, applied symmetrically, downgraded the very candidate it +was written for. `anxious/agent-wip/release-policy-derivation` reads `thin, 0%` +over 7 days and `measured, 11.9%` over 27 — which looks like the §5b failure +until you measure two more windows: 0 → 2 → 12 → 15 invocations at 7, 14, 27 and +60 days. It runs about once every two days. The short window did not catch an +unstable candidate; it failed to observe a real one, and the wider window kept +raising the reading rather than diluting it. Punishing the candidate for that +gets the artifact backwards: the *window* was the artifact. + +So a weak reading only overturns a `measured` one when it was sampled over a +comparably wide window — at least 90% of the width that produced the `measured` +reading. Below that, the answer is neither "file" nor "boundary question" but a +third, honest one: **`insufficient-window` — cannot tell either way, come back +with a wider window.** Nothing is filed, nothing is self-reported, and the wider +snapshot stays the standing comparison point so the next adequate run compares +against it. + +The parity is deliberately relative, not "at least N days": the adequate width +is a fact about the candidate's invocation rate, which the auditor learns from +the measurement rather than knowing in advance, so an absolute floor would be a +guess about frequency dressed up as a threshold. And the carve-out is +one-directional. A `thin` *prior* against a `measured` current is still a +downgrade — there the earlier reading is the weak one — and so is a threshold +crossing where both windows read `measured`, because a window that reached +`measured` did observe the candidate; what moved was its share of spend, which +is signal, not sampling. + The instability, when it fires, is a finding **about the auditor**. The target did not change; the tool described it two different ways. So it is reported against this plugin's own repo, not the target's — the same self-application diff --git a/design/spec.md b/design/spec.md index 50dad22..379f696 100644 --- a/design/spec.md +++ b/design/spec.md @@ -237,6 +237,42 @@ fact about the target. A candidate with no prior window to compare against is marked `unchecked` and files normally; an absence of history is not evidence of instability, and manufacturing one from it would contradict FR-2.4. +**FR-4.5.1 — Minimum window width for a downgrade.** A weak reading may only +overturn a `measured` one when it was taken over a comparably wide window. Let +`width(w) = w.until - w.since` in days, read off the window each reading was +taken over (the observed session span the scan records, not the requested +range). A comparison is **not read as instability**, and takes a third verdict +`insufficient-window`, when all of the following hold: + +1. the prior reading's `measurement_strength` is `measured`; +2. the current reading's `measurement_strength` is `thin` or `unmeasured`; +3. `width(current) < 0.9 × width(prior)`. + +An `insufficient-window` candidate is **neither filed nor downgraded to a +boundary question**, emits **no FR-6.5 self-report**, and is carried forward: the +wider prior snapshot remains the standing comparison point, so the next audit +over an adequately wide window compares against it and not against this +window's under-sample. The run records the widths it decided on, the parity +floor, and the flips it suppressed. + +Three cases remain downgrades, and must not be excused by narrowness: the +reverse direction (`thin`/`unmeasured` prior, `measured` current — the earlier +reading is then the weak one); any flip between windows that satisfy the 0.9 +parity; and a threshold crossing in which **both** readings are `measured`, +since a window that reached `measured` observed the candidate often enough for +the share to be a claim about relative spend rather than a sampling failure. + +The rule is stated as a ratio against the prior window rather than an absolute +number of days because the adequate width is a property of the candidate's +invocation rate, which the auditor does not know in advance: +`anxious/agent-wip/release-policy-derivation` runs roughly once every two days +and needs ~27 days; a once-per-hour step is fully measured in one. The ratio is +0.9 rather than 1.0 because window bounds are observed session spans, so two +nominally identical windows differ by hours. When either window does not carry +both bounds, width is not computable, the flip is **not** excused, and the run +records that the check could not be made — an unreadable window must be visible, +never a silent exemption. + ### FR-5 — Snapshot accumulation and diffing **FR-5.1** Every run appends one machine-readable snapshot record to an @@ -323,7 +359,10 @@ ranked candidate table with boundary classification and value; the diff against the previous snapshot; the boundary questions posed to the user — both the low-confidence candidates and any downgraded by the FR-4.5 stability gate, the latter marked as a defect in the auditor rather than a fact about the -target; and the issues filed or updated, in full `owner/repo#num` form, +target; any candidate the gate could not judge either way (`insufficient-window`, +FR-4.5.1), stated as a window too narrow to compare rather than as a finding +about the target or the auditor; and the issues filed or updated, in full +`owner/repo#num` form, separating those on the target's tracker (FR-6.1) from those on this plugin's own (FR-6.5). @@ -488,7 +527,7 @@ is verified by hand (TASKS 7.9.5). | S1 | Audits any plugin by path or name, with no target-specific code | Run against `token-budget`, `worktree-discipline`, `anxious`, `memory` | | S2 | Correctly returns "nothing to offload" for a well-cut plugin | Run against `token-budget` → no `high`-confidence candidates | | S3 | Rediscovers a known-correct historical cut | Run against `worktree-discipline` with `bin/worktree-audit` removed from the inventory → must flag the classification step as `high` confidence | -| S4 | Mechanically refuses a candidate whose evidence is a window artifact | The two-window `anxious` fixture → the FR-4.5 gate downgrades `agent-wip/release-policy-derivation`, with no human in the loop | +| S4 | Mechanically refuses a candidate whose evidence is a window artifact | The two-window `anxious` fixture → the FR-4.5 gate refuses `agent-wip/release-policy-derivation` (as `insufficient-window`, per FR-4.5.1, since the 7-day current window cannot fairly contradict the 26-day `measured` one), with no human in the loop; a flip between comparably wide windows is still downgraded to a boundary question | | S5 | Re-running does not duplicate issues | Run twice; second run comments, does not file | | S6 | Trend across ≥2 snapshots is computable and volume-normalized | Two runs on different windows; diff reports per-invocation deltas | | S7 | Passes its own audit | FR-8.2 | @@ -505,6 +544,17 @@ that a human review caught a candidate whose evidence flipped between windows so the criterion is now that gate, run mechanically (FR-4.5), against the real two-window evidence that produced the original finding. +The *reason* S4's candidate is refused changed in v0.7.0, and the change is +itself instructive. Two further real windows (14 and 60 days, same script, same +candidate) showed 0 → 2 → 12 → 15 invocations as the window widened: monotonic +accumulation of a low-frequency step, not a label flipping about. The 7-day +window had not caught an unstable candidate, it had failed to observe a real one, +and calling that instability blamed the candidate for the auditor's window. So +FR-4.5.1 makes the refusal honest — `insufficient-window`, cannot tell either +way, compare again over a wider window — while keeping the property S4 exists to +test: nothing is filed off a window artifact, with no human in the loop. +(kotkan/claude-plugin-inference-arbitrage#15.) + --- ## 5. Signal catalog diff --git a/skills/offload-audit/SKILL.md b/skills/offload-audit/SKILL.md index 00f233b..32b877c 100644 --- a/skills/offload-audit/SKILL.md +++ b/skills/offload-audit/SKILL.md @@ -154,6 +154,19 @@ changed, or whose share crossed the 2% filing threshold, between the two windows. **Use `stable.json` for steps 5 and 6.** Never file from `classified.json` once this has run. +One comparison it deliberately refuses to make (FR-4.5.1): when the prior window +read `measured`, this window reads `thin`/`unmeasured`, **and** this window is +narrower than 90% of that prior window's width, the candidate comes back with +verdict **`insufficient-window`** and lands in `stability_deferrals`, not +`stability_findings`. A low-frequency-but-real candidate looks thin in any short +window — that is what kotkan/claude-plugin-inference-arbitrage#15 turned out to +be — so this is neither a filing nor a defect: **do not file it on the target's +repo, and do not self-report it on this plugin's repo.** Mention it in the run +summary as "window too narrow to compare" and, if you want the answer, re-audit +over at least the `carry_forward` width the deferral names. Flips between +comparably wide windows, and the reverse direction (`thin` prior → `measured` +now), still downgrade and still self-report. + If the store has no history for this target even after the wiki fetch above — a genuine first audit — every candidate comes back `stability: unchecked` and files normally. That is the FR-2.4 outcome, not a failure. diff --git a/tests/calibration/README.md b/tests/calibration/README.md index 99eb87a..4747c8c 100644 --- a/tests/calibration/README.md +++ b/tests/calibration/README.md @@ -50,7 +50,7 @@ rather than in agent prose (FR-8.1 — the plugin obeying its own thesis). | --- | --- | --- | | 1 (S2) | `token-budget` | zero `high`-confidence candidates; conclusion "nothing to offload here" | | 2 (S3) | `worktree-discipline`, masked | `sweep-worktrees` worktree-safety classification at `high`, position `llm-over-script-digest` | -| 3 (S4) | `anxious`, two windows | `release-policy-derivation` downgraded to a boundary question by the FR-4.5 stability gate, with no human in the loop | +| 3 (S4) | `anxious`, two windows | `release-policy-derivation` refused by the FR-4.5 stability gate — as `insufficient-window` (FR-4.5.1), the 7-day window being too narrow to contradict the 26-day `measured` one — with no human in the loop | ## The two-window `anxious` fixture — gate 3 @@ -68,11 +68,32 @@ actually got filed, not a case constructed to be caught: - `anxious-7d.scan.json` — the 7-day scan, for signature identity. Masked shapes and counts only (FR-3.5); no transcript content. +- `anxious-60d.scan.json` — the real 60-day scan + (`bin/offload-scan --plugin anxious --days 60`, 2026-06-26 → 2026-07-30, 176 + sessions), added while resolving + kotkan/claude-plugin-inference-arbitrage#15. Same masking rules. +- `anxious-60d.classified.json` — the same candidate's unchanged rubric grade + with the 60-day window's real measurement: 15 invocations, `offload_value` + 4,953,520 of 35,722,973 weighted tokens = 13.87% of audited spend, `measured`. + It carries only that one candidate, so no other row can be read as a 60-day + number it was not measured over. + The test excludes the 7-day run from the store so it is the run being -classified, exactly as at filing time, and asserts the gate downgrades it. The -comparison is symmetric: classifying the 30-day window against the 7-day -snapshot downgrades it too. Both windows cannot be right, which is the finding — -recorded as kotkan/claude-plugin-inference-arbitrage#11. +classified, exactly as at filing time, and asserts the gate **refuses** it. What +it does *not* assert any more is that the refusal is a downgrade. The comparison +is not symmetric, and treating it as symmetric was the bug behind +kotkan/claude-plugin-inference-arbitrage#15: measuring two further real windows +gave 0 → 2 → 12 → 15 invocations at 7, 14, 27 and 60 days, i.e. a candidate that +runs about once every two days and that a 7-day window simply never sees. So the +7-day-against-26-day pairing is now `insufficient-window` (FR-4.5.1) with no +self-report, while the 60-day-against-26-day pairing — two adequately wide +windows, `measured` both times, same side of the 2% threshold — is `stable` and +files normally. Both directions are asserted, so a regression in either shows up +as a test failure rather than as a wrong issue on someone's tracker. + +The original two-window finding, and the fact that both windows cannot be right +about a candidate they sample equally, is recorded as +kotkan/claude-plugin-inference-arbitrage#11. The candidate matches by **slug**, not signature: the 7-day scan produced no recurring n-gram for that skill, so its signature falls back to the slug and the diff --git a/tests/calibration/anxious-60d.classified.json b/tests/calibration/anxious-60d.classified.json new file mode 100644 index 0000000..bc51e23 --- /dev/null +++ b/tests/calibration/anxious-60d.classified.json @@ -0,0 +1,42 @@ +{ + "candidates": [ + { + "boundary_confidence": "high", + "candidate_id": "anxious/agent-wip/release-policy-derivation", + "category": "usage", + "determinism_tests": { + "T1": true, + "T2": true, + "T3": true, + "T4": true, + "T5": true + }, + "falsifiability_triple": { + "examples": true, + "overrule_case": true, + "signature": true + }, + "measurement_strength": "measured", + "offload_value": 4953520, + "position": "llm-over-script-digest", + "reasons": [ + "all five determinism tests pass; triple complete; escalation path defined", + "measured over the 60-day window: 15 invocations, offload_value 4,953,520 weighted tokens, 13.87% of audited spend" + ], + "share_of_audited_spend": 0.1387, + "skill": "anxious:agent-wip", + "verdict": "file" + } + ], + "rubric_version": "1.0.0", + "summary": { + "boundary_questions": 0, + "candidates": 1, + "conclusion": "1 high-confidence candidate(s)", + "drift_notes": 0, + "high_confidence": 1, + "no_offload": 0, + "to_file": 1 + }, + "target": "anxious (60-day window)" +} diff --git a/tests/calibration/anxious-60d.scan.json b/tests/calibration/anxious-60d.scan.json new file mode 100644 index 0000000..de5a4cb --- /dev/null +++ b/tests/calibration/anxious-60d.scan.json @@ -0,0 +1,335 @@ +{ + "window": { + "since": "2026-06-26T17:41:40.988000+00:00", + "until": "2026-07-30T02:30:59.111000+00:00", + "requested_since": "2026-05-31T02:30:03.118644+00:00", + "requested_until": null, + "sessions": 176 + }, + "plugin": "anxious", + "tool_versions": { + "cc_tokens_path": "/home/oleks/projects/claude-plugins/token-budget/bin/cc-tokens" + }, + "coverage": { + "attributed_turns": 5197, + "candidate_turns": 249648, + "ratio": 0.021, + "caveat": "hook-driven and unattributed agent turns excluded" + }, + "totals": { + "weighted_tokens": 35722973, + "cost_estimate_usd": 178.61, + "invocations": 700, + "mechanical_share": 0.106 + }, + "skills": [ + { + "skill": "anxious:agent-wip", + "invocations": 15, + "turns": 375, + "sidechain_turns": 0, + "weighted_tokens": 14007053, + "cost_estimate_usd": 70.04, + "share_of_plugin": 0.392, + "mtr": 0.144, + "judgment_density": 0.027, + "retry_density": 0.069, + "offload_waste": 1532536, + "read_amplification": 383.83, + "fanout_multiplier": 1.0, + "repetition_factor": 3.322, + "ngrams": [ + { + "sig": [ + "label_write(color=, method=, name=, owner=, repo=)", + "label_write(color=, method=, name=, owner=, repo=)", + "issue_write(index=, labels=, method=, owner=, repo=)" + ], + "recurrences": 5, + "invocations": 4, + "waste": 345101 + }, + { + "sig": [ + "Edit(file_path=, new_string=, old_string=, replace_all=)", + "Read(file_path=)", + "Edit(file_path=, new_string=, old_string=, replace_all=)" + ], + "recurrences": 3, + "invocations": 3, + "waste": 342125 + }, + { + "sig": [ + "Read(file_path=)", + "Read(file_path=)", + "Bash(command=grep , description=)" + ], + "recurrences": 3, + "invocations": 3, + "waste": 302043 + }, + { + "sig": [ + "Bash(command=echo , description=)", + "issue_read(index=, method=, owner=, repo=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 3, + "invocations": 3, + "waste": 299691 + }, + { + "sig": [ + "ToolSearch(max_results=, query=)", + "issue_read(index=, method=, owner=, repo=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 4, + "invocations": 4, + "waste": 258683 + } + ], + "offload_value": 4953520 + }, + { + "skill": "anxious:delivery-flow", + "invocations": 1, + "turns": 1, + "sidechain_turns": 0, + "weighted_tokens": 48152, + "cost_estimate_usd": 0.24, + "share_of_plugin": 0.001, + "mtr": 0.0, + "judgment_density": 0.0, + "retry_density": 0.0, + "offload_waste": 0, + "read_amplification": 378.0, + "fanout_multiplier": 1.0, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 0 + } + ], + "agents": [ + { + "agent": "anxious:issuer-agent", + "invocations": 608, + "turns": 4207, + "sidechain_turns": 4207, + "weighted_tokens": 13682748, + "cost_estimate_usd": 68.41, + "share_of_plugin": 0.383, + "mtr": 0.097, + "judgment_density": 0.051, + "retry_density": 0.082, + "offload_waste": 778919, + "read_amplification": 545.98, + "fanout_multiplier": 0.2, + "repetition_factor": 7.741, + "ngrams": [ + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)" + ], + "recurrences": 107, + "invocations": 47, + "waste": 874165 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)" + ], + "recurrences": 56, + "invocations": 25, + "waste": 496902 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 28, + "invocations": 26, + "waste": 318874 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)" + ], + "recurrences": 31, + "invocations": 10, + "waste": 207299 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "issue_read(index=, method=, owner=, repo=)" + ], + "recurrences": 13, + "invocations": 13, + "waste": 184334 + } + ], + "offload_value": 5722452 + }, + { + "agent": "repo-worker", + "invocations": 2, + "turns": 268, + "sidechain_turns": 268, + "weighted_tokens": 1816949, + "cost_estimate_usd": 9.08, + "share_of_plugin": 0.051, + "mtr": 0.19, + "judgment_density": 0.004, + "retry_density": 0.075, + "offload_waste": 303294, + "read_amplification": 650.6, + "fanout_multiplier": 0.66, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 302081 + }, + { + "agent": "anxious:steward-agent", + "invocations": 56, + "turns": 225, + "sidechain_turns": 225, + "weighted_tokens": 5597218, + "cost_estimate_usd": 27.99, + "share_of_plugin": 0.157, + "mtr": 0.009, + "judgment_density": 0.08, + "retry_density": 0.076, + "offload_waste": 17835, + "read_amplification": 677.33, + "fanout_multiplier": 0.34, + "repetition_factor": 7.728, + "ngrams": [ + { + "sig": [ + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)" + ], + "recurrences": 106, + "invocations": 29, + "waste": 2987563 + }, + { + "sig": [ + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)" + ], + "recurrences": 77, + "invocations": 22, + "waste": 2434027 + }, + { + "sig": [ + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)" + ], + "recurrences": 55, + "invocations": 15, + "waste": 1816278 + }, + { + "sig": [ + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)", + "Agent(description=, prompt=, run_in_background=, subagent_type=)" + ], + "recurrences": 40, + "invocations": 8, + "waste": 1196514 + }, + { + "sig": [ + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)", + "Agent(description=, prompt=, subagent_type=)" + ], + "recurrences": 5, + "invocations": 3, + "waste": 205780 + } + ], + "offload_value": 126805 + }, + { + "agent": "anxious:docs-agent", + "invocations": 10, + "turns": 98, + "sidechain_turns": 98, + "weighted_tokens": 502645, + "cost_estimate_usd": 2.51, + "share_of_plugin": 0.014, + "mtr": 0.276, + "judgment_density": 0.061, + "retry_density": 0.051, + "offload_waste": 107413, + "read_amplification": 164.08, + "fanout_multiplier": 0.34, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 100861 + }, + { + "agent": "feature-dev:code-reviewer", + "invocations": 3, + "turns": 4, + "sidechain_turns": 4, + "weighted_tokens": 26969, + "cost_estimate_usd": 0.13, + "share_of_plugin": 0.001, + "mtr": 0.75, + "judgment_density": 0.25, + "retry_density": 0.0, + "offload_waste": 13850, + "read_amplification": 797.8, + "fanout_multiplier": 0.09, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 10388 + }, + { + "agent": "cluster:gitea-agent", + "invocations": 5, + "turns": 19, + "sidechain_turns": 19, + "weighted_tokens": 41235, + "cost_estimate_usd": 0.21, + "share_of_plugin": 0.001, + "mtr": 0.105, + "judgment_density": 0.158, + "retry_density": 0.0, + "offload_waste": 4163, + "read_amplification": 299.0, + "fanout_multiplier": 0.24, + "repetition_factor": 1.0, + "ngrams": [], + "offload_value": 3505 + } + ] +} diff --git a/tests/stability.test.sh b/tests/stability.test.sh index 5e57084..e31b0ca 100755 --- a/tests/stability.test.sh +++ b/tests/stability.test.sh @@ -33,6 +33,16 @@ gatefield() { python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['stability_gate'][sys.argv[2]])" "$@" } +# scanwin +# The only field the gate reads out of a scan for the FR-4.5 parity rule is the +# window's observed span, so the fixture carries exactly that. +scanwin() { + python3 -c " +import json, sys +json.dump({'window': {'since': sys.argv[2], 'until': sys.argv[3], 'sessions': 10}}, + open(sys.argv[1], 'w'))" "$@" +} + # classified # One position-3 candidate the boundary gate already graded `file`. classified() { @@ -59,17 +69,18 @@ json.dump({ EOF } -# snapshot-store -# A prior snapshot recording the same candidate by slug identity. +# snapshot-store [since] [until] +# A prior snapshot recording the same candidate by slug identity. The window +# defaults to 30 days; the width matters for the FR-4.5 parity rule (cases j-m). prior() { local store=$1 mkdir -p "$store/Data/synth" - python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" <<'EOF' + python3 - "$store/Data/synth/snapshots.jsonl" "$2" "$3" "${4:-2026-06-01}" "${5:-2026-07-01}" <<'EOF' import json, sys -out, strength, share = sys.argv[1:] +out, strength, share, since, until = sys.argv[1:] json.dump({ "run_id": "2026-07-01T00-00-00Z", - "window": {"since": "2026-06-01", "until": "2026-07-01"}, + "window": {"since": since, "until": until}, "target": {"name": "synth", "version": "1.0.0", "skills": ["synth:skill"]}, "coverage": {"ratio": 0.1, "attributed_turns": 500}, "totals": {"weighted_tokens": 5000000}, @@ -184,12 +195,20 @@ check "not counted as checked" "$(gatefield "$tmp/e.out.json" checked)" "0" check "no finding" "$(gatefield "$tmp/e.out.json" downgraded)" "0" # -------------------------------------------------------------------------- -echo "== (f) S4 — the real anxious candidate is caught with no human in the loop ==" +echo "== (f) S4 — the real anxious 7-day window is refused, as insufficient-window ==" # The exact evidence from kotkan/claude-plugin-inference-arbitrage#11: the -# 30-day window graded `anxious/agent-wip/release-policy-derivation` measured at +# ~26-day window graded `anxious/agent-wip/release-policy-derivation` measured at # 11.94% of audited spend, the 7-day window graded the same candidate thin at # 0.00%. Both windows' snapshots are in the store; --exclude-run makes the 7-day # run the one being classified, exactly as it is at filing time. +# +# S4 still holds — the candidate is mechanically refused with no human in the +# loop — but per kotkan/claude-plugin-inference-arbitrage#15 the honest reason is +# not "unstable". Re-measuring the same candidate over 14/27/60 days gave 0 -> 2 +# -> 12 -> 15 invocations: a monotonic accumulation of a ~once-every-two-days +# step, so the 7-day window under-sampled it rather than contradicting it. A +# 7.0-day window cannot overturn a 26.1-day `measured` reading, so this is a +# deferral, not a downgrade, and nothing is self-reported against this plugin. store=$tmp/store-anxious mkdir -p "$store/Data/anxious" cp calibration/anxious-windows.snapshots.jsonl "$store/Data/anxious/snapshots.jsonl" @@ -198,25 +217,67 @@ $BIN --classified calibration/anxious-7d.classified.json \ --scan calibration/anxious-7d.scan.json \ --exclude-run 2026-07-29T01-00-00Z >"$tmp/f.out.json" check "verdict" "$(field "$tmp/f.out.json" anxious/agent-wip/release-policy-derivation verdict)" \ - "boundary-question" -check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "1" + "insufficient-window" +check "downgraded" "$(gatefield "$tmp/f.out.json" downgraded)" "0" +check "insufficient_window" "$(gatefield "$tmp/f.out.json" insufficient_window)" "1" +check "findings" "$(python3 -c "import json;print(len(json.load(open('$tmp/f.out.json'))['stability_findings']))")" "0" python3 -c " import json d=json.load(open('$tmp/f.out.json')) -f=[x for x in d['stability_findings'] +f=[x for x in d['stability_deferrals'] if x['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0] assert f['prior']['measurement_strength']=='measured', f['prior'] assert abs(f['prior']['share_of_audited_spend']-0.1194) < 1e-9, f['prior'] assert f['current']['measurement_strength']=='thin', f['current'] assert f['current']['share_of_audited_spend']==0.0, f['current'] assert f['prior']['run_id']=='2026-07-29T00-00-00Z', f['prior'] -print(' ok finding carries the real numbers: measured 11.94% vs thin 0.00%') +p=f['window_parity'] +assert p['comparable'] is False, p +assert abs(p['current_days']-7.0) < 0.05, p +assert 25.9 < p['prior_days'] < 26.2, p +assert {x['field'] for x in f['suppressed_flips']} == { + 'measurement_strength','share_of_audited_spend'}, f['suppressed_flips'] +assert 'self_report' not in f, f +print(' ok deferral carries the real widths (7.0d vs 26.1d) and the suppressed flips') " || fail=1 python3 -c " import json d=json.load(open('$tmp/f.out.json')) assert d['summary']['to_file']==0, d['summary'] -print(' ok nothing is left to file from that window') +assert d['summary']['boundary_questions']==2, d['summary'] +r=[c for c in d['candidates'] + if c['candidate_id']=='anxious/agent-wip/release-policy-derivation'][0]['reasons'][-1] +assert 'DEFERRED (FR-4.5)' in r and 'carried' in r, r +print(' ok not filed, not counted as a boundary question, reason cites the deferral') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (f2) real 60-day window against the real 27-day prior -> reports normally ==" +# The other half of kotkan/claude-plugin-inference-arbitrage#15's evidence: the +# 60-day window (15 invocations, offload_value 4,953,520 of 35,722,973 weighted +# tokens = 13.87%) against the ~27-day `measured` 11.94% prior. Two adequately +# wide windows, same label, same side of the 2% threshold -> stable, files +# normally. The parity rule must not suppress this: the fix is about a narrow +# CURRENT window, not about refusing to ever conclude anything. +# +# --exclude-run drops the 7-day snapshot so the pairing under test really is +# 27d-vs-60d; the 60d-against-7d pairing is the reverse direction, and it still +# downgrades — see (k). +$BIN --classified calibration/anxious-60d.classified.json \ + --target anxious --store "$store" \ + --scan calibration/anxious-60d.scan.json \ + --exclude-run 2026-07-29T01-00-00Z >"$tmp/f2.out.json" +check "verdict" "$(field "$tmp/f2.out.json" anxious/agent-wip/release-policy-derivation verdict)" \ + "file" +check "downgraded" "$(gatefield "$tmp/f2.out.json" downgraded)" "0" +check "insufficient_window" "$(gatefield "$tmp/f2.out.json" insufficient_window)" "0" +python3 -c " +import json +d=json.load(open('$tmp/f2.out.json')) +c=d['candidates'][0] +assert c['stability'].startswith('stable against 2026-07-29T00-00-00Z'), c['stability'] +assert abs(c['share_of_audited_spend']-0.1387) < 1e-9, c['share_of_audited_spend'] +print(' ok the 60-day reading (13.87%) is stable against the 27-day one (11.94%)') " || fail=1 # -------------------------------------------------------------------------- @@ -266,4 +327,116 @@ $BIN --classified "$tmp/i.json" --target synth --store "$store" \ check "verdict" "$(field "$tmp/i.out.json" synth/skill/step verdict)" "boundary-question" check "downgraded" "$(gatefield "$tmp/i.out.json" downgraded)" "1" +# -------------------------------------------------------------------------- +echo "== (j) narrow current window, measured wide prior -> insufficient-window ==" +# The kotkan/claude-plugin-inference-arbitrage#15 shape, synthetically: a 30-day +# window measured the candidate at 12%, a 7-day window sees nothing. 7d is below +# the 27.0d parity floor, so the weak reading does not get to overturn the +# measured one — neither filed nor downgraded, and no self-report. +classified "$tmp/j.json" thin 0.0 +prior "$tmp/store-j" measured 0.12 +scanwin "$tmp/j.scan.json" 2026-07-23 2026-07-30 +$BIN --classified "$tmp/j.json" --target synth --store "$tmp/store-j" \ + --scan "$tmp/j.scan.json" >"$tmp/j.out.json" +check "verdict" "$(field "$tmp/j.out.json" synth/skill/step verdict)" "insufficient-window" +check "downgraded" "$(gatefield "$tmp/j.out.json" downgraded)" "0" +check "insufficient_window" "$(gatefield "$tmp/j.out.json" insufficient_window)" "1" +check "self-reports" "$(python3 -c "import json;print(len(json.load(open('$tmp/j.out.json'))['stability_findings']))")" "0" +python3 -c " +import json +d=json.load(open('$tmp/j.out.json')) +f=d['stability_deferrals'][0] +p=f['window_parity'] +assert p['prior_days']==30.0 and p['current_days']==7.0 and p['required_days']==27.0, p +assert p['comparable'] is False and p['parity_ratio']==0.9, p +assert 'at least 27.0d' in f['carry_forward'], f['carry_forward'] +assert f['prior']['run_id']=='2026-07-01T00-00-00Z', f['prior'] +s=d['candidates'][0]['stability'] +assert s.startswith('insufficient-window against 2026-07-01T00-00-00Z'), s +assert d['summary']['to_file']==0 and d['summary']['boundary_questions']==0, d['summary'] +assert d['summary']['stability_insufficient_window']==1, d['summary'] +print(' ok deferral names both widths, the floor, and the window it defers against') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (k) reverse direction — thin wide prior, measured narrow current -> downgraded ==" +# The narrowness carve-out is one-directional. Here the WIDE window is the one +# that saw nothing and the narrow one measures 12%: the weak reading is the +# earlier one, the contradiction is genuine, and the human would still be told +# two different things. Downgrade, exactly as before the fix. +classified "$tmp/k.json" measured 0.12 +prior "$tmp/store-k" thin 0.0 +scanwin "$tmp/k.scan.json" 2026-07-23 2026-07-30 +$BIN --classified "$tmp/k.json" --target synth --store "$tmp/store-k" \ + --scan "$tmp/k.scan.json" >"$tmp/k.out.json" +check "verdict" "$(field "$tmp/k.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/k.out.json" downgraded)" "1" +check "insufficient_window" "$(gatefield "$tmp/k.out.json" insufficient_window)" "0" +python3 -c " +import json +d=json.load(open('$tmp/k.out.json')) +f=d['stability_findings'][0] +assert f['self_report']['repo']=='kotkan/claude-plugin-inference-arbitrage', f['self_report'] +p=f['window_parity'] +assert p['comparable'] is False, p +print(' ok still self-reported, and the parity block records the widths anyway') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (l) two comparably wide windows, genuine flip -> downgraded ==" +# 28 days against 30 is at the parity floor (27.0d), so the windows are +# comparably sampled and a measured -> thin flip is real instability. +classified "$tmp/l.json" thin 0.0 +prior "$tmp/store-l" measured 0.12 +scanwin "$tmp/l.scan.json" 2026-07-02 2026-07-30 +$BIN --classified "$tmp/l.json" --target synth --store "$tmp/store-l" \ + --scan "$tmp/l.scan.json" >"$tmp/l.out.json" +check "verdict" "$(field "$tmp/l.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/l.out.json" downgraded)" "1" +check "insufficient_window" "$(gatefield "$tmp/l.out.json" insufficient_window)" "0" +python3 -c " +import json +p=json.load(open('$tmp/l.out.json'))['stability_findings'][0]['window_parity'] +assert p['current_days']==28.0 and p['required_days']==27.0 and p['comparable'] is True, p +print(' ok 28.0d clears the 27.0d floor, so the flip stands') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (m) measured both windows, threshold crossed on a narrow window -> downgraded ==" +# A window that still reached `measured` observed the candidate often enough; a +# share that then crosses the 2% filing threshold is a claim about relative +# spend, not a sampling failure, so narrowness does not excuse it. +classified "$tmp/m.json" measured 0.01 +prior "$tmp/store-m" measured 0.12 +scanwin "$tmp/m.scan.json" 2026-07-23 2026-07-30 +$BIN --classified "$tmp/m.json" --target synth --store "$tmp/store-m" \ + --scan "$tmp/m.scan.json" >"$tmp/m.out.json" +check "verdict" "$(field "$tmp/m.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/m.out.json" downgraded)" "1" +check "insufficient_window" "$(gatefield "$tmp/m.out.json" insufficient_window)" "0" +python3 -c " +import json +f=json.load(open('$tmp/m.out.json'))['stability_findings'][0] +assert {x['field'] for x in f['flips']} == {'share_of_audited_spend'}, f['flips'] +print(' ok only the threshold flip is reported, and it is not excused') +" || fail=1 + +# -------------------------------------------------------------------------- +echo "== (n) unreadable window widths -> downgraded, and said so explicitly ==" +# No --scan means no current window bounds. Narrowness cannot be established, so +# the flip is NOT quietly excused — the finding records that the check could not +# run, rather than defaulting either way. +classified "$tmp/n.json" thin 0.0 +prior "$tmp/store-n" measured 0.12 +$BIN --classified "$tmp/n.json" --target synth --store "$tmp/store-n" >"$tmp/n.out.json" +check "verdict" "$(field "$tmp/n.out.json" synth/skill/step verdict)" "boundary-question" +check "downgraded" "$(gatefield "$tmp/n.out.json" downgraded)" "1" +python3 -c " +import json +p=json.load(open('$tmp/n.out.json'))['stability_findings'][0]['window_parity'] +assert p['comparable'] is None and p['current_days'] is None, p +assert 'unreadable' in p['why'], p +print(' ok comparability is null with an explicit reason, not a silent exemption') +" || fail=1 + exit "$fail"