Phase 7 acceptance: two FR-5.4/S3 fidelity fixes found by running the real pipeline
S1-S3 and S5-S8 pass; S4 is prepared but deliberately unfiled pending user sign-off (tasks.md 7.4 stays unchecked). Two bugs surfaced by running the composed pipeline against real data rather than synthetic fixtures, both in load-bearing places: audit-snapshot: cost_per_invocation folded to 0 when a candidate had zero invocations in a window, so a step that simply did not run reported -100% and direction() called it `shrunk`. That is the quiet-window artefact FR-5.4 exists to prevent, one level up from absolute tokens, and it contradicted direction()'s own docstring. It is now None, metric_delta marks the pair `unmeasured`, direction returns a new `unmeasured` status, and the pages render "not run". tests/calibration/mask-worktree-audit.py: recomputed script_loc from scripts alone, silently dropping all 1051 LOC of worktree-discipline's hooks. Masking one 272-LOC script appeared to remove 1323, taking script_coverage 0.353 -> 0.083 instead of 0.297, which let S3 pass partly on an artefact of the mask. The mask now matches build_aggregate. S3 still flags the known cut `high` / llm-over-script-digest on the corrected, harder input. Both fixes carry regression tests; suite is 156 assertions, exit 0.
This commit is contained in:
Oleks
+34
-5
@@ -61,7 +61,10 @@ MIN_INVOCATIONS = 3
|
||||
NOISE_BAND = 0.10
|
||||
|
||||
STATUSES = ("new", "persisting", "grown", "shrunk", "resolved",
|
||||
"signature-drifted", "stale", "claimed-fixed-unconfirmed")
|
||||
"signature-drifted", "stale", "claimed-fixed-unconfirmed",
|
||||
# the step ran in one window and not the other: no continuous
|
||||
# series, so no direction (FR-5.4)
|
||||
"unmeasured")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -229,7 +232,12 @@ def snapshot_candidate(cand, scan, explicit, issue_map):
|
||||
"offload_waste": row.get("offload_waste", 0),
|
||||
"offload_value": value,
|
||||
"share_of_plugin": cand.get("share_of_audited_spend", 0.0),
|
||||
"cost_per_invocation": int(value / invocations) if invocations else 0,
|
||||
# None, never 0, when the step did not run: a window in which a
|
||||
# candidate was never invoked has no cost-per-invocation to report,
|
||||
# and folding that to 0 makes "did not run" indistinguishable from
|
||||
# "improved to zero" — the quiet-window artefact FR-5.4 exists to
|
||||
# prevent, one level up from absolute tokens.
|
||||
"cost_per_invocation": int(value / invocations) if invocations else None,
|
||||
},
|
||||
"measurement_strength": cand.get("measurement_strength"),
|
||||
"digest_schema": cand.get("digest_schema"),
|
||||
@@ -304,8 +312,17 @@ def rel(before, after):
|
||||
|
||||
|
||||
def metric_delta(prev_m, curr_m, key):
|
||||
# A missing measurement dict means the candidate was absent from that
|
||||
# snapshot entirely (new, or departed) and 0 is the right base. An explicit
|
||||
# None means the candidate was present but not invoked — that is an
|
||||
# absence of measurement and must not be folded to a number.
|
||||
a = (prev_m or {}).get(key, 0)
|
||||
b = (curr_m or {}).get(key, 0)
|
||||
if a is None or b is None:
|
||||
# At least one window has no measurement for this metric. There is no
|
||||
# delta to take against an absence, and inventing one is how a window
|
||||
# the step sat out becomes a reported improvement.
|
||||
return {"from": a, "to": b, "delta": None, "rel": None, "unmeasured": True}
|
||||
return {"from": a, "to": b, "delta": b - a, "rel": rel(a, b)}
|
||||
|
||||
|
||||
@@ -313,7 +330,13 @@ def direction(primary, secondary):
|
||||
"""Volume-normalized verdict (FR-5.4). cost-per-invocation leads: it is the
|
||||
metric a quiet week cannot move. share-of-spend only breaks the tie, and
|
||||
absolute tokens never vote."""
|
||||
if primary.get("unmeasured"):
|
||||
# The step was not invoked in one of the two windows, so the series is
|
||||
# not continuous and no direction can be read from it.
|
||||
return "unmeasured"
|
||||
for m in (primary, secondary):
|
||||
if m.get("unmeasured"):
|
||||
continue
|
||||
r = m["rel"]
|
||||
if r is None:
|
||||
# Base of zero: the ratio is undefined, so the sign of the move is
|
||||
@@ -479,6 +502,11 @@ def fmt_rel(r):
|
||||
return f"{r:+.0%}"
|
||||
|
||||
|
||||
def fmt_cpi(v):
|
||||
"""`not run`, never 0 — the step had no invocations in that window."""
|
||||
return "not run" if v is None else f"{v:,}"
|
||||
|
||||
|
||||
def candidate_table(rows):
|
||||
if not rows:
|
||||
return "— none —\n"
|
||||
@@ -489,7 +517,7 @@ def candidate_table(rows):
|
||||
share = r["primary"]["share_of_plugin"]
|
||||
out.append(
|
||||
f"| `{r['candidate_id']}` | {r['status']} | "
|
||||
f"{cpi['from']:,} → {cpi['to']:,} ({fmt_rel(cpi['rel'])}) | "
|
||||
f"{fmt_cpi(cpi['from'])} → {fmt_cpi(cpi['to'])} ({fmt_rel(cpi['rel'])}) | "
|
||||
f"{share['from']:.2%} → {share['to']:.2%} ({fmt_rel(share['rel'])}) | "
|
||||
f"{r['issue'] or '—'} |")
|
||||
return "\n".join(out) + "\n"
|
||||
@@ -521,7 +549,8 @@ def render_run(snap, diffdoc):
|
||||
"",
|
||||
f"- Skill: `{c['skill']}` — position {c['position']}, confidence {c['boundary_confidence']}",
|
||||
f"- {m['offload_value']:,} weighted tokens of offload value over {m['invocations']} "
|
||||
f"invocations ({m['cost_per_invocation']:,}/invocation, {m['share_of_plugin']:.2%} of audited spend)",
|
||||
f"invocations ({fmt_cpi(m['cost_per_invocation'])}/invocation, "
|
||||
f"{m['share_of_plugin']:.2%} of audited spend)",
|
||||
f"- Evidence: {c.get('measurement_strength')}; signature `{c['signature_hash']}` "
|
||||
f"({c.get('signature_source')})",
|
||||
f"- Issue: {c.get('issue') or 'not yet filed'}",
|
||||
@@ -592,7 +621,7 @@ def render_latest(snap, diffdoc, history):
|
||||
for c in snap.get("candidates", []):
|
||||
m = c["measurement"]
|
||||
lines.append(f"| `{c['candidate_id']}` | `{c['skill']}` | {m['share_of_plugin']:.2%} | "
|
||||
f"{m['cost_per_invocation']:,} | {c.get('issue') or '—'} |")
|
||||
f"{fmt_cpi(m['cost_per_invocation'])} | {c.get('issue') or '—'} |")
|
||||
if not snap.get("candidates"):
|
||||
lines.append("| — none — | | | | |")
|
||||
lines += ["", "## Run history", "", "| Run | Target version | Coverage | Candidates |",
|
||||
|
||||
+7
-7
@@ -136,17 +136,17 @@ These block design, not just code. Answers go into `Methodology/Calibration.md`.
|
||||
|
||||
## Phase 7 — Acceptance
|
||||
|
||||
- [ ] **7.1** S1 — audits four different plugins with no target-specific code.
|
||||
- [ ] **7.2** S2 — `token-budget` → "nothing to offload."
|
||||
- [ ] **7.3** S3 — masked `worktree-discipline` → rediscovers the known cut.
|
||||
- [x] **7.1** S1 — audits four different plugins with no target-specific code.
|
||||
- [x] **7.2** S2 — `token-budget` → "nothing to offload."
|
||||
- [x] **7.3** S3 — masked `worktree-discipline` → rediscovers the known cut.
|
||||
- [ ] **7.4** S4 — a real candidate in `anxious` that the user agrees is genuine.
|
||||
**User sign-off required before the first issue is filed to a real repo.**
|
||||
- [ ] **7.5** S5 — double-run files no duplicate.
|
||||
- [ ] **7.6** S6 — two-snapshot trend is computable and volume-normalized.
|
||||
- [ ] **7.7** S7 — **audit `inference-arbitrage` with `inference-arbitrage`**
|
||||
- [x] **7.5** S5 — double-run files no duplicate.
|
||||
- [x] **7.6** S6 — two-snapshot trend is computable and volume-normalized.
|
||||
- [x] **7.7** S7 — **audit `inference-arbitrage` with `inference-arbitrage`**
|
||||
(FR-8.2). Any parsing or aggregation found living in skill prose is a bug
|
||||
to fix before release.
|
||||
- [ ] **7.8** S8 — confirm the run stays inside emmett's constraints.
|
||||
- [x] **7.8** S8 — confirm the run stays inside emmett's constraints.
|
||||
|
||||
## Phase 8 — Publish
|
||||
|
||||
|
||||
@@ -21,8 +21,10 @@ The mask never touches the `worktree-discipline` checkout. It removes
|
||||
`script_coverage`, and drops the one command block that invokes it — a skill
|
||||
cannot prescribe a script that does not exist.
|
||||
|
||||
Mask effect: `script_loc` 1730 → 407, `script_coverage` 0.353 → 0.083. What is
|
||||
left in `sweep-worktrees` is the raw git plumbing the skill prescribed:
|
||||
Mask effect: `script_loc` 1730 → 1458, `script_coverage` 0.353 → 0.297. The drop
|
||||
is deliberately small — `worktree-discipline` keeps its hooks and its two other
|
||||
scripts, so S3 cannot be passed by noticing a script-starved plugin. What is left
|
||||
in `sweep-worktrees` is the raw git plumbing the skill prescribed:
|
||||
|
||||
```text
|
||||
git -C "$repo" fetch --all # ALWAYS first
|
||||
|
||||
@@ -19,7 +19,12 @@ def main():
|
||||
before = d["aggregate"]["script_loc"]
|
||||
|
||||
d["scripts"] = [s for s in d["scripts"] if not s["file"].endswith(MASKED)]
|
||||
d["aggregate"]["script_loc"] = sum(s["loc"] for s in d["scripts"])
|
||||
# Must match build_aggregate's definition, which counts hooks/ as shipped
|
||||
# executable volume too — recomputing from scripts alone silently deletes
|
||||
# every hook and overstates how script-starved the masked plugin is.
|
||||
d["aggregate"]["script_loc"] = sum(s["loc"] for s in d["scripts"]) + sum(
|
||||
h["loc"] for h in d.get("hooks", [])
|
||||
)
|
||||
body = d["aggregate"]["skill_body_words"] + d["aggregate"]["agent_body_words"]
|
||||
d["aggregate"]["script_coverage"] = (
|
||||
round(d["aggregate"]["script_loc"] / body, 3) if body else 0.0
|
||||
|
||||
@@ -119,5 +119,15 @@ check "worktree-audit absent" \
|
||||
check "its command block is gone too" \
|
||||
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(len(d['_mask']['removed_command_blocks']))")" \
|
||||
"1"
|
||||
# The mask must recompute script_loc the same way build_aggregate does, hooks
|
||||
# included; recomputing from scripts alone deletes every hook and makes the
|
||||
# masked plugin look far more script-starved than it is, which would let S3
|
||||
# pass on an artefact of the mask instead of on the missing script.
|
||||
check "script_loc still counts hooks" \
|
||||
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));print(d['aggregate']['script_loc'] == sum(s['loc'] for s in d['scripts']) + sum(h['loc'] for h in d['hooks']))")" \
|
||||
"True"
|
||||
check "only worktree-audit's LOC was removed" \
|
||||
"$(python3 -c "import json;d=json.load(open('$tmp/masked.json'));m=d['_mask'];print(m['script_loc_before'] - m['script_loc_after'] < 500)")" \
|
||||
"True"
|
||||
|
||||
exit "$fail"
|
||||
|
||||
@@ -283,6 +283,40 @@ $BIN --store "$store3" import --target synth --snapshots "$store/Data/synth/snap
|
||||
check "re-import skipped" "$(jq_path "$tmp/i2.json" skipped)" "2"
|
||||
check "still two runs" "$(wc -l <"$store3/Data/synth/snapshots.jsonl")" "2"
|
||||
|
||||
echo "== a window the candidate sat out is 'unmeasured', never 'shrunk' (FR-5.4) =="
|
||||
# The quiet-window artefact one level up from absolute tokens: `steady` runs 20
|
||||
# times in run 1 and 0 times in run 5. Folding cost-per-invocation to 0 on a
|
||||
# zero denominator would report -100% and read as a large improvement, when in
|
||||
# fact nothing was observed at all.
|
||||
store5=$tmp/store5
|
||||
inventory "$tmp/inv5.json" 1.0.0 steady
|
||||
scan "$tmp/scan5a.json" 10000000 "steady:20:2000000:list_issues,issue_read"
|
||||
classified "$tmp/cls5a.json" "synth/steady/a:steady:2000000:0.20:file"
|
||||
$BIN --store "$store5" write --target synth --run-id 2026-07-01T00:00:00Z \
|
||||
--from "$tmp/inv5.json" "$tmp/scan5a.json" --classified "$tmp/cls5a.json" >/dev/null
|
||||
# same signature, but zero invocations this window
|
||||
scan "$tmp/scan5b.json" 8000000 "steady:0:0:list_issues,issue_read"
|
||||
classified "$tmp/cls5b.json" "synth/steady/a:steady:0:0.0:file"
|
||||
$BIN --store "$store5" write --target synth --run-id 2026-07-08T00:00:00Z \
|
||||
--from "$tmp/inv5.json" "$tmp/scan5b.json" --classified "$tmp/cls5b.json" >/dev/null
|
||||
$BIN --store "$store5" diff --target synth >"$tmp/d5.json"
|
||||
check "status is unmeasured, not shrunk" "$(status "$tmp/d5.json" synth/steady/a)" "unmeasured"
|
||||
check "cost/invocation is null, not 0" \
|
||||
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation to)" "None"
|
||||
check "no fabricated rel" \
|
||||
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation rel)" "None"
|
||||
check "flagged unmeasured" \
|
||||
"$(metric "$tmp/d5.json" synth/steady/a primary cost_per_invocation unmeasured)" "True"
|
||||
check "counted in the summary" "$(jq_path "$tmp/d5.json" summary.unmeasured)" "1"
|
||||
check "not counted as shrunk" "$(jq_path "$tmp/d5.json" summary.shrunk)" "0"
|
||||
if grep -qF "not run" "$store5/Audits/synth/Latest.md" 2>/dev/null ||
|
||||
grep -qF "not run" "$store5/Audits/synth/2026-07-08.md"; then
|
||||
echo " ok page renders 'not run' rather than 0"
|
||||
else
|
||||
echo " FAIL page rendered a zero cost/invocation for a window with no invocations" >&2
|
||||
fail=1
|
||||
fi
|
||||
|
||||
echo "== a re-run of the same run_id is refused =="
|
||||
if $BIN --store "$store" write --target synth --run-id 2026-07-08T00:00:00Z \
|
||||
--from "$tmp/inv2.json" "$tmp/scan2.json" --classified "$tmp/cls2.json" >/dev/null 2>&1; then
|
||||
|
||||
Reference in New Issue
Block a user