From 1b3b050919da03209aa6a9a24fe12cafd53ca78b Mon Sep 17 00:00:00 2001 From: Dreamcore Date: Sun, 20 Sep 2026 15:43:28 +0300 Subject: [PATCH 1/2] judge: tell the worker's changes from dirt that was already there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found on the first real job (the investigation dataset): its tree held 19 unrelated uncommitted files — someone else's webfetch cache and an edit in progress. The judge was shown them as the evidence of this run, so at 20 seconds it reported work off track 85% / needs human 80% and the policy stopped a worker that had done nothing wrong yet. Warm-up did not save it either, because a dirty tree counted as evidence. Now the first assessment records the tree's baseline, and only files appearing after it count as this run's work. The prompt states both sets separately, so the judge can see what the worker did and what it merely inherited. Also renames the evidence field to changed_by_this_run — the old name invited exactly the reading that caused the false stop. 14 offline tests pass, including the two new ones for this case. Refs kotkan/dreamcore-mp#179 Co-Authored-By: Claude Opus 5 --- bin/foreman-claude.py | 45 +++++++++++++++++++++++++++++----- tests/test_foreman_claude.py | 47 +++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 10 deletions(-) diff --git a/bin/foreman-claude.py b/bin/foreman-claude.py index 3ac951d..b4477e5 100644 --- a/bin/foreman-claude.py +++ b/bin/foreman-claude.py @@ -321,6 +321,7 @@ class FreeModelForeman: self.free_tokens = 0 self.failures = 0 self.base_commit: str | None = None + self.baseline_dirty: frozenset[str] | None = None async def _git(self, *args: str, limit: int = 2_000) -> str: try: @@ -339,6 +340,30 @@ class FreeModelForeman: return "" return stdout.decode("utf-8", errors="replace")[-limit:] + @staticmethod + def _status_paths(git_status: str) -> frozenset[str]: + paths = set() + for line in git_status.splitlines(): + entry = line[3:].strip() if len(line) > 3 else "" + if entry: + paths.add(entry.split(" -> ")[-1]) + return frozenset(paths) + + def worker_touched(self, observation: FactoryObservation) -> list[str]: + """Files this RUN changed — not files that were already dirty when it started. + + Measured 2026-09-20 on the investigation dataset: the tree held 19 unrelated + uncommitted files (someone else's cache and edits). Counting them as the worker's + work both defeated the warm-up rule and made the judge call the run off track at 20 + seconds, because the evidence it saw had nothing to do with the job. + """ + + current = self._status_paths(observation.git_status) + if self.baseline_dirty is None: + self.baseline_dirty = current + return [] + return sorted(current - self.baseline_dirty) + async def work_since_start(self) -> dict[str, str]: """Upstream only watches the working tree; our workers commit, which empties `git diff`. @@ -380,8 +405,12 @@ class FreeModelForeman: return "" def build_prompt( - self, observation: FactoryObservation, committed: dict[str, str] | None = None + self, + observation: FactoryObservation, + committed: dict[str, str] | None = None, + touched: list[str] | None = None, ) -> str: + touched = touched if touched is not None else [] active = [ { "worker_id": worker.get("worker_id"), @@ -412,8 +441,9 @@ class FreeModelForeman: "active_workers": active, "worker_history": history, "worker_exit_status": observation.worker_exit_status, + "changed_by_this_run": touched[:40], + "pre_existing_dirty_files_NOT_the_workers": sorted(self.baseline_dirty or [])[:20], "git_status": observation.git_status, - "changed_files": observation.changed_files[:40], "git_diff_uncommitted": observation.git_diff, "work_committed_since_start": committed or {}, "latest_worker_output_tail": observation.latest_worker_output, @@ -474,7 +504,8 @@ class FreeModelForeman: async def assess(self, observation: FactoryObservation) -> FactoryAssessment: committed = await self.work_since_start() - prompt = self.build_prompt(observation, committed) + touched = self.worker_touched(observation) + prompt = self.build_prompt(observation, committed, touched) # A free model refuses more often than a paid endpoint; upstream issue #7 makes the same # point about transient Jev failures — one blip must not escalate a healthy run. last_error: ForemanModelError | None = None @@ -514,7 +545,9 @@ class FreeModelForeman: except ForemanModelError: self.failures += 1 raise - assessment, muted = self._mute_warnings_without_evidence(assessment, observation, committed) + assessment, muted = self._mute_warnings_without_evidence( + assessment, observation, committed, touched + ) assessment, held = self._require_confirmation(assessment, observation) if self.console is not None: suffix = " · warm-up: warnings muted (no evidence yet)" if muted else "" @@ -530,6 +563,7 @@ class FreeModelForeman: assessment: FactoryAssessment, observation: FactoryObservation, committed: dict[str, str], + touched: list[str] | None = None, ) -> tuple[FactoryAssessment, bool]: """Measured 2026-09-20: a small judge escalated at 25 s, before the worker touched a file. @@ -545,8 +579,7 @@ class FreeModelForeman: for worker in observation.worker_history ) has_evidence = bool( - observation.changed_files - or observation.git_status.strip() + touched or committed.get("commits_since_start", "").strip() or finished ) diff --git a/tests/test_foreman_claude.py b/tests/test_foreman_claude.py index 01f0d86..aec9518 100644 --- a/tests/test_foreman_claude.py +++ b/tests/test_foreman_claude.py @@ -161,7 +161,8 @@ def test_judge_parses_prose_wrapped_and_nested_answers(tmp_path: Path) -> None: model._call_free_worker = fake_call # type: ignore[assignment] model.work_since_start = lambda: _completed({}) # type: ignore[assignment] - assessment = asyncio.run(model.assess(observation(changed_files=["a.py"]))) + model.baseline_dirty = frozenset() + assessment = asyncio.run(model.assess(observation(git_status="?? a.py\n"))) assert assessment.implementation_complete == 0.5 assert assessment.needs_human == 0.5 assert model.free_tokens == 42 @@ -181,7 +182,8 @@ def test_judge_retries_transient_failures_then_succeeds(tmp_path: Path) -> None: model._call_free_worker = flaky # type: ignore[assignment] model.work_since_start = lambda: _completed({}) # type: ignore[assignment] - assessment = asyncio.run(model.assess(observation(changed_files=["a.py"]))) + model.baseline_dirty = frozenset() + assessment = asyncio.run(model.assess(observation(git_status="?? a.py\n"))) assert calls["n"] == 3 assert assessment.worker_stuck == 0.1 assert model.failures == 2 @@ -212,19 +214,23 @@ def test_warnings_are_muted_until_the_factory_produced_evidence(tmp_path: Path) worker_history=[{"worker_id": "worker-1", "status": "running"}], ), {}, + touched=[], ) assert applied is True assert muted.needs_human == 0.1 assert muted.worker_stuck == 0.0 and muted.agents_md_drift == 0.0 kept, applied_again = model._mute_warnings_without_evidence( - loud, observation(elapsed_factory_seconds=20.0, changed_files=["fetcher.py"]), {} + loud, + observation(elapsed_factory_seconds=20.0, changed_files=["fetcher.py"]), + {}, + touched=["fetcher.py"], ) assert applied_again is False assert kept.needs_human == 0.9 late, applied_late = model._mute_warnings_without_evidence( - loud, observation(elapsed_factory_seconds=600.0), {} + loud, observation(elapsed_factory_seconds=600.0), {}, touched=[] ) assert applied_late is False assert late.agents_md_drift == 0.9 @@ -277,3 +283,36 @@ def test_confirmation_is_off_unless_asked(tmp_path: Path) -> None: loud = _assessment(needs_human=0.95) kept, held = model._require_confirmation(loud, observation(previous_assessment=None)) assert held == [] and kept.needs_human == 0.95 + + +# ------------------------------------------------ pre-existing dirt vs worker's work + + +def test_pre_existing_dirty_files_are_not_the_workers_work(tmp_path: Path) -> None: + model = judge(tmp_path) + dirty = " M facts/other.md\n?? .cache/x.json\n" + + first = model.worker_touched(observation(git_status=dirty)) + assert first == [] # the first assessment only records the baseline + + later = model.worker_touched(observation(git_status=dirty + "?? facts/new-fact.md\n")) + assert later == ["facts/new-fact.md"] + + +def test_warmup_ignores_a_tree_that_was_already_dirty(tmp_path: Path) -> None: + model = judge(tmp_path, warmup_seconds=120.0) + loud = _assessment(needs_human=0.9, work_off_track=0.85) + dirty = observation( + elapsed_factory_seconds=20.0, + git_status=" M someone/else.md\n", + changed_files=["someone/else.md"], + worker_history=[{"worker_id": "worker-1", "status": "running"}], + ) + + muted, applied = model._mute_warnings_without_evidence(loud, dirty, {}, touched=[]) + assert applied is True and muted.work_off_track == 0.0 + + acting, applied_now = model._mute_warnings_without_evidence( + loud, dirty, {}, touched=["facts/new-fact.md"] + ) + assert applied_now is False and acting.work_off_track == 0.85 -- 2.54.0 From 4cc6a3fb7aedd7d0f1461650b696629431e2d3ba Mon Sep 17 00:00:00 2001 From: "Claude Code (Gitea Agent)" Date: Sun, 20 Sep 2026 17:45:50 +0300 Subject: [PATCH 2/2] =?UTF-8?q?0.2.2:=20=D1=81=D1=83=D0=B4=D1=8C=D1=8F=20?= =?UTF-8?q?=D0=BE=D1=82=D0=BB=D0=B8=D1=87=D0=B0=D0=B5=D1=82=20=D0=BF=D1=80?= =?UTF-8?q?=D0=B0=D0=B2=D0=BA=D0=B8=20=D0=B2=D0=BE=D1=80=D0=BA=D0=B5=D1=80?= =?UTF-8?q?=D0=B0=20=D0=BE=D1=82=20=D0=B3=D1=80=D1=8F=D0=B7=D0=B8,=20?= =?UTF-8?q?=D0=BA=D0=BE=D1=82=D0=BE=D1=80=D0=B0=D1=8F=20=D0=B1=D1=8B=D0=BB?= =?UTF-8?q?=D0=B0=20=D0=B4=D0=BE=20=D0=BD=D0=B5=D0=B3=D0=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Коммит нашёлся неотправленным в старом чекауте ~/pro/claude-plugins-root/foreman-supervisor (локальный c831b1c, в origin его не было — версии 0.2.0/0.2.1 ушли из другого чекаута мимо него). Перенесён сюда cherry-pick'ом, чтобы знание не потерялось вместе с тем каталогом. Суть правки автора: на первом реальном задании дерево держало 19 чужих незакоммиченных файлов, судья принял их за свидетельства этого прогона и на 20-й секунде дал off track 85 % / needs human 80 % — политика остановила воркера, который ещё ничего не сделал. Прогрев не спасал: грязное дерево считалось свидетельством. Прогон: 19 passed. --- .claude-plugin/plugin.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 5531520..bb91f1d 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "foreman-supervisor", - "version": "0.2.1", + "version": "0.2.2", "description": "Semantic supervision for our own Claude agents. Wraps thruwire/foreman (MIT) — a runtime that watches a coding agent with a bounded observation (job, git status, diff, output tails, repository instructions), scores ten independent yes/no questions, and lets a deterministic Python policy decide continue / stop / retry / verify / finish / escalate. This plugin replaces both of upstream's paid dependencies: the worker is `claude -p` (Worker protocol, steer() returns False so the policy degrades to stop/retry, per upstream's own docs/workers.md), and the judge is the free-model gateway (free-worker.sh) instead of TypeSafe Jev, so a supervised run costs no paid tokens. Adds two things measured here rather than assumed: the observation carries commits since the run's base commit (our agents commit, which empties `git diff` exactly when the worker does the right thing), and warning dimensions stay muted until the factory has produced evidence (a small judge escalated a healthy run at 25 seconds, before the worker touched a file). Ships foreman-calibrate.py, which replays thresholds over the runs' own JSONL timelines — the calibration upstream says it has not done. v0.2.0 adds bin/foreman-attach.py: supervision of a background worker someone ELSE already started (worker.sh / `claude agents`), which upstream cannot do because its runtime launches the worker itself. The observation is rebuilt from traces the worker already leaves — its state.json, the commits in its worktree, the tail of its transcript — so nothing is injected into the worker. Measured here on a live worker: warning dimensions read stuck 0.80 / off-track 0.80 at 362 s and 0.05 / 0.10 at 766 s on the same run with no commit in between, which is why warmup and confirmation matter more than threshold tuning. Also records that upstream's policy gates STEER_WORKER on `codex_backend == \"app-server\"` — the name of another backend rather than whether the worker accepts direction — so a steerable third-party worker can only ever be killed; the adapter works around it by printing the steering message for the lead to deliver.", "author": { "name": "oleks", -- 2.54.0