0.2.3: судить дерево, ГДЕ воркер работает, а не откуда его запустили (#3) #4
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "foreman-supervisor",
|
"name": "foreman-supervisor",
|
||||||
"version": "0.2.2",
|
"version": "0.2.3",
|
||||||
"description": "Semantic supervision for our own Claude agents. Wraps thruwire/foreman (MIT) — a runtime that watches a coding agent with a bounded observation (job, git status, diff, output tails, repository instructions), scores ten independent yes/no questions, and lets a deterministic Python policy decide continue / stop / retry / verify / finish / escalate. This plugin replaces both of upstream's paid dependencies: the worker is `claude -p` (Worker protocol, steer() returns False so the policy degrades to stop/retry, per upstream's own docs/workers.md), and the judge is the free-model gateway (free-worker.sh) instead of TypeSafe Jev, so a supervised run costs no paid tokens. Adds two things measured here rather than assumed: the observation carries commits since the run's base commit (our agents commit, which empties `git diff` exactly when the worker does the right thing), and warning dimensions stay muted until the factory has produced evidence (a small judge escalated a healthy run at 25 seconds, before the worker touched a file). Ships foreman-calibrate.py, which replays thresholds over the runs' own JSONL timelines — the calibration upstream says it has not done. v0.2.0 adds bin/foreman-attach.py: supervision of a background worker someone ELSE already started (worker.sh / `claude agents`), which upstream cannot do because its runtime launches the worker itself. The observation is rebuilt from traces the worker already leaves — its state.json, the commits in its worktree, the tail of its transcript — so nothing is injected into the worker. Measured here on a live worker: warning dimensions read stuck 0.80 / off-track 0.80 at 362 s and 0.05 / 0.10 at 766 s on the same run with no commit in between, which is why warmup and confirmation matter more than threshold tuning. Also records that upstream's policy gates STEER_WORKER on `codex_backend == \"app-server\"` — the name of another backend rather than whether the worker accepts direction — so a steerable third-party worker can only ever be killed; the adapter works around it by printing the steering message for the lead to deliver.",
|
"description": "Semantic supervision for our own Claude agents. Wraps thruwire/foreman (MIT) — a runtime that watches a coding agent with a bounded observation (job, git status, diff, output tails, repository instructions), scores ten independent yes/no questions, and lets a deterministic Python policy decide continue / stop / retry / verify / finish / escalate. This plugin replaces both of upstream's paid dependencies: the worker is `claude -p` (Worker protocol, steer() returns False so the policy degrades to stop/retry, per upstream's own docs/workers.md), and the judge is the free-model gateway (free-worker.sh) instead of TypeSafe Jev, so a supervised run costs no paid tokens. Adds two things measured here rather than assumed: the observation carries commits since the run's base commit (our agents commit, which empties `git diff` exactly when the worker does the right thing), and warning dimensions stay muted until the factory has produced evidence (a small judge escalated a healthy run at 25 seconds, before the worker touched a file). Ships foreman-calibrate.py, which replays thresholds over the runs' own JSONL timelines — the calibration upstream says it has not done. v0.2.0 adds bin/foreman-attach.py: supervision of a background worker someone ELSE already started (worker.sh / `claude agents`), which upstream cannot do because its runtime launches the worker itself. The observation is rebuilt from traces the worker already leaves — its state.json, the commits in its worktree, the tail of its transcript — so nothing is injected into the worker. Measured here on a live worker: warning dimensions read stuck 0.80 / off-track 0.80 at 362 s and 0.05 / 0.10 at 766 s on the same run with no commit in between, which is why warmup and confirmation matter more than threshold tuning. Also records that upstream's policy gates STEER_WORKER on `codex_backend == \"app-server\"` — the name of another backend rather than whether the worker accepts direction — so a steerable third-party worker can only ever be killed; the adapter works around it by printing the steering message for the lead to deliver. v0.2.3 resolves the worker's tree from the LIVE session process (/proc/<pid>/cwd, skipping the bg-pty-host wrapper whose cwd is where the session was launched) instead of trusting state.json's starting cwd: measured on a live worker whose state.json said /home/a/pro/siege-lines while the session worked in /data/siege-arena364, so the judge read commits from the wrong checkout and printed \"no commits\" at the moment the worker was opening a PR.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "oleks",
|
"name": "oleks",
|
||||||
"email": "plugins@oleks.space"
|
"email": "plugins@oleks.space"
|
||||||
|
|||||||
+41
-5
@@ -158,8 +158,44 @@ def read_briefs(paths: list[str], repo: Path, budget: int = 12_000) -> str:
|
|||||||
return "\n\n".join(parts)
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def worker_repo(state: dict) -> Path:
|
||||||
|
"""Рабочее дерево воркера — по ЖИВОМУ процессу, а не по стартовому `cwd` из state.json.
|
||||||
|
|
||||||
|
kotkan/claude-plugin-foreman-supervisor#3: у воркера `6ce5556f` state.json отдавал
|
||||||
|
`/home/a/pro/siege-lines`, а сессия жила в `/data/siege-arena364` — судья строил наблюдение
|
||||||
|
по ЧУЖОМУ чекауту и печатал «коммиты: нет» ровно тогда, когда воркер делал PR. Коммиты и есть
|
||||||
|
главное свидетельство нормальной работы (грабля №1 скилла), так что промах по дереву ослепляет
|
||||||
|
судью именно на здоровом воркере. Процесс сессии узнаётся по `<sessionId>.jsonl` в cmdline.
|
||||||
|
"""
|
||||||
|
sid = str(state.get("sessionId") or "")
|
||||||
|
if sid:
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
["ps", "-eo", "pid=,args="], capture_output=True, text=True, timeout=30
|
||||||
|
).stdout
|
||||||
|
except (OSError, subprocess.TimeoutExpired):
|
||||||
|
out = ""
|
||||||
|
for line in out.splitlines():
|
||||||
|
if "%s.jsonl" % sid not in line:
|
||||||
|
continue
|
||||||
|
# ⚠️ Транскрипт сессии стоит в cmdline ДВУХ процессов: обёртки `claude bg-pty-host`
|
||||||
|
# и самого интерпретатора. У обёртки cwd — тот каталог, откуда сессию ЗАПУСКАЛИ
|
||||||
|
# (главный чекаут), у интерпретатора — где она РАБОТАЕТ. Замер 20.09: 871379
|
||||||
|
# bg-pty-host cwd=/home/a/pro/siege-lines, 871385 сама сессия cwd=/data/siege-arena364.
|
||||||
|
if "bg-pty-host" in line or "bg-spare" in line:
|
||||||
|
continue
|
||||||
|
pid = line.strip().split(None, 1)[0]
|
||||||
|
try:
|
||||||
|
cwd = os.readlink("/proc/%s/cwd" % pid)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
if Path(cwd).is_dir():
|
||||||
|
return Path(cwd)
|
||||||
|
return Path(state.get("worktreePath") or state.get("cwd") or ".")
|
||||||
|
|
||||||
|
|
||||||
def build_observation(short_id: str, state: dict, base: str, briefs: str = "") -> FactoryObservation:
|
def build_observation(short_id: str, state: dict, base: str, briefs: str = "") -> FactoryObservation:
|
||||||
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
repo = worker_repo(state)
|
||||||
status = str(state.get("state", "unknown"))
|
status = str(state.get("state", "unknown"))
|
||||||
started = state.get("createdAt") or ""
|
started = state.get("createdAt") or ""
|
||||||
elapsed = 0.0
|
elapsed = 0.0
|
||||||
@@ -267,7 +303,7 @@ def assess(worker_id: str, base: str, channel: str | None, attempts: int,
|
|||||||
warmup_seconds: float, briefs: list[str] | None = None) -> tuple[dict, str, str, dict, bool]:
|
warmup_seconds: float, briefs: list[str] | None = None) -> tuple[dict, str, str, dict, bool]:
|
||||||
"""Один замер: наблюдение → бесплатный судья → решение. Возвращает (state, decision, why, scores, warmup_ok)."""
|
"""Один замер: наблюдение → бесплатный судья → решение. Возвращает (state, decision, why, scores, warmup_ok)."""
|
||||||
state = read_state(worker_id)
|
state = read_state(worker_id)
|
||||||
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
repo = worker_repo(state)
|
||||||
obs = build_observation(worker_id, state, base, read_briefs(briefs or [], repo))
|
obs = build_observation(worker_id, state, base, read_briefs(briefs or [], repo))
|
||||||
workers = obs.active_workers or obs.worker_history
|
workers = obs.active_workers or obs.worker_history
|
||||||
has_commits = bool(workers and str(workers[0].get("commits_since_base", "")).strip())
|
has_commits = bool(workers and str(workers[0].get("commits_since_base", "")).strip())
|
||||||
@@ -366,7 +402,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
return watch(args)
|
return watch(args)
|
||||||
|
|
||||||
state = read_state(args.worker_id)
|
state = read_state(args.worker_id)
|
||||||
_repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
_repo = worker_repo(state)
|
||||||
obs = build_observation(args.worker_id, state, args.base, read_briefs(args.brief, _repo))
|
obs = build_observation(args.worker_id, state, args.base, read_briefs(args.brief, _repo))
|
||||||
|
|
||||||
elapsed = obs.elapsed_factory_seconds
|
elapsed = obs.elapsed_factory_seconds
|
||||||
@@ -376,7 +412,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
)
|
)
|
||||||
warmup_ok = has_commits or elapsed >= args.warmup_seconds
|
warmup_ok = has_commits or elapsed >= args.warmup_seconds
|
||||||
|
|
||||||
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
repo = worker_repo(state)
|
||||||
scores = asyncio.run(judge_once(obs, repo, args.judge_channel, args.judge_attempts))
|
scores = asyncio.run(judge_once(obs, repo, args.judge_channel, args.judge_attempts))
|
||||||
decision, why = decide(scores, warmup_ok)
|
decision, why = decide(scores, warmup_ok)
|
||||||
|
|
||||||
@@ -385,7 +421,7 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
"warmup_ok": warmup_ok, "scores": scores}, ensure_ascii=False, indent=1))
|
"warmup_ok": warmup_ok, "scores": scores}, ensure_ascii=False, indent=1))
|
||||||
else:
|
else:
|
||||||
print("воркер %s (%s)" % (args.worker_id, state.get("name", "")))
|
print("воркер %s (%s)" % (args.worker_id, state.get("name", "")))
|
||||||
print("дерево %s" % (state.get("worktreePath") or state.get("cwd")))
|
print("дерево %s" % worker_repo(state))
|
||||||
print("возраст %.0f с · коммиты от %s: %s" % (elapsed, args.base, "есть" if has_commits else "нет"))
|
print("возраст %.0f с · коммиты от %s: %s" % (elapsed, args.base, "есть" if has_commits else "нет"))
|
||||||
for k in ("implementation_complete", "meaningful_progress", "worker_stuck",
|
for k in ("implementation_complete", "meaningful_progress", "worker_stuck",
|
||||||
"work_off_track", "agents_md_drift", "needs_verification",
|
"work_off_track", "agents_md_drift", "needs_verification",
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Тесты выбора рабочего дерева: судить надо дерево, ГДЕ воркер работает, а не откуда запущен.
|
||||||
|
|
||||||
|
Зачем именно так (kotkan/claude-plugin-foreman-supervisor#3): у живого воркера `6ce5556f`
|
||||||
|
state.json отдавал `/home/a/pro/siege-lines`, а сессия жила в `/data/siege-arena364`. Судья строил
|
||||||
|
наблюдение по чужому чекауту и печатал «коммиты: нет» ровно тогда, когда воркер делал PR — то есть
|
||||||
|
слеп на главном свидетельстве здоровой работы. Вторая ловушка: транскрипт стоит в cmdline ДВУХ
|
||||||
|
процессов, и у обёртки `bg-pty-host` cwd — каталог запуска, а не работы.
|
||||||
|
|
||||||
|
~/pro/foreman/.venv/bin/python -m pytest tests/test_worker_repo.py -q
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MODULE_PATH = Path(__file__).resolve().parents[1] / "bin" / "foreman-attach.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("foreman_attach", MODULE_PATH)
|
||||||
|
assert spec and spec.loader
|
||||||
|
attach = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules["foreman_attach"] = attach
|
||||||
|
spec.loader.exec_module(attach)
|
||||||
|
|
||||||
|
SID = "6ce5556f-3627-47f2-bb56-e2704a613fe3"
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_ps(monkeypatch, lines: str) -> None:
|
||||||
|
class R:
|
||||||
|
stdout = lines
|
||||||
|
|
||||||
|
monkeypatch.setattr(attach.subprocess, "run", lambda *a, **k: R())
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_process_wins_over_stale_state(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""Дерево берётся из /proc живой сессии, а не из стартового cwd в state.json."""
|
||||||
|
live = tmp_path / "worktree"
|
||||||
|
live.mkdir()
|
||||||
|
_fake_ps(monkeypatch, "871385 /path/claude --resume /p/%s.jsonl\n" % SID)
|
||||||
|
monkeypatch.setattr(attach.os, "readlink", lambda p: str(live))
|
||||||
|
got = attach.worker_repo({"sessionId": SID, "cwd": str(tmp_path / "главный-чекаут")})
|
||||||
|
assert got == live
|
||||||
|
|
||||||
|
|
||||||
|
def test_bg_pty_host_wrapper_is_skipped(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""У обёртки cwd — каталог ЗАПУСКА; берём процесс самой сессии, даже если обёртка идёт первой."""
|
||||||
|
launched_from = tmp_path / "главный-чекаут"
|
||||||
|
working_in = tmp_path / "worktree"
|
||||||
|
launched_from.mkdir()
|
||||||
|
working_in.mkdir()
|
||||||
|
_fake_ps(
|
||||||
|
monkeypatch,
|
||||||
|
"871379 claude bg-pty-host --bg-pty-host /tmp/x.sock -- /v/2.1.278 --resume /p/%s.jsonl\n"
|
||||||
|
"871385 /v/2.1.278 --resume /p/%s.jsonl\n" % (SID, SID),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
attach.os, "readlink",
|
||||||
|
lambda p: str(launched_from) if p.endswith("871379/cwd") else str(working_in),
|
||||||
|
)
|
||||||
|
assert attach.worker_repo({"sessionId": SID, "cwd": str(launched_from)}) == working_in
|
||||||
|
|
||||||
|
|
||||||
|
def test_falls_back_to_state_when_process_gone(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""Воркер уже умер — присмотр не падает, дерево берётся из state.json."""
|
||||||
|
_fake_ps(monkeypatch, "")
|
||||||
|
stale = tmp_path / "из-state"
|
||||||
|
assert attach.worker_repo({"sessionId": SID, "cwd": str(stale)}) == stale
|
||||||
|
|
||||||
|
|
||||||
|
def test_unreadable_proc_does_not_crash(monkeypatch, tmp_path: Path) -> None:
|
||||||
|
"""/proc может исчезнуть между ps и readlink — это не повод ронять надзор."""
|
||||||
|
_fake_ps(monkeypatch, "871385 /v/2.1.278 --resume /p/%s.jsonl\n" % SID)
|
||||||
|
|
||||||
|
def boom(_p):
|
||||||
|
raise OSError("процесс исчез")
|
||||||
|
|
||||||
|
monkeypatch.setattr(attach.os, "readlink", boom)
|
||||||
|
stale = tmp_path / "из-state"
|
||||||
|
assert attach.worker_repo({"sessionId": SID, "cwd": str(stale)}) == stale
|
||||||
Reference in New Issue
Block a user