0.2.1: --brief — ТЗ проекта в наблюдение судьи #1
@@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "foreman-supervisor",
|
"name": "foreman-supervisor",
|
||||||
"version": "0.2.0",
|
"version": "0.2.1",
|
||||||
"description": "Semantic supervision for our own Claude agents. Wraps thruwire/foreman (MIT) — a runtime that watches a coding agent with a bounded observation (job, git status, diff, output tails, repository instructions), scores ten independent yes/no questions, and lets a deterministic Python policy decide continue / stop / retry / verify / finish / escalate. This plugin replaces both of upstream's paid dependencies: the worker is `claude -p` (Worker protocol, steer() returns False so the policy degrades to stop/retry, per upstream's own docs/workers.md), and the judge is the free-model gateway (free-worker.sh) instead of TypeSafe Jev, so a supervised run costs no paid tokens. Adds two things measured here rather than assumed: the observation carries commits since the run's base commit (our agents commit, which empties `git diff` exactly when the worker does the right thing), and warning dimensions stay muted until the factory has produced evidence (a small judge escalated a healthy run at 25 seconds, before the worker touched a file). Ships foreman-calibrate.py, which replays thresholds over the runs' own JSONL timelines — the calibration upstream says it has not done. v0.2.0 adds bin/foreman-attach.py: supervision of a background worker someone ELSE already started (worker.sh / `claude agents`), which upstream cannot do because its runtime launches the worker itself. The observation is rebuilt from traces the worker already leaves — its state.json, the commits in its worktree, the tail of its transcript — so nothing is injected into the worker. Measured here on a live worker: warning dimensions read stuck 0.80 / off-track 0.80 at 362 s and 0.05 / 0.10 at 766 s on the same run with no commit in between, which is why warmup and confirmation matter more than threshold tuning. Also records that upstream's policy gates STEER_WORKER on `codex_backend == \"app-server\"` — the name of another backend rather than whether the worker accepts direction — so a steerable third-party worker can only ever be killed; the adapter works around it by printing the steering message for the lead to deliver.",
|
"description": "Semantic supervision for our own Claude agents. Wraps thruwire/foreman (MIT) — a runtime that watches a coding agent with a bounded observation (job, git status, diff, output tails, repository instructions), scores ten independent yes/no questions, and lets a deterministic Python policy decide continue / stop / retry / verify / finish / escalate. This plugin replaces both of upstream's paid dependencies: the worker is `claude -p` (Worker protocol, steer() returns False so the policy degrades to stop/retry, per upstream's own docs/workers.md), and the judge is the free-model gateway (free-worker.sh) instead of TypeSafe Jev, so a supervised run costs no paid tokens. Adds two things measured here rather than assumed: the observation carries commits since the run's base commit (our agents commit, which empties `git diff` exactly when the worker does the right thing), and warning dimensions stay muted until the factory has produced evidence (a small judge escalated a healthy run at 25 seconds, before the worker touched a file). Ships foreman-calibrate.py, which replays thresholds over the runs' own JSONL timelines — the calibration upstream says it has not done. v0.2.0 adds bin/foreman-attach.py: supervision of a background worker someone ELSE already started (worker.sh / `claude agents`), which upstream cannot do because its runtime launches the worker itself. The observation is rebuilt from traces the worker already leaves — its state.json, the commits in its worktree, the tail of its transcript — so nothing is injected into the worker. Measured here on a live worker: warning dimensions read stuck 0.80 / off-track 0.80 at 362 s and 0.05 / 0.10 at 766 s on the same run with no commit in between, which is why warmup and confirmation matter more than threshold tuning. Also records that upstream's policy gates STEER_WORKER on `codex_backend == \"app-server\"` — the name of another backend rather than whether the worker accepts direction — so a steerable third-party worker can only ever be killed; the adapter works around it by printing the steering message for the lead to deliver.",
|
||||||
"author": {
|
"author": {
|
||||||
"name": "oleks",
|
"name": "oleks",
|
||||||
|
|||||||
+44
-6
@@ -126,7 +126,39 @@ def repo_instructions(repo: Path) -> tuple[str | None, str]:
|
|||||||
return None, ""
|
return None, ""
|
||||||
|
|
||||||
|
|
||||||
def build_observation(short_id: str, state: dict, base: str) -> FactoryObservation:
|
def read_briefs(paths: list[str], repo: Path, budget: int = 12_000) -> str:
|
||||||
|
"""ТЗ проекта для судьи: чем «хорошо» отличается от «плохо» ЗДЕСЬ, а не вообще.
|
||||||
|
|
||||||
|
Зачем: наблюдение несло только промпт воркера и первые 4 КБ CLAUDE.md репозитория. Судья видел,
|
||||||
|
ЧТО поручено, но не видел, ЗАЧЕМ и по какому канону это принимается, — и «работа в рамках
|
||||||
|
задания» оценивалась вслепую (слово Антона 2026-09-20: «ознакомь надзирателя, над чем мы
|
||||||
|
работаем»). Пути берём как есть или относительно рабочего дерева воркера; каждый файл
|
||||||
|
урезается по бюджету, чтобы наблюдение осталось в пределах окна бесплатной модели.
|
||||||
|
"""
|
||||||
|
if not paths:
|
||||||
|
return ""
|
||||||
|
share = max(1_000, budget // len(paths))
|
||||||
|
parts: list[str] = []
|
||||||
|
for raw in paths:
|
||||||
|
f = Path(raw)
|
||||||
|
if not f.is_absolute() and not f.exists():
|
||||||
|
f = repo / raw
|
||||||
|
if not f.exists():
|
||||||
|
parts.append("=== %s === (файла нет — пропущен)" % raw)
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
text = f.read_text(encoding="utf-8", errors="replace")
|
||||||
|
except OSError as e:
|
||||||
|
parts.append("=== %s === (не читается: %s)" % (raw, e))
|
||||||
|
continue
|
||||||
|
cut = text[:share]
|
||||||
|
if len(text) > share:
|
||||||
|
cut += "\n… (урезано, всего %d символов)" % len(text)
|
||||||
|
parts.append("=== %s ===\n%s" % (f, cut))
|
||||||
|
return "\n\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def build_observation(short_id: str, state: dict, base: str, briefs: str = "") -> FactoryObservation:
|
||||||
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
||||||
status = str(state.get("state", "unknown"))
|
status = str(state.get("state", "unknown"))
|
||||||
started = state.get("createdAt") or ""
|
started = state.get("createdAt") or ""
|
||||||
@@ -149,7 +181,9 @@ def build_observation(short_id: str, state: dict, base: str) -> FactoryObservati
|
|||||||
"committed_diffstat": diffstat,
|
"committed_diffstat": diffstat,
|
||||||
}
|
}
|
||||||
return FactoryObservation(
|
return FactoryObservation(
|
||||||
original_job=str(state.get("intent") or state.get("name") or "")[:6_000],
|
original_job=(str(state.get("intent") or state.get("name") or "")[:6_000]
|
||||||
|
+ ("\n\n=== ТЕХНИЧЕСКОЕ ЗАДАНИЕ ПРОЕКТА (канон, по которому работа принимается) ===\n"
|
||||||
|
+ briefs if briefs else "")),
|
||||||
run_id="attach-%s" % short_id,
|
run_id="attach-%s" % short_id,
|
||||||
factory_status=status,
|
factory_status=status,
|
||||||
iteration=1,
|
iteration=1,
|
||||||
@@ -230,11 +264,11 @@ def decide(scores: dict, warmup_ok: bool) -> tuple[str, str]:
|
|||||||
|
|
||||||
|
|
||||||
def assess(worker_id: str, base: str, channel: str | None, attempts: int,
|
def assess(worker_id: str, base: str, channel: str | None, attempts: int,
|
||||||
warmup_seconds: float) -> tuple[dict, str, str, dict, bool]:
|
warmup_seconds: float, briefs: list[str] | None = None) -> tuple[dict, str, str, dict, bool]:
|
||||||
"""Один замер: наблюдение → бесплатный судья → решение. Возвращает (state, decision, why, scores, warmup_ok)."""
|
"""Один замер: наблюдение → бесплатный судья → решение. Возвращает (state, decision, why, scores, warmup_ok)."""
|
||||||
state = read_state(worker_id)
|
state = read_state(worker_id)
|
||||||
obs = build_observation(worker_id, state, base)
|
|
||||||
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
||||||
|
obs = build_observation(worker_id, state, base, read_briefs(briefs or [], repo))
|
||||||
workers = obs.active_workers or obs.worker_history
|
workers = obs.active_workers or obs.worker_history
|
||||||
has_commits = bool(workers and str(workers[0].get("commits_since_base", "")).strip())
|
has_commits = bool(workers and str(workers[0].get("commits_since_base", "")).strip())
|
||||||
warmup_ok = has_commits or obs.elapsed_factory_seconds >= warmup_seconds
|
warmup_ok = has_commits or obs.elapsed_factory_seconds >= warmup_seconds
|
||||||
@@ -256,7 +290,7 @@ def watch(args: argparse.Namespace) -> int:
|
|||||||
try:
|
try:
|
||||||
state, decision, why, scores, warmup_ok = assess(
|
state, decision, why, scores, warmup_ok = assess(
|
||||||
args.worker_id, args.base, args.judge_channel,
|
args.worker_id, args.base, args.judge_channel,
|
||||||
args.judge_attempts, args.warmup_seconds,
|
args.judge_attempts, args.warmup_seconds, args.brief,
|
||||||
)
|
)
|
||||||
except SystemExit:
|
except SystemExit:
|
||||||
raise
|
raise
|
||||||
@@ -309,6 +343,9 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
ap.add_argument("worker_id", help="короткий id воркера, как в `claude agents`")
|
ap.add_argument("worker_id", help="короткий id воркера, как в `claude agents`")
|
||||||
ap.add_argument("--base", default="origin/main",
|
ap.add_argument("--base", default="origin/main",
|
||||||
help="точка отсчёта коммитов воркера (по умолчанию origin/main)")
|
help="точка отсчёта коммитов воркера (по умолчанию origin/main)")
|
||||||
|
ap.add_argument("--brief", action="append", default=[], metavar="ФАЙЛ",
|
||||||
|
help="файл ТЗ/канона проекта в наблюдение судьи (повторяемый): спека, docs/LOOK.md, "
|
||||||
|
"тело issue. Путь абсолютный или относительно рабочего дерева воркера")
|
||||||
ap.add_argument("--judge-channel", default=None, help="принудительный канал free-worker.sh")
|
ap.add_argument("--judge-channel", default=None, help="принудительный канал free-worker.sh")
|
||||||
ap.add_argument("--judge-attempts", type=int, default=3)
|
ap.add_argument("--judge-attempts", type=int, default=3)
|
||||||
ap.add_argument("--warmup-seconds", type=float, default=420.0,
|
ap.add_argument("--warmup-seconds", type=float, default=420.0,
|
||||||
@@ -329,7 +366,8 @@ def main(argv: list[str] | None = None) -> int:
|
|||||||
return watch(args)
|
return watch(args)
|
||||||
|
|
||||||
state = read_state(args.worker_id)
|
state = read_state(args.worker_id)
|
||||||
obs = build_observation(args.worker_id, state, args.base)
|
_repo = Path(state.get("worktreePath") or state.get("cwd") or ".")
|
||||||
|
obs = build_observation(args.worker_id, state, args.base, read_briefs(args.brief, _repo))
|
||||||
|
|
||||||
elapsed = obs.elapsed_factory_seconds
|
elapsed = obs.elapsed_factory_seconds
|
||||||
has_commits = bool(
|
has_commits = bool(
|
||||||
|
|||||||
@@ -33,10 +33,18 @@ description: "Поставить надзирателя над агентом,
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
~/pro/foreman/.venv/bin/python "$CLAUDE_PLUGIN_ROOT/bin/foreman-attach.py" <id> \
|
~/pro/foreman/.venv/bin/python "$CLAUDE_PLUGIN_ROOT/bin/foreman-attach.py" <id> \
|
||||||
|
--brief docs/LOOK.md --brief specs/<фича>/spec.md \
|
||||||
--watch 300 --confirm-warnings # присмотр в цикле, пока воркер жив
|
--watch 300 --confirm-warnings # присмотр в цикле, пока воркер жив
|
||||||
~/pro/foreman/.venv/bin/python "$CLAUDE_PLUGIN_ROOT/bin/foreman-attach.py" <id> # одна оценка
|
~/pro/foreman/.venv/bin/python "$CLAUDE_PLUGIN_ROOT/bin/foreman-attach.py" <id> # одна оценка
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**`--brief` — дай судье ТЗ, а не только промпт воркера** (повторяемый; путь абсолютный или
|
||||||
|
относительно рабочего дерева воркера). Без него судья видит задание воркера и первые 4 КБ
|
||||||
|
`CLAUDE.md` репозитория — то есть ЧТО поручено, но не ЗАЧЕМ и по какому канону это принимается,
|
||||||
|
и «работа в рамках задания» оценивается вслепую. Замер на живом прогоне 2026-09-20: задание для
|
||||||
|
судьи 4.4 КБ → 14.3 КБ, когда в него вклеили `docs/LOOK.md`, `docs/AI_GUIDE.md` и тело issue.
|
||||||
|
Скармливай СУЩЕСТВУЮЩИЕ доки проекта, а не пересказ: канон внешнего вида, спеку, тело issue.
|
||||||
|
|
||||||
⚠️ Только интерпретатором `~/pro/foreman/.venv/bin/python` — модуль `foreman` живёт там.
|
⚠️ Только интерпретатором `~/pro/foreman/.venv/bin/python` — модуль `foreman` живёт там.
|
||||||
|
|
||||||
Наблюдение строится из следов, которые воркер и так оставляет; в него ничего не внедряется:
|
Наблюдение строится из следов, которые воркер и так оставляет; в него ничего не внедряется:
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
"""Тесты флага --brief: ТЗ проекта должно доезжать до судьи, а не только промпт воркера.
|
||||||
|
|
||||||
|
Зачем именно так: наблюдение несло `intent` воркера и первые 4 КБ CLAUDE.md репозитория, и судья
|
||||||
|
оценивал «в рамках ли задания» вслепую — без канона, по которому работа принимается
|
||||||
|
(слово Антона 2026-09-20). Тесты держат три свойства: файлы читаются и относительно рабочего
|
||||||
|
дерева воркера, бюджет делится между ними, отсутствующий файл не роняет присмотр.
|
||||||
|
|
||||||
|
~/pro/foreman/.venv/bin/python -m pytest tests/test_attach_brief.py -q
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
MODULE_PATH = Path(__file__).resolve().parents[1] / "bin" / "foreman-attach.py"
|
||||||
|
spec = importlib.util.spec_from_file_location("foreman_attach", MODULE_PATH)
|
||||||
|
assert spec and spec.loader
|
||||||
|
attach = importlib.util.module_from_spec(spec)
|
||||||
|
sys.modules["foreman_attach"] = attach
|
||||||
|
spec.loader.exec_module(attach)
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_briefs_is_empty(tmp_path: Path) -> None:
|
||||||
|
assert attach.read_briefs([], tmp_path) == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_relative_path_resolves_against_worker_tree(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "docs").mkdir()
|
||||||
|
(tmp_path / "docs" / "LOOK.md").write_text("канон картинки", encoding="utf-8")
|
||||||
|
out = attach.read_briefs(["docs/LOOK.md"], tmp_path)
|
||||||
|
assert "канон картинки" in out
|
||||||
|
assert "LOOK.md" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_file_is_reported_not_fatal(tmp_path: Path) -> None:
|
||||||
|
out = attach.read_briefs(["docs/НЕТУ.md"], tmp_path)
|
||||||
|
assert "файла нет" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_budget_is_split_between_briefs(tmp_path: Path) -> None:
|
||||||
|
for name in ("a.md", "b.md"):
|
||||||
|
(tmp_path / name).write_text("я" * 50_000, encoding="utf-8")
|
||||||
|
out = attach.read_briefs(["a.md", "b.md"], tmp_path, budget=4_000)
|
||||||
|
assert "урезано" in out
|
||||||
|
assert len(out) < 12_000 # два файла по доле бюджета, а не два полных файла
|
||||||
|
|
||||||
|
|
||||||
|
def test_brief_lands_in_the_job_the_judge_sees(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "SPEC.md").write_text("приёмка: здание читается как постройка", encoding="utf-8")
|
||||||
|
state = {"intent": "почини казарму", "worktreePath": str(tmp_path), "state": "working"}
|
||||||
|
obs = attach.build_observation("deadbeef", state, "main", attach.read_briefs(["SPEC.md"], tmp_path))
|
||||||
|
assert "почини казарму" in obs.original_job
|
||||||
|
assert "ТЕХНИЧЕСКОЕ ЗАДАНИЕ" in obs.original_job
|
||||||
|
assert "здание читается как постройка" in obs.original_job
|
||||||
Reference in New Issue
Block a user