From b80361b788605dd2108282e52391b14b40ded207 Mon Sep 17 00:00:00 2001 From: repo-worker Date: Sun, 12 Jul 2026 20:41:32 +0300 Subject: [PATCH] =?UTF-8?q?generator:=20estimator-driven=20on-demand=20CI?= =?UTF-8?q?=20pipeline=20(estimate=E2=86=92order=E2=86=92provide=E2=86=92b?= =?UTF-8?q?uild=E2=86=92reap)=20(oleks/ci-scripts#25)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add generator/render_ondemand.py rendering a kind: ondemand pipeline whose pre-step runs nix-estimate --json (asserts sizing.version==1), orders an EphemeralBuilder class:ci via nix-estimate --provision ci | kubectl apply, waits Ready, builds pinned via backend_options.kubernetes.nodeSelector (Option B plain-worker), and reaps the CR (runs_on success+failure). Keeps the GENERATED header + arch banner, no || true (reap uses --ignore-not-found). Adds manifests/ondemand-release-amd64.yaml + rendered example; render is pure and deterministic. render_thin_quartet skips kind: ondemand by design. --- .../ondemand-release-amd64/.woodpecker.yaml | 62 ++++ generator/render_ondemand.py | 268 ++++++++++++++++++ manifests/ondemand-release-amd64.yaml | 46 +++ 3 files changed, 376 insertions(+) create mode 100644 examples/ondemand-release-amd64/.woodpecker.yaml create mode 100644 generator/render_ondemand.py create mode 100644 manifests/ondemand-release-amd64.yaml diff --git a/examples/ondemand-release-amd64/.woodpecker.yaml b/examples/ondemand-release-amd64/.woodpecker.yaml new file mode 100644 index 0000000..c783f1f --- /dev/null +++ b/examples/ondemand-release-amd64/.woodpecker.yaml @@ -0,0 +1,62 @@ +# GENERATED by oleks/ci-scripts — do not hand-edit. +# Source: oleks/ci-scripts manifests/ondemand-release-amd64.yaml + generator/render_ondemand.py (oleks/ci-scripts#25). +# Re-render with: python3 generator/render_ondemand.py --out + +labels: + arch: amd64 + +when: + - event: tag + ref: "refs/tags/v*" + +steps: + # 1) ESTIMATE + ORDER + PROVIDE (oleks/ci-scripts#25): size the build, order + # an on-demand EphemeralBuilder of class ci, wait for the node to join. + - name: provision + image: git.oleks.space/oleks/nix-ci:latest + commands: + - echo "▸ arch=$(uname -m)" + - ATTR=".#release" + - SYS="x86_64-linux" + - echo "▸ estimating sizing for $ATTR ($SYS)" + - nix-estimate "$ATTR" --system "$SYS" --json > estimate.json + - jq -e ".sizing.version == 1" estimate.json > /dev/null + - SIZING=$(jq -c ".sizing" estimate.json) + - echo "▸ sizing: $SIZING" + - echo "▸ ordering on-demand builder(s) (class ci)" + - nix-estimate "$ATTR" --system "$SYS" --provision ci | kubectl apply -f - -o name | tee ordered-builders.txt + - echo "▸ waiting for ordered builder(s) to become Ready" + - while read -r cr; do echo "▸ waiting $cr"; kubectl wait --for=condition=Ready "$cr" --timeout=1200s; done < ordered-builders.txt + + # 2) BUILD on the freshly-joined ephemeral node. Pinned via nodeSelector; the + # existing k8s-backend Woodpecker agent schedules this step-pod there. + - name: build + image: git.oleks.space/oleks/nix-ci:latest + depends_on: + - provision + backend_options: + kubernetes: + nodeSelector: + kubernetes.io/arch: amd64 + resources: + requests: + memory: 8Gi + limits: + memory: 32Gi + commands: + - echo "▸ arch=$(uname -m)" + - nixos-ci-entrypoint true + - PUBLISH=1 nix run .#publish + + # 3) REAP: delete the ordered CR(s) to tear the node down at end of window. + # runs_on success+failure so a failed build never strands a paid node. + - name: reap + image: git.oleks.space/oleks/nix-ci:latest + depends_on: + - build + runs_on: + - success + - failure + commands: + - echo "▸ arch=$(uname -m)" + - if [ -f ordered-builders.txt ]; then while read -r cr; do echo "▸ reaping $cr"; kubectl delete "$cr" --ignore-not-found; done < ordered-builders.txt; else echo "▸ no ordered-builders.txt; nothing to reap"; fi diff --git a/generator/render_ondemand.py b/generator/render_ondemand.py new file mode 100644 index 0000000..04fc221 --- /dev/null +++ b/generator/render_ondemand.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +"""Render the estimator-driven on-demand CI pipeline (oleks/ci-scripts#25). + +FROZEN CONTRACT — estimate -> order -> provide -> build -> reap. The generated +pipeline's pre-step drives the nix-estimator + builder-arbitrage on-demand +window (estimate the sizing, order an EphemeralBuilder of class `ci`, wait for +the node to join, build pinned to it, then reap the CR) so a heavy release +build gets a fresh, right-sized cloud node that is online only for the window. + +SHAPE (`kind: ondemand` in manifests/*.yaml): + 1. provision pre-step: + - `nix-estimate --system --json` -> parse the `sizing` + block (version-pinned; the pre-step asserts sizing.version == 1); + - `nix-estimate --system --provision ci | kubectl apply -f -` + emits N EphemeralBuilder docs (class: ci) and applies them, capturing + the applied CR names into a workspace file; + - `kubectl wait --for=condition=Ready ephemeralbuilder/` on each + ordered CR until the backing node has joined and is serving. + 2. build step: the real build, pinned to the freshly-joined node via + `backend_options.kubernetes.nodeSelector` (Option B / plain-worker MVP — + the ephemeral node joins armer k3s as a plain `kubernetes.io/arch=` + worker and the EXISTING k8s-backend Woodpecker agent schedules this + amd64 step-pod onto it; no per-node Woodpecker agent is baked in). + 3. reap step: `kubectl delete` every ordered CR (idle-park would also reap on + `parking.policy: cost`, but an explicit delete tears the node down at the + end of the window). Runs on `success` AND `failure` so a failed build + never strands a paid node. + +Convention notes (match generate.py / render_thin_*): + - GENERATED header is emitted verbatim; edit the manifest + re-render. + - Every step's first command is `echo "▸ arch=$(uname -m)"` (Woodpecker + arch banner) — kept, never dropped. + - No `|| true`: the reap uses `kubectl delete --ignore-not-found` (a CR the + idle-park already reaped is not an error) rather than masking failures. + +gVisor / PodGater caveat (out of scope for #25, noted for the follow-up): the +build step-pod is spawned by the shared Woodpecker k8s-backend agent, so it is +NOT stamped by the builder-arbitrage operator's PodGater. A Kyverno backstop on +the ci node is the follow-up that enforces the gVisor runtime class there. + +Usage: + python3 generator/render_ondemand.py --out /path/to/scratch +""" + +import argparse +import math +import pathlib + +import yaml + +HERE = pathlib.Path(__file__).parent.parent + +GENERATED_HEADER = ( + "# GENERATED by oleks/ci-scripts — do not hand-edit.\n" + "# Source: oleks/ci-scripts manifests/{repo}.yaml + " + "generator/render_ondemand.py (oleks/ci-scripts#25).\n" + "# Re-render with: python3 generator/render_ondemand.py --out \n" + "\n" +) + +# Default images. The orchestrator step needs `nix-estimate` + `kubectl` + `jq` +# on PATH; the build step is the repo's own build image. Both are manifest +# overridable (orchestrator_image / build_image). +DEFAULT_ORCHESTRATOR_IMAGE = "git.oleks.space/oleks/nix-ci:latest" +DEFAULT_BUILD_IMAGE = "git.oleks.space/oleks/nix-ci:latest" + +# nix-estimate is invoked verbatim per the contract; the binary name is +# manifest-overridable (e.g. a `nix run #nix-estimate --` prefix) but +# defaults to the bare CLI assumed present in the orchestrator image. +DEFAULT_ESTIMATE_CMD = "nix-estimate" +DEFAULT_CLASS = "ci" +DEFAULT_WAIT_TIMEOUT = "1200s" +# Workspace-relative file the provision step writes the applied CR names into; +# read back by the reap step (Woodpecker mounts the workspace across steps, +# so this is how the ordered set crosses the step boundary). +ORDERED_FILE = "ordered-builders.txt" + + +def _when_block(manifest: dict) -> str: + """The `when:` trigger. Defaults to the release-tag shape every other + renderer here uses; a manifest may instead pin `push` branches.""" + branches = manifest.get("branches") + if branches: + joined = "[" + ", ".join(branches) + "]" + return f"when:\n - event: push\n branch: {joined}\n" + ref = manifest.get("tag_ref", "refs/tags/v*") + return f'when:\n - event: tag\n ref: "{ref}"\n' + + +def _provision_commands(manifest: dict) -> str: + estimate_cmd = manifest.get("estimate_cmd", DEFAULT_ESTIMATE_CMD) + attr = manifest["target"] + system = manifest["system"] + cls = manifest.get("provision_class", DEFAULT_CLASS) + wait_timeout = manifest.get("wait_timeout", DEFAULT_WAIT_TIMEOUT) + + # Each list item is one command; Woodpecker concatenates a step's commands + # into a single `set -e` shell script, so vars set here persist within the + # step. Kept shellcheck-clean (quoted expansions, `read -r`). + lines = [ + 'echo "▸ arch=$(uname -m)"', + f'ATTR="{attr}"', + f'SYS="{system}"', + 'echo "▸ estimating sizing for $ATTR ($SYS)"', + f'{estimate_cmd} "$ATTR" --system "$SYS" --json > estimate.json', + # Assert the versioned sizing block the contract froze (schema v1); + # -e makes jq exit non-zero (failing the step) on a false/absent match. + 'jq -e ".sizing.version == 1" estimate.json > /dev/null', + 'SIZING=$(jq -c ".sizing" estimate.json)', + 'echo "▸ sizing: $SIZING"', + 'echo "▸ ordering on-demand builder(s) (class ' + cls + ')"', + # The provision path emits N EphemeralBuilder docs; -o name echoes the + # applied CR identities (ephemeralbuilder.arbitrage.oleks.space/) + # which we persist for the wait + reap. + f'{estimate_cmd} "$ATTR" --system "$SYS" --provision {cls} ' + f"| kubectl apply -f - -o name | tee {ORDERED_FILE}", + 'echo "▸ waiting for ordered builder(s) to become Ready"', + f'while read -r cr; do echo "▸ waiting $cr"; ' + f'kubectl wait --for=condition=Ready "$cr" --timeout={wait_timeout}; ' + f"done < {ORDERED_FILE}", + ] + return "\n".join(f" - {line}" for line in lines) + + +def _reap_commands(manifest: dict) -> str: + lines = [ + 'echo "▸ arch=$(uname -m)"', + # --ignore-not-found: a CR the idle-park (parking.policy: cost) already + # reaped is not an error — this is the no-`|| true` equivalent. + f"if [ -f {ORDERED_FILE} ]; then " + f'while read -r cr; do echo "▸ reaping $cr"; ' + f'kubectl delete "$cr" --ignore-not-found; ' + f"done < {ORDERED_FILE}; " + f'else echo "▸ no {ORDERED_FILE}; nothing to reap"; fi', + ] + return "\n".join(f" - {line}" for line in lines) + + +def _build_commands(manifest: dict) -> str: + run = manifest["build_run"] + lines = ['echo "▸ arch=$(uname -m)"'] + lines += [line for line in run.rstrip("\n").splitlines()] + return "\n".join(f" - {line}" for line in lines) + + +def _kube_env_block(manifest: dict) -> str: + """Optional scoped-kubeconfig secret (emmett consumes the fleet via a + scoped SA kubeconfig). Absent -> rely on the in-cluster ServiceAccount.""" + secret = manifest.get("kubeconfig_secret") + if not secret: + return "" + return ( + f" environment:\n KUBECONFIG_CONTENT:\n from_secret: {secret}\n" + ) + + +def _memory_block(manifest: dict, indent: str) -> str: + mem_req = manifest.get("memory_request") + mem_lim = manifest.get("memory_limit") + if not (mem_req or mem_lim): + return "" + req = mem_req or mem_lim + lim = mem_lim or mem_req + return ( + f"{indent}resources:\n" + f"{indent} requests:\n" + f"{indent} memory: {req}\n" + f"{indent} limits:\n" + f"{indent} memory: {lim}\n" + ) + + +def render(manifest: dict) -> str: + repo = manifest["repo"] + arch = manifest["arch"] + orchestrator_image = manifest.get("orchestrator_image", DEFAULT_ORCHESTRATOR_IMAGE) + build_image = manifest.get("build_image", DEFAULT_BUILD_IMAGE) + + header = GENERATED_HEADER.format(repo=repo) + when_block = _when_block(manifest) + provision_cmds = _provision_commands(manifest) + build_cmds = _build_commands(manifest) + reap_cmds = _reap_commands(manifest) + kube_env = _kube_env_block(manifest) + # The build step pins to the freshly-joined node via nodeSelector (Option B + # plain-worker: kubernetes.io/arch=). Any extra manifest node_selector + # keys are merged in (e.g. a per-order node label once the operator stamps + # one — reserved, not required for the MVP). + node_selector = {"kubernetes.io/arch": arch} + node_selector.update(manifest.get("node_selector", {})) + node_selector_lines = "\n".join( + f" {k}: {v}" for k, v in node_selector.items() + ) + build_mem = _memory_block(manifest, " ") + + # est_makespan_min echo helps derive the on-demand window; parking/maxDuration + # themselves are set by the estimator's --provision path, not here. + return f"""{header}labels: + arch: {arch} + +{when_block} +steps: + # 1) ESTIMATE + ORDER + PROVIDE (oleks/ci-scripts#25): size the build, order + # an on-demand EphemeralBuilder of class ci, wait for the node to join. + - name: provision + image: {orchestrator_image} +{kube_env} commands: +{provision_cmds} + + # 2) BUILD on the freshly-joined ephemeral node. Pinned via nodeSelector; the + # existing k8s-backend Woodpecker agent schedules this step-pod there. + - name: build + image: {build_image} + depends_on: + - provision + backend_options: + kubernetes: + nodeSelector: +{node_selector_lines} +{build_mem} commands: +{build_cmds} + + # 3) REAP: delete the ordered CR(s) to tear the node down at end of window. + # runs_on success+failure so a failed build never strands a paid node. + - name: reap + image: {orchestrator_image} + depends_on: + - build + runs_on: + - success + - failure +{kube_env} commands: +{reap_cmds} +""" + + +def render_makespan_hint(est_makespan_min: float) -> int: + """maxDuration = ceil(makespan * 1.5) minutes — mirrors the estimator's + --provision derivation so the two stay legible together. Provided as a + pure helper for tests/docs; the generated pipeline does not set it (the + estimator's --provision path stamps requirements.maxDuration itself).""" + return math.ceil(est_makespan_min * 1.5) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", type=pathlib.Path, required=True) + args = ap.parse_args() + + manifests_dir = HERE / "manifests" + wrote_any = False + for path in sorted(manifests_dir.glob("*.yaml")): + manifest = yaml.safe_load(path.read_text()) + if manifest.get("kind") != "ondemand": + continue + repo = manifest["repo"] + out_dir = args.out / repo + out_dir.mkdir(parents=True, exist_ok=True) + target = out_dir / ".woodpecker.yaml" + target.write_text(render(manifest)) + print(f"wrote {target}") + wrote_any = True + if not wrote_any: + print("no ondemand manifests found (kind: ondemand)") + + +if __name__ == "__main__": + main() diff --git a/manifests/ondemand-release-amd64.yaml b/manifests/ondemand-release-amd64.yaml new file mode 100644 index 0000000..b07e586 --- /dev/null +++ b/manifests/ondemand-release-amd64.yaml @@ -0,0 +1,46 @@ +# Example estimator-driven on-demand CI target (oleks/ci-scripts#25). +# +# Renders with generator/render_ondemand.py (NOT the quartet renderer — the +# `kind: ondemand` dispatch is handled by its own script; render_thin_quartet +# skips this manifest as an unknown kind by design). +# +# SHAPE: a heavy amd64 release build that must NOT serialise through the single +# emmett-amd64 Woodpecker agent. The pre-step estimates the sizing, orders a +# fresh on-demand amd64 cloud node (EphemeralBuilder class: ci), waits for it to +# join armer k3s as a plain kubernetes.io/arch=amd64 worker, builds there, then +# reaps the node. See the reference order CR: +# builder-arbitrage config/samples/arbitrage_v1alpha1_ephemeralbuilder_ci-amd64-release-slot.yaml +# +# Fields: +# repo - logical name (used in the GENERATED header + out path) +# kind - `ondemand` (dispatch key for render_ondemand.py) +# target - nix flake attr to build/estimate (e.g. .#release) +# system - nix system double (x86_64-linux) fed to nix-estimate +# --system; the CR arch is derived by the estimator's +# provision.arch_for_system(system). +# arch - k8s/Go arch double (amd64) for the node label + the +# build step's nodeSelector pin. MUST be the k8s double, +# not the nix double (kubernetes.io/arch enum is amd64). +# provision_class - EphemeralBuilder class (ci) passed to --provision. +# build_run - the actual build command(s) run on the ordered node. +# wait_timeout - kubectl wait --for=condition=Ready timeout. +# kubeconfig_secret - (optional) Woodpecker secret holding a scoped SA +# kubeconfig; omit to use the step-pod's in-cluster SA. +# node_selector - (optional) extra nodeSelector keys merged onto the +# arch pin (reserved; the MVP needs only the arch pin). +# orchestrator_image - (optional) image with nix-estimate + kubectl + jq. +# build_image - (optional) the repo's build image. +# memory_request / +# memory_limit - (optional) build step k8s memory sizing. +repo: ondemand-release-amd64 +kind: ondemand +target: .#release +system: x86_64-linux +arch: amd64 +provision_class: ci +wait_timeout: 1200s +build_run: | + nixos-ci-entrypoint true + PUBLISH=1 nix run .#publish +memory_request: 8Gi +memory_limit: 32Gi