repo-worker
b80361b788
ci/woodpecker/cron/drift Pipeline was successful
Add generator/render_ondemand.py rendering a kind: ondemand pipeline whose pre-step runs nix-estimate --json (asserts sizing.version==1), orders an EphemeralBuilder class:ci via nix-estimate --provision ci | kubectl apply, waits Ready, builds pinned via backend_options.kubernetes.nodeSelector (Option B plain-worker), and reaps the CR (runs_on success+failure). Keeps the GENERATED header + arch banner, no || true (reap uses --ignore-not-found). Adds manifests/ondemand-release-amd64.yaml + rendered example; render is pure and deterministic. render_thin_quartet skips kind: ondemand by design.
269 lines
11 KiB
Python
269 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""Render the estimator-driven on-demand CI pipeline (oleks/ci-scripts#25).
|
|
|
|
FROZEN CONTRACT — estimate -> order -> provide -> build -> reap. The generated
|
|
pipeline's pre-step drives the nix-estimator + builder-arbitrage on-demand
|
|
window (estimate the sizing, order an EphemeralBuilder of class `ci`, wait for
|
|
the node to join, build pinned to it, then reap the CR) so a heavy release
|
|
build gets a fresh, right-sized cloud node that is online only for the window.
|
|
|
|
SHAPE (`kind: ondemand` in manifests/*.yaml):
|
|
1. provision pre-step:
|
|
- `nix-estimate <attr> --system <sys> --json` -> parse the `sizing`
|
|
block (version-pinned; the pre-step asserts sizing.version == 1);
|
|
- `nix-estimate <attr> --system <sys> --provision ci | kubectl apply -f -`
|
|
emits N EphemeralBuilder docs (class: ci) and applies them, capturing
|
|
the applied CR names into a workspace file;
|
|
- `kubectl wait --for=condition=Ready ephemeralbuilder/<name>` on each
|
|
ordered CR until the backing node has joined and is serving.
|
|
2. build step: the real build, pinned to the freshly-joined node via
|
|
`backend_options.kubernetes.nodeSelector` (Option B / plain-worker MVP —
|
|
the ephemeral node joins armer k3s as a plain `kubernetes.io/arch=<arch>`
|
|
worker and the EXISTING k8s-backend Woodpecker agent schedules this
|
|
amd64 step-pod onto it; no per-node Woodpecker agent is baked in).
|
|
3. reap step: `kubectl delete` every ordered CR (idle-park would also reap on
|
|
`parking.policy: cost`, but an explicit delete tears the node down at the
|
|
end of the window). Runs on `success` AND `failure` so a failed build
|
|
never strands a paid node.
|
|
|
|
Convention notes (match generate.py / render_thin_*):
|
|
- GENERATED header is emitted verbatim; edit the manifest + re-render.
|
|
- Every step's first command is `echo "▸ arch=$(uname -m)"` (Woodpecker
|
|
arch banner) — kept, never dropped.
|
|
- No `|| true`: the reap uses `kubectl delete --ignore-not-found` (a CR the
|
|
idle-park already reaped is not an error) rather than masking failures.
|
|
|
|
gVisor / PodGater caveat (out of scope for #25, noted for the follow-up): the
|
|
build step-pod is spawned by the shared Woodpecker k8s-backend agent, so it is
|
|
NOT stamped by the builder-arbitrage operator's PodGater. A Kyverno backstop on
|
|
the ci node is the follow-up that enforces the gVisor runtime class there.
|
|
|
|
Usage:
|
|
python3 generator/render_ondemand.py --out /path/to/scratch
|
|
"""
|
|
|
|
import argparse
|
|
import math
|
|
import pathlib
|
|
|
|
import yaml
|
|
|
|
HERE = pathlib.Path(__file__).parent.parent
|
|
|
|
GENERATED_HEADER = (
|
|
"# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
|
|
"# Source: oleks/ci-scripts manifests/{repo}.yaml + "
|
|
"generator/render_ondemand.py (oleks/ci-scripts#25).\n"
|
|
"# Re-render with: python3 generator/render_ondemand.py --out <repo>\n"
|
|
"\n"
|
|
)
|
|
|
|
# Default images. The orchestrator step needs `nix-estimate` + `kubectl` + `jq`
|
|
# on PATH; the build step is the repo's own build image. Both are manifest
|
|
# overridable (orchestrator_image / build_image).
|
|
DEFAULT_ORCHESTRATOR_IMAGE = "git.oleks.space/oleks/nix-ci:latest"
|
|
DEFAULT_BUILD_IMAGE = "git.oleks.space/oleks/nix-ci:latest"
|
|
|
|
# nix-estimate is invoked verbatim per the contract; the binary name is
|
|
# manifest-overridable (e.g. a `nix run <flake>#nix-estimate --` prefix) but
|
|
# defaults to the bare CLI assumed present in the orchestrator image.
|
|
DEFAULT_ESTIMATE_CMD = "nix-estimate"
|
|
DEFAULT_CLASS = "ci"
|
|
DEFAULT_WAIT_TIMEOUT = "1200s"
|
|
# Workspace-relative file the provision step writes the applied CR names into;
|
|
# read back by the reap step (Woodpecker mounts the workspace across steps,
|
|
# so this is how the ordered set crosses the step boundary).
|
|
ORDERED_FILE = "ordered-builders.txt"
|
|
|
|
|
|
def _when_block(manifest: dict) -> str:
|
|
"""The `when:` trigger. Defaults to the release-tag shape every other
|
|
renderer here uses; a manifest may instead pin `push` branches."""
|
|
branches = manifest.get("branches")
|
|
if branches:
|
|
joined = "[" + ", ".join(branches) + "]"
|
|
return f"when:\n - event: push\n branch: {joined}\n"
|
|
ref = manifest.get("tag_ref", "refs/tags/v*")
|
|
return f'when:\n - event: tag\n ref: "{ref}"\n'
|
|
|
|
|
|
def _provision_commands(manifest: dict) -> str:
|
|
estimate_cmd = manifest.get("estimate_cmd", DEFAULT_ESTIMATE_CMD)
|
|
attr = manifest["target"]
|
|
system = manifest["system"]
|
|
cls = manifest.get("provision_class", DEFAULT_CLASS)
|
|
wait_timeout = manifest.get("wait_timeout", DEFAULT_WAIT_TIMEOUT)
|
|
|
|
# Each list item is one command; Woodpecker concatenates a step's commands
|
|
# into a single `set -e` shell script, so vars set here persist within the
|
|
# step. Kept shellcheck-clean (quoted expansions, `read -r`).
|
|
lines = [
|
|
'echo "▸ arch=$(uname -m)"',
|
|
f'ATTR="{attr}"',
|
|
f'SYS="{system}"',
|
|
'echo "▸ estimating sizing for $ATTR ($SYS)"',
|
|
f'{estimate_cmd} "$ATTR" --system "$SYS" --json > estimate.json',
|
|
# Assert the versioned sizing block the contract froze (schema v1);
|
|
# -e makes jq exit non-zero (failing the step) on a false/absent match.
|
|
'jq -e ".sizing.version == 1" estimate.json > /dev/null',
|
|
'SIZING=$(jq -c ".sizing" estimate.json)',
|
|
'echo "▸ sizing: $SIZING"',
|
|
'echo "▸ ordering on-demand builder(s) (class ' + cls + ')"',
|
|
# The provision path emits N EphemeralBuilder docs; -o name echoes the
|
|
# applied CR identities (ephemeralbuilder.arbitrage.oleks.space/<name>)
|
|
# which we persist for the wait + reap.
|
|
f'{estimate_cmd} "$ATTR" --system "$SYS" --provision {cls} '
|
|
f"| kubectl apply -f - -o name | tee {ORDERED_FILE}",
|
|
'echo "▸ waiting for ordered builder(s) to become Ready"',
|
|
f'while read -r cr; do echo "▸ waiting $cr"; '
|
|
f'kubectl wait --for=condition=Ready "$cr" --timeout={wait_timeout}; '
|
|
f"done < {ORDERED_FILE}",
|
|
]
|
|
return "\n".join(f" - {line}" for line in lines)
|
|
|
|
|
|
def _reap_commands(manifest: dict) -> str:
|
|
lines = [
|
|
'echo "▸ arch=$(uname -m)"',
|
|
# --ignore-not-found: a CR the idle-park (parking.policy: cost) already
|
|
# reaped is not an error — this is the no-`|| true` equivalent.
|
|
f"if [ -f {ORDERED_FILE} ]; then "
|
|
f'while read -r cr; do echo "▸ reaping $cr"; '
|
|
f'kubectl delete "$cr" --ignore-not-found; '
|
|
f"done < {ORDERED_FILE}; "
|
|
f'else echo "▸ no {ORDERED_FILE}; nothing to reap"; fi',
|
|
]
|
|
return "\n".join(f" - {line}" for line in lines)
|
|
|
|
|
|
def _build_commands(manifest: dict) -> str:
|
|
run = manifest["build_run"]
|
|
lines = ['echo "▸ arch=$(uname -m)"']
|
|
lines += [line for line in run.rstrip("\n").splitlines()]
|
|
return "\n".join(f" - {line}" for line in lines)
|
|
|
|
|
|
def _kube_env_block(manifest: dict) -> str:
|
|
"""Optional scoped-kubeconfig secret (emmett consumes the fleet via a
|
|
scoped SA kubeconfig). Absent -> rely on the in-cluster ServiceAccount."""
|
|
secret = manifest.get("kubeconfig_secret")
|
|
if not secret:
|
|
return ""
|
|
return (
|
|
f" environment:\n KUBECONFIG_CONTENT:\n from_secret: {secret}\n"
|
|
)
|
|
|
|
|
|
def _memory_block(manifest: dict, indent: str) -> str:
|
|
mem_req = manifest.get("memory_request")
|
|
mem_lim = manifest.get("memory_limit")
|
|
if not (mem_req or mem_lim):
|
|
return ""
|
|
req = mem_req or mem_lim
|
|
lim = mem_lim or mem_req
|
|
return (
|
|
f"{indent}resources:\n"
|
|
f"{indent} requests:\n"
|
|
f"{indent} memory: {req}\n"
|
|
f"{indent} limits:\n"
|
|
f"{indent} memory: {lim}\n"
|
|
)
|
|
|
|
|
|
def render(manifest: dict) -> str:
|
|
repo = manifest["repo"]
|
|
arch = manifest["arch"]
|
|
orchestrator_image = manifest.get("orchestrator_image", DEFAULT_ORCHESTRATOR_IMAGE)
|
|
build_image = manifest.get("build_image", DEFAULT_BUILD_IMAGE)
|
|
|
|
header = GENERATED_HEADER.format(repo=repo)
|
|
when_block = _when_block(manifest)
|
|
provision_cmds = _provision_commands(manifest)
|
|
build_cmds = _build_commands(manifest)
|
|
reap_cmds = _reap_commands(manifest)
|
|
kube_env = _kube_env_block(manifest)
|
|
# The build step pins to the freshly-joined node via nodeSelector (Option B
|
|
# plain-worker: kubernetes.io/arch=<arch>). Any extra manifest node_selector
|
|
# keys are merged in (e.g. a per-order node label once the operator stamps
|
|
# one — reserved, not required for the MVP).
|
|
node_selector = {"kubernetes.io/arch": arch}
|
|
node_selector.update(manifest.get("node_selector", {}))
|
|
node_selector_lines = "\n".join(
|
|
f" {k}: {v}" for k, v in node_selector.items()
|
|
)
|
|
build_mem = _memory_block(manifest, " ")
|
|
|
|
# est_makespan_min echo helps derive the on-demand window; parking/maxDuration
|
|
# themselves are set by the estimator's --provision path, not here.
|
|
return f"""{header}labels:
|
|
arch: {arch}
|
|
|
|
{when_block}
|
|
steps:
|
|
# 1) ESTIMATE + ORDER + PROVIDE (oleks/ci-scripts#25): size the build, order
|
|
# an on-demand EphemeralBuilder of class ci, wait for the node to join.
|
|
- name: provision
|
|
image: {orchestrator_image}
|
|
{kube_env} commands:
|
|
{provision_cmds}
|
|
|
|
# 2) BUILD on the freshly-joined ephemeral node. Pinned via nodeSelector; the
|
|
# existing k8s-backend Woodpecker agent schedules this step-pod there.
|
|
- name: build
|
|
image: {build_image}
|
|
depends_on:
|
|
- provision
|
|
backend_options:
|
|
kubernetes:
|
|
nodeSelector:
|
|
{node_selector_lines}
|
|
{build_mem} commands:
|
|
{build_cmds}
|
|
|
|
# 3) REAP: delete the ordered CR(s) to tear the node down at end of window.
|
|
# runs_on success+failure so a failed build never strands a paid node.
|
|
- name: reap
|
|
image: {orchestrator_image}
|
|
depends_on:
|
|
- build
|
|
runs_on:
|
|
- success
|
|
- failure
|
|
{kube_env} commands:
|
|
{reap_cmds}
|
|
"""
|
|
|
|
|
|
def render_makespan_hint(est_makespan_min: float) -> int:
|
|
"""maxDuration = ceil(makespan * 1.5) minutes — mirrors the estimator's
|
|
--provision derivation so the two stay legible together. Provided as a
|
|
pure helper for tests/docs; the generated pipeline does not set it (the
|
|
estimator's --provision path stamps requirements.maxDuration itself)."""
|
|
return math.ceil(est_makespan_min * 1.5)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--out", type=pathlib.Path, required=True)
|
|
args = ap.parse_args()
|
|
|
|
manifests_dir = HERE / "manifests"
|
|
wrote_any = False
|
|
for path in sorted(manifests_dir.glob("*.yaml")):
|
|
manifest = yaml.safe_load(path.read_text())
|
|
if manifest.get("kind") != "ondemand":
|
|
continue
|
|
repo = manifest["repo"]
|
|
out_dir = args.out / repo
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
target = out_dir / ".woodpecker.yaml"
|
|
target.write_text(render(manifest))
|
|
print(f"wrote {target}")
|
|
wrote_any = True
|
|
if not wrote_any:
|
|
print("no ondemand manifests found (kind: ondemand)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|