#!/usr/bin/env python3 """Render thin ci-fleet-publish-based pipelines for the quartet family (oleks/ci-scripts#17, #19) — PREP/PLANNING script, not wired into anything yet. SHAPES (from manifests/*.yaml, `kind:`): - xonsh-image, csi-s3: kind: matrix — real buildx per-arch-build + manifest-consolidate quartets. Maps onto the plugin's buildx-build/buildx-manifest flow. (oleks/ci-scripts#17, done first.) - the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch `docker buildx` build via `ci/local.sh`, no manifest step (the-eye also has a second `grpc` component, same shape, second step in the same file). Maps onto the plugin's `flow: publish` — same single-step shape render_thin_fleet.py uses for the fleet family. - flake-hub, woodpecker-peek: kind: split — separate per-arch files (amd64.yaml/arm64.yaml), `nix run .#publish[-]` against parity-lib's attic-closure archetype, no docker buildx at all. Also maps onto `flow: publish`, one thin step per arch file — per-arch file layout is kept (NOT collapsed into one matrix file): the two arches build against different node-bound PVCs and were already split for that reason (see manifests/flake-hub.yaml header), a constraint the thin migration doesn't change. (oleks/ci-scripts#19 implements the single/split renderers below; the matrix renderer was already done for #17.) For xonsh-image/csi-s3, migrating to the plugin's buildx-build/buildx-manifest flow means the plugin's generic `docker buildx build --push ` (building the repo's own Dockerfile directly) REPLACES what `ci/local.sh --arch/--manifest` used to do internally. This is a bigger behavioral swap than the fleet family's migration (which just moves inline shell into `run:`) — needs explicit confirmation that ci/local.sh's Dockerfile build is the ONLY thing it does before this ships (see report). `kind: matrix` manifests must set `image_name:` explicitly, verified against the live Gitea package registry (oleks/ci-scripts#22). It is NEVER inferred from the manifest's `repo:`/dir field — `xonsh-image`'s repo field and its actually-published image (`oleks/xonsh`, confirmed live) genuinely differ; inferring one from the other silently pushes a migrated pipeline to a different, wrong (and possibly empty) image path. For the-eye/emdash (single) and flake-hub/woodpecker-peek (split), the existing pipelines already invoke a repo-owned script (`ci/local.sh`, `nix run .#publish-*`) as their entire build logic — same shape as the fleet family's `run:` migration, not the bigger buildx-quartet swap above. Behavior-preservation notes (single/split): - The manual `echo "▸ arch=$(uname -m)"` command line is dropped from `run:` — entrypoint.sh already prints that banner unconditionally before any flow runs (line ~35), so keeping it in `run:` too would just double it. Same convention render_thin_fleet.py already uses. - `clone:` step overrides are dropped, matching the precedent already set for the fleet family (render_thin_fleet.py) and the matrix renderer below: Woodpecker's implicit clone is judged equivalent for the PLUGIN_TAGS=false/PLUGIN_DEPTH=1 shape all four of these repos use. None of the 4 use the `tags_full` (full-history) shape render_thin_fleet.py flagged as an unresolved gap — this repo set doesn't hit that gap. - `backend_options.kubernetes.nodeSelector`/`resources`/`labels` (the per-pod tracing labels: commit-branch/commit-sha/commit-tag/ pipeline-number) are NOT manifest-parametrized inputs today — render.py's single/split templates hardcode them per component/leg. They are preserved here as the same fixed shape, still populated from the manifest's `arch`/leg key. - ATTIC_TOKEN handling: split legs already have `attic_token` wired as a plain `environment:` secret (consumed by nixos-ci-entrypoint's own ATTIC_TOKEN handling today). The plugin's `attic_start`/`attic_stop` (entrypoint.sh) already replaces that lifecycle when `cache: "true"` — same mapping render_thin_fleet.py uses for the identical-repos family. the-eye/emdash never reference ATTIC_TOKEN today, so they get `cache: "false"`, not silently turned on. Usage: python3 generator/render_thin_quartet.py --out /path/to/scratch """ import argparse import pathlib import yaml HERE = pathlib.Path(__file__).parent.parent PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1" GENERATED_HEADER = ( "# GENERATED by oleks/ci-scripts — do not hand-edit.\n" "# Source: oleks/ci-scripts manifests/{repo}.yaml + " "generator/render_thin_quartet.py (oleks/ci-scripts#17, #19).\n" "# Re-render with: python3 generator/render_thin_quartet.py --out /.woodpecker\n" "\n" ) def render_build(manifest: dict, arch: str, image_name: str) -> str: node_selector = manifest.get("node_selector") backend_options_block = "" if node_selector: lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items()) backend_options_block = ( " backend_options:\n kubernetes:\n nodeSelector:\n" + lines + "\n" ) # oleks/ci-scripts#21: opt-in only — reproduces a repo's own ci/local.sh # tag normalization (strip leading "v" + trailing "-N"). Default off # (entrypoint.sh's resolve_version() default matches today's raw-tag # behavior), set per-manifest via `tag_strip_v: true`. tag_strip_v_line = ( ' tag_strip_v: "true"\n' if manifest.get("tag_strip_v") else "" ) return f"""{GENERATED_HEADER}labels: arch: {arch} when: - event: tag ref: "refs/tags/v*" steps: - name: build-{arch} image: {PLUGIN_IMAGE} environment: REGISTRY_TOKEN: from_secret: registry_token settings: flow: buildx-build image_name: "{image_name}" tag_scheme: "{{version}}-{{arch}}" {tag_strip_v_line}{backend_options_block}""" def render_manifest(manifest: dict, arches: list, image_name: str) -> str: platforms = ",".join(arches) tag_strip_v_line = ( ' tag_strip_v: "true"\n' if manifest.get("tag_strip_v") else "" ) return f"""{GENERATED_HEADER}when: - event: tag ref: "refs/tags/v*" depends_on: {chr(10).join(f" - build-{a}" for a in arches)} # oleks/ci-scripts#9: run even if a per-arch build failed — the plugin's # buildx-manifest flow itself probes the registry for which per-arch tags # actually landed and builds the manifest from whatever exists. runs_on: - success - failure steps: - name: manifest image: {PLUGIN_IMAGE} environment: REGISTRY_TOKEN: from_secret: registry_token settings: flow: buildx-manifest image_name: "{image_name}" platforms: "{platforms}" tag_scheme: "{{version}}-{{arch}}" {tag_strip_v_line}""" def _comment_block(text: str, indent: str) -> str: if not text: return "" lines = text.rstrip("\n").splitlines() return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines) def render_single_step(component: dict, arch: str) -> str: """One thin `flow: publish` step for a `kind: single` component (the-eye's build-and-push / build-and-push-grpc, emdash's build-and-push). Mirrors templates/single/COMPONENT_STEP.tmpl's fixed env/backend_options shape, swapping the `commands:` block for a `settings: {{run, flow, cache}}`. """ command = f"./ci/local.sh --arch {arch}" if component.get("component"): command += f" --component {component['component']}" step_comment = _comment_block(component.get("step_comment", ""), " ") buildkit_comment = _comment_block(component.get("buildkit_comment", ""), " ") version_comment = _comment_block(component.get("version_comment", ""), " ") memory = component.get("memory", "4Gi") buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234" return f"""{step_comment} - name: {component["name"]} image: {PLUGIN_IMAGE} environment: REGISTRY_TOKEN: from_secret: registry_token {buildkit_comment} BUILDKIT_ADDR: {buildkit_addr} PUBLISH: "1" BRANCH: "${{CI_COMMIT_BRANCH}}" {version_comment} VERSION: "0.1.${{CI_PIPELINE_NUMBER}}" settings: run: | {command} flow: publish cache: "false" backend_options: kubernetes: nodeSelector: kubernetes.io/arch: {arch} resources: requests: memory: {memory} limits: memory: {memory} labels: commit-branch: "${{CI_COMMIT_BRANCH}}" commit-sha: "${{CI_COMMIT_SHA}}" pipeline-number: "${{CI_PIPELINE_NUMBER}}" """ def render_single(manifest: dict) -> dict: """`kind: single` (the-eye, emdash.kotkanagrilli.fi) → one file, one-or-more thin `flow: publish` steps (the-eye has a second `grpc` component alongside the web build).""" arch = manifest["arch"] branches = "[" + ", ".join(manifest["branches"]) + "]" header = manifest["header"].rstrip("\n") + "\n" label_lines = ( _comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}" ) steps = "\n".join( render_single_step(component, arch) for component in manifest["components"] ) body = f"""{header} labels: {label_lines} when: - event: push branch: {branches} steps: {steps}""" return {"container.yaml": GENERATED_HEADER.format(repo=manifest["repo"]) + body} def render_split_leg(manifest: dict, arch: str, leg: dict) -> str: """One thin `flow: publish` file for a `kind: split` leg (flake-hub's amd64.yaml/arm64.yaml, woodpecker-peek's amd64.yaml/arm64.yaml).""" setup_script = manifest.get("setup_script", "ci/setup.sh") header = leg["header"].rstrip("\n") + "\n" entrypoint_comment = _comment_block(leg.get("entrypoint_comment", ""), " ") tag_event_block = ( "\n - event: tag\n" if manifest.get("when_tag_event", True) else "\n" ) publish_flag = " -- --publish" if leg.get("publish_flag", True) else "" run_lines = f" sh {setup_script}\n" if entrypoint_comment: run_lines += entrypoint_comment run_lines += f" PUBLISH=1 nix run .#{leg['app']}{publish_flag}\n" return f"""{GENERATED_HEADER.format(repo=manifest["repo"])}{header} when: - event: push branch: main{tag_event_block} steps: - name: build-{arch} image: {PLUGIN_IMAGE} environment: ATTIC_TOKEN: from_secret: attic_token GITEA_CLONE_TOKEN: from_secret: gitea_clone_token settings: run: | {run_lines} flow: publish cache: "true" backend_options: kubernetes: nodeSelector: kubernetes.io/arch: {arch} labels: commit-tag: "${{CI_COMMIT_TAG}}" commit-branch: "${{CI_COMMIT_BRANCH}}" pipeline-number: "${{CI_PIPELINE_NUMBER}}" """ def render_split(manifest: dict) -> dict: """`kind: split` (flake-hub, woodpecker-peek) → one file per arch leg, kept separate (never collapsed into a single matrix file — the two arches build against different node-bound PVCs, see manifests/flake-hub.yaml).""" return { f"{arch}.yaml": render_split_leg(manifest, arch, leg) for arch, leg in manifest["legs"].items() } def main(): ap = argparse.ArgumentParser() ap.add_argument("--out", type=pathlib.Path, required=True) args = ap.parse_args() manifests_dir = HERE / "manifests" for path in sorted(manifests_dir.glob("*.yaml")): manifest = yaml.safe_load(path.read_text()) kind = manifest.get("kind") repo = manifest["repo"] out_dir = args.out / repo if kind == "matrix": # oleks/ci-scripts#22: image_name must be explicit in the # manifest, never inferred from the repo/dir name — the two # silently diverge for xonsh-image (repo field "xonsh-image", # actually-published image "oleks/xonsh") and inference would # have pushed a migrated pipeline to an empty, wrong image path. image_name = manifest.get("image_name") if not image_name: raise SystemExit( f"manifest for {repo!r} (kind=matrix) is missing required " "'image_name' field (oleks/ci-scripts#22: must be explicit, " "verified against the live package registry — do not infer " "it from the repo/dir name)" ) arches = manifest["arches"] out_dir.mkdir(parents=True, exist_ok=True) for arch in arches: target = out_dir / f"build-{arch}.yaml" target.write_text(render_build(manifest, arch, image_name)) print(f"wrote {target}") target = out_dir / "manifest.yaml" target.write_text(render_manifest(manifest, arches, image_name)) print(f"wrote {target}") elif kind == "single": out_dir.mkdir(parents=True, exist_ok=True) for name, body in render_single(manifest).items(): target = out_dir / name target.write_text(body) print(f"wrote {target}") elif kind == "split": out_dir.mkdir(parents=True, exist_ok=True) for name, body in render_split(manifest).items(): target = out_dir / name target.write_text(body) print(f"wrote {target}") else: print(f"skip {repo} (kind={kind}, unknown)") if __name__ == "__main__": main()