From 38bb59e3d034d78bf852cc8bb28792faf5b7688d Mon Sep 17 00:00:00 2001 From: repo-worker Date: Thu, 9 Jul 2026 19:21:56 +0300 Subject: [PATCH] generator: thin-render single/split quartet kinds (oleks/ci-scripts#19) Extend render_thin_quartet.py to render the 4 single/split quartet repos (the-eye, emdash.kotkanagrilli.fi, flake-hub, woodpecker-peek) onto the plugin's flow: publish shape, alongside the already-done matrix kind. Behavior diffs vs each repo's current .woodpecker files posted on #19. --- generator/render_thin_quartet.py | 243 +++++++++++++++++++++++++++---- 1 file changed, 214 insertions(+), 29 deletions(-) diff --git a/generator/render_thin_quartet.py b/generator/render_thin_quartet.py index 4b812cf..819c313 100644 --- a/generator/render_thin_quartet.py +++ b/generator/render_thin_quartet.py @@ -1,21 +1,27 @@ #!/usr/bin/env python3 """Render thin ci-fleet-publish-based pipelines for the quartet family -(oleks/ci-scripts#17) — PREP/PLANNING script, not wired into anything yet. +(oleks/ci-scripts#17, #19) — PREP/PLANNING script, not wired into anything yet. -SCOPE NOTE (surfaced to team-lead, not silently assumed): #17 lists 6 repos, -but manifests/*.yaml shows only 2 are actually `kind: matrix` (real buildx -per-arch-build + manifest-consolidate quartets): xonsh-image, csi-s3. The -other 4 are shaped differently and don't map onto the plugin's -buildx-build/buildx-manifest flow at all: - - the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch build, - no manifest step. These want `flow: publish` (single-step, like the - fleet family), not the buildx quartet flow. - - flake-hub, woodpecker-peek: kind: split — separate per-arch files using - `nix run .#publish-*`, no docker buildx involved at all. Same as above, - `flow: publish` per arch file, not buildx-build/buildx-manifest. -This script only renders the 2 true matrix-kind repos. The other 4 need a -render_thin_fleet.py-style treatment instead — flagged as follow-up, not -done here. +SHAPES (from manifests/*.yaml, `kind:`): + - xonsh-image, csi-s3: kind: matrix — real buildx per-arch-build + + manifest-consolidate quartets. Maps onto the plugin's + buildx-build/buildx-manifest flow. (oleks/ci-scripts#17, done first.) + - the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch + `docker buildx` build via `ci/local.sh`, no manifest step (the-eye also + has a second `grpc` component, same shape, second step in the same + file). Maps onto the plugin's `flow: publish` — same single-step shape + render_thin_fleet.py uses for the fleet family. + - flake-hub, woodpecker-peek: kind: split — separate per-arch files + (amd64.yaml/arm64.yaml), `nix run .#publish[-]` against parity-lib's + attic-closure archetype, no docker buildx at all. Also maps onto + `flow: publish`, one thin step per arch file — per-arch file layout is + kept (NOT collapsed into one matrix file): the two arches build against + different node-bound PVCs and were already split for that reason (see + manifests/flake-hub.yaml header), a constraint the thin migration doesn't + change. + +(oleks/ci-scripts#19 implements the single/split renderers below; the matrix +renderer was already done for #17.) For xonsh-image/csi-s3, migrating to the plugin's buildx-build/buildx-manifest flow means the plugin's generic `docker buildx build --push ` @@ -25,6 +31,36 @@ than the fleet family's migration (which just moves inline shell into `run:`) — needs explicit confirmation that ci/local.sh's Dockerfile build is the ONLY thing it does before this ships (see report). +For the-eye/emdash (single) and flake-hub/woodpecker-peek (split), the +existing pipelines already invoke a repo-owned script (`ci/local.sh`, +`nix run .#publish-*`) as their entire build logic — same shape as the fleet +family's `run:` migration, not the bigger buildx-quartet swap above. + +Behavior-preservation notes (single/split): + - The manual `echo "▸ arch=$(uname -m)"` command line is dropped from + `run:` — entrypoint.sh already prints that banner unconditionally before + any flow runs (line ~35), so keeping it in `run:` too would just double + it. Same convention render_thin_fleet.py already uses. + - `clone:` step overrides are dropped, matching the precedent already set + for the fleet family (render_thin_fleet.py) and the matrix renderer + below: Woodpecker's implicit clone is judged equivalent for the + PLUGIN_TAGS=false/PLUGIN_DEPTH=1 shape all four of these repos use. None + of the 4 use the `tags_full` (full-history) shape render_thin_fleet.py + flagged as an unresolved gap — this repo set doesn't hit that gap. + - `backend_options.kubernetes.nodeSelector`/`resources`/`labels` (the + per-pod tracing labels: commit-branch/commit-sha/commit-tag/ + pipeline-number) are NOT manifest-parametrized inputs today — render.py's + single/split templates hardcode them per component/leg. They are + preserved here as the same fixed shape, still populated from the + manifest's `arch`/leg key. + - ATTIC_TOKEN handling: split legs already have `attic_token` wired as a + plain `environment:` secret (consumed by nixos-ci-entrypoint's own + ATTIC_TOKEN handling today). The plugin's `attic_start`/`attic_stop` + (entrypoint.sh) already replaces that lifecycle when `cache: "true"` — + same mapping render_thin_fleet.py uses for the identical-repos family. + the-eye/emdash never reference ATTIC_TOKEN today, so they get + `cache: "false"`, not silently turned on. + Usage: python3 generator/render_thin_quartet.py --out /path/to/scratch """ @@ -40,7 +76,7 @@ PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1" GENERATED_HEADER = ( "# GENERATED by oleks/ci-scripts — do not hand-edit.\n" "# Source: oleks/ci-scripts manifests/{repo}.yaml + " - "generator/render_thin_quartet.py (oleks/ci-scripts#17).\n" + "generator/render_thin_quartet.py (oleks/ci-scripts#17, #19).\n" "# Re-render with: python3 generator/render_thin_quartet.py --out /.woodpecker\n" "\n" ) @@ -52,7 +88,9 @@ def render_build(manifest: dict, arch: str, image_name: str) -> str: if node_selector: lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items()) backend_options_block = ( - " backend_options:\n kubernetes:\n nodeSelector:\n" + lines + "\n" + " backend_options:\n kubernetes:\n nodeSelector:\n" + + lines + + "\n" ) return f"""{GENERATED_HEADER}labels: @@ -105,6 +143,139 @@ steps: """ +def _comment_block(text: str, indent: str) -> str: + if not text: + return "" + lines = text.rstrip("\n").splitlines() + return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines) + + +def render_single_step(component: dict, arch: str) -> str: + """One thin `flow: publish` step for a `kind: single` component + (the-eye's build-and-push / build-and-push-grpc, emdash's build-and-push). + Mirrors templates/single/COMPONENT_STEP.tmpl's fixed env/backend_options + shape, swapping the `commands:` block for a `settings: {{run, flow, cache}}`. + """ + command = f"./ci/local.sh --arch {arch}" + if component.get("component"): + command += f" --component {component['component']}" + step_comment = _comment_block(component.get("step_comment", ""), " ") + buildkit_comment = _comment_block(component.get("buildkit_comment", ""), " ") + version_comment = _comment_block(component.get("version_comment", ""), " ") + memory = component.get("memory", "4Gi") + buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234" + + return f"""{step_comment} - name: {component["name"]} + image: {PLUGIN_IMAGE} + environment: + REGISTRY_TOKEN: + from_secret: registry_token +{buildkit_comment} BUILDKIT_ADDR: {buildkit_addr} + PUBLISH: "1" + BRANCH: "${{CI_COMMIT_BRANCH}}" +{version_comment} VERSION: "0.1.${{CI_PIPELINE_NUMBER}}" + settings: + run: | + {command} + flow: publish + cache: "false" + backend_options: + kubernetes: + nodeSelector: + kubernetes.io/arch: {arch} + resources: + requests: + memory: {memory} + limits: + memory: {memory} + labels: + commit-branch: "${{CI_COMMIT_BRANCH}}" + commit-sha: "${{CI_COMMIT_SHA}}" + pipeline-number: "${{CI_PIPELINE_NUMBER}}" +""" + + +def render_single(manifest: dict) -> dict: + """`kind: single` (the-eye, emdash.kotkanagrilli.fi) → one file, + one-or-more thin `flow: publish` steps (the-eye has a second `grpc` + component alongside the web build).""" + arch = manifest["arch"] + branches = "[" + ", ".join(manifest["branches"]) + "]" + header = manifest["header"].rstrip("\n") + "\n" + label_lines = ( + _comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}" + ) + + steps = "\n".join( + render_single_step(component, arch) for component in manifest["components"] + ) + + body = f"""{header} +labels: +{label_lines} + +when: + - event: push + branch: {branches} + +steps: +{steps}""" + return {"container.yaml": GENERATED_HEADER.format(repo=manifest["repo"]) + body} + + +def render_split_leg(manifest: dict, arch: str, leg: dict) -> str: + """One thin `flow: publish` file for a `kind: split` leg (flake-hub's + amd64.yaml/arm64.yaml, woodpecker-peek's amd64.yaml/arm64.yaml).""" + setup_script = manifest.get("setup_script", "ci/setup.sh") + header = leg["header"].rstrip("\n") + "\n" + entrypoint_comment = _comment_block(leg.get("entrypoint_comment", ""), " ") + tag_event_block = ( + "\n - event: tag\n" if manifest.get("when_tag_event", True) else "\n" + ) + publish_flag = " -- --publish" if leg.get("publish_flag", True) else "" + + run_lines = f" sh {setup_script}\n" + if entrypoint_comment: + run_lines += entrypoint_comment + run_lines += f" PUBLISH=1 nix run .#{leg['app']}{publish_flag}\n" + + return f"""{GENERATED_HEADER.format(repo=manifest["repo"])}{header} +when: + - event: push + branch: main{tag_event_block} +steps: + - name: build-{arch} + image: {PLUGIN_IMAGE} + environment: + ATTIC_TOKEN: + from_secret: attic_token + GITEA_CLONE_TOKEN: + from_secret: gitea_clone_token + settings: + run: | +{run_lines} flow: publish + cache: "true" + backend_options: + kubernetes: + nodeSelector: + kubernetes.io/arch: {arch} + labels: + commit-tag: "${{CI_COMMIT_TAG}}" + commit-branch: "${{CI_COMMIT_BRANCH}}" + pipeline-number: "${{CI_PIPELINE_NUMBER}}" +""" + + +def render_split(manifest: dict) -> dict: + """`kind: split` (flake-hub, woodpecker-peek) → one file per arch leg, + kept separate (never collapsed into a single matrix file — the two arches + build against different node-bound PVCs, see manifests/flake-hub.yaml).""" + return { + f"{arch}.yaml": render_split_leg(manifest, arch, leg) + for arch, leg in manifest["legs"].items() + } + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--out", type=pathlib.Path, required=True) @@ -113,21 +284,35 @@ def main(): manifests_dir = HERE / "manifests" for path in sorted(manifests_dir.glob("*.yaml")): manifest = yaml.safe_load(path.read_text()) - if manifest.get("kind") != "matrix": - print(f"skip {manifest['repo']} (kind={manifest.get('kind')}, not a buildx quartet)") - continue + kind = manifest.get("kind") repo = manifest["repo"] - image_name = f"git.oleks.space/oleks/{repo}" - arches = manifest["arches"] out_dir = args.out / repo - out_dir.mkdir(parents=True, exist_ok=True) - for arch in arches: - target = out_dir / f"build-{arch}.yaml" - target.write_text(render_build(manifest, arch, image_name)) + + if kind == "matrix": + image_name = f"git.oleks.space/oleks/{repo}" + arches = manifest["arches"] + out_dir.mkdir(parents=True, exist_ok=True) + for arch in arches: + target = out_dir / f"build-{arch}.yaml" + target.write_text(render_build(manifest, arch, image_name)) + print(f"wrote {target}") + target = out_dir / "manifest.yaml" + target.write_text(render_manifest(manifest, arches, image_name)) print(f"wrote {target}") - target = out_dir / "manifest.yaml" - target.write_text(render_manifest(manifest, arches, image_name)) - print(f"wrote {target}") + elif kind == "single": + out_dir.mkdir(parents=True, exist_ok=True) + for name, body in render_single(manifest).items(): + target = out_dir / name + target.write_text(body) + print(f"wrote {target}") + elif kind == "split": + out_dir.mkdir(parents=True, exist_ok=True) + for name, body in render_split(manifest).items(): + target = out_dir / name + target.write_text(body) + print(f"wrote {target}") + else: + print(f"skip {repo} (kind={kind}, unknown)") if __name__ == "__main__":