ci-scripts/generator/render.py

#!/usr/bin/env python3
"""Render Woodpecker buildx-quartet pipelines from a per-repo manifest.

oleks/ci-scripts#2 — one generator, three supported shapes (`kind:` in the
manifest), because the repos in scope aren't actually one shape:

  matrix  - single build.yaml with a `matrix: {ARCH: [...]}` + manifest.yaml
            combining the per-arch tags (xonsh-image, csi-s3, ...).
  split   - separate per-arch files (amd64.yaml, arm64.yaml, ...), no
            manifest step — used for attic-closure "nix run .#publish-*"
            publishes rather than docker buildx (woodpecker-peek, ...).
  single  - one container.yaml, one native-arch build (+ optional extra
            components like a grpc image), no matrix, no manifest
            (the-eye, emdash.kotkanagrilli.fi).

Usage:
    python3 generator/render.py manifests/<repo>.yaml --out /path/to/repo/.woodpecker

Every rendered file gets a "GENERATED by oleks/ci-scripts — do not hand-edit"
header. Re-running is idempotent: same manifest in, byte-identical files out.
"""

import argparse
import pathlib
import sys

import yaml

HERE = pathlib.Path(__file__).parent
TEMPLATES = HERE / "templates"

GENERATED_HEADER = (
    "# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
    "# Source manifest: {manifest_path}\n"
    "# Regenerate: python3 generator/render.py {manifest_path} --out <repo>/.woodpecker\n"
    "\n"
)


def load_template(*parts):
    return (TEMPLATES / pathlib.Path(*parts)).read_text()


def render_matrix(manifest: dict) -> dict:
    if manifest.get("explicit_clone"):
        return render_matrix_explicit_clone(manifest)

    arches = manifest["arches"]
    image = manifest["image"]
    script = manifest.get("script", "ci/local.sh")
    buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
    node_selector = manifest.get("node_selector")
    timeout = manifest.get("timeout")
    manifest_arches_env = manifest.get("manifest_arches_env")

    arch_list = "\n".join(f"    - {a}" for a in arches)
    timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
    if node_selector:
        lines = "\n".join(f"          {k}: {v}" for k, v in node_selector.items())
        node_selector_block = f"        nodeSelector:\n{lines}\n"
        node_selector_top_block = (
            "    backend_options:\n      kubernetes:\n        nodeSelector:\n"
            + lines
            + "\n"
        )
    else:
        node_selector_block = ""
        node_selector_top_block = ""

    manifest_extra_env = ""
    if manifest_arches_env:
        manifest_extra_env = f'      MANIFEST_ARCHES: "{manifest_arches_env}"\n'
    if node_selector:
        manifest_extra_env = (
            f'      BUILDKIT_ADDR: "tcp://{buildkit_prefix}-{arches[0]}.infra.svc.cluster.local:1234"\n'
            + manifest_extra_env
        )

    build = load_template("matrix", "build.yaml.tmpl").format(
        arch_list=arch_list,
        timeout_block=timeout_block,
        script=script,
        image=image,
        buildkit_prefix=buildkit_prefix,
        node_selector_block=node_selector_block,
    )
    mnf = load_template("matrix", "manifest.yaml.tmpl").format(
        script=script,
        image=image,
        manifest_extra_env=manifest_extra_env,
        node_selector_top_block=node_selector_top_block,
    )
    return {"build.yaml": build, "manifest.yaml": mnf}


def render_matrix_explicit_clone(manifest: dict) -> dict:
    """Variant for repos that keep a Dockerfile + explicit clone + a fixed
    (non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3.
    Selected by `explicit_clone: true` in the manifest."""
    arches = manifest["arches"]
    image = manifest["image"]
    script = manifest.get("script", "ci/local.sh")
    var_name = manifest.get("var_name", "ARCH")
    buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
    git_host = manifest.get("git_host", "git.oleks.space")
    fixed_label = manifest["fixed_label"]
    node_selector = manifest["node_selector"]
    timeout = manifest.get("timeout")
    manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches))
    manifest_buildkit_addr = manifest["manifest_buildkit_addr"]
    build_header = manifest["build_header"].rstrip("\n")
    manifest_header = manifest["manifest_header"].rstrip("\n")

    arch_lines = [f"    - {a}" for a in arches]
    for extra in manifest.get("commented_arches", []):
        arch_lines.append(f"    # - {extra}")
    arch_list = "\n".join(arch_lines)
    timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""

    lines = "\n".join(f"          {k}: {v}" for k, v in node_selector.items())
    node_selector_block = f"        nodeSelector:\n{lines}\n"

    build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format(
        fixed_label=fixed_label,
        git_host=git_host,
        timeout_block=timeout_block,
        header=build_header,
        var_name=var_name,
        arch_list=arch_list,
        image=image,
        buildkit_prefix=buildkit_prefix,
        script=script,
        node_selector_block=node_selector_block,
    )
    mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format(
        fixed_label=fixed_label,
        script=script,
        header=manifest_header,
        image=image,
        manifest_buildkit_addr=manifest_buildkit_addr,
        manifest_arches_env=manifest_arches_env,
        node_selector_block=node_selector_block,
    )
    return {"build.yaml": build, "manifest.yaml": mnf}


def render_split(manifest: dict) -> dict:
    repo = manifest["repo"]
    git_host = manifest.get("git_host", "git.oleks.space")
    setup_script = manifest.get("setup_script", "ci/setup.sh")
    out = {}
    tmpl = load_template("split", "PERARCH.yaml.tmpl")
    for arch, leg in manifest["legs"].items():
        header = leg["header"].rstrip("\n") + "\n"
        entrypoint_comment = "".join(
            f"      # {line}\n" if line else "      #\n"
            for line in leg["entrypoint_comment"].rstrip("\n").splitlines()
        )
        tag_event_line = (
            "  - event: tag\n" if manifest.get("when_tag_event", True) else ""
        )
        publish_flag = " -- --publish" if leg.get("publish_flag", True) else ""
        default_setup_comment = (
            f"nix.conf substituters + {git_host} netrc (to fetch the parity-lib\n"
            "flake input). Same bootstrap as flake-hub/ci/setup.sh."
        )
        setup_comment = _comment_block(
            leg.get("setup_comment", default_setup_comment), "      "
        )
        out[f"{arch}.yaml"] = tmpl.format(
            repo=repo,
            nix_system=leg["nix_system"],
            arch=arch,
            header=header,
            entrypoint_comment=entrypoint_comment,
            tag_event_line=tag_event_line,
            setup_comment=setup_comment,
            git_host=git_host,
            image=leg["image"],
            setup_script=setup_script,
            app=leg["app"],
            publish_flag=publish_flag,
        )
    return out


def _comment_block(text, indent):
    if not text:
        return ""
    lines = text.rstrip("\n").splitlines()
    return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)


def render_single(manifest: dict) -> dict:
    arch = manifest["arch"]
    git_host = manifest.get("git_host", "git.oleks.space")
    branches = "[" + ", ".join(manifest["branches"]) + "]"
    buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
    header = manifest["header"].rstrip("\n") + "\n"
    label_lines = (
        _comment_block(manifest.get("label_comment", ""), "  ") + f"  arch: {arch}"
    )

    step_tmpl = load_template("single", "COMPONENT_STEP.tmpl")
    steps = []
    for c in manifest["components"]:
        command = f"./ci/local.sh --arch {arch}"
        if c.get("component"):
            command += f" --component {c['component']}"
        step_comment = _comment_block(c.get("step_comment", ""), "  ")
        steps.append(
            step_tmpl.format(
                step_comment=step_comment,
                step_name=c["name"],
                image=manifest["image"],
                buildkit_comment=_comment_block(
                    c.get("buildkit_comment", ""), "      "
                ),
                buildkit_addr=buildkit_addr,
                version_comment=_comment_block(c.get("version_comment", ""), "      "),
                command=command,
                arch=arch,
                memory=c.get("memory", "4Gi"),
            )
        )
    container = load_template("single", "container.yaml.tmpl").format(
        header=header,
        label_lines=label_lines,
        branches=branches,
        git_host=git_host,
        component_steps="\n".join(steps),
    )
    return {"container.yaml": container}


RENDERERS = {"matrix": render_matrix, "split": render_split, "single": render_single}


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("manifest", type=pathlib.Path)
    ap.add_argument(
        "--out", type=pathlib.Path, required=True, help="target .woodpecker/ dir"
    )
    ap.add_argument(
        "--check", action="store_true", help="don't write, just diff against --out"
    )
    args = ap.parse_args()

    manifest = yaml.safe_load(args.manifest.read_text())
    kind = manifest["kind"]
    if kind not in RENDERERS:
        sys.exit(f"unknown kind: {kind}")

    files = RENDERERS[kind](manifest)
    header = GENERATED_HEADER.format(manifest_path=args.manifest)

    args.out.mkdir(parents=True, exist_ok=True)
    changed = False
    for name, body in files.items():
        target = args.out / name
        content = header + body
        if args.check:
            existing = target.read_text() if target.exists() else ""
            if existing != content:
                changed = True
                print(f"DIFF: {target}")
        else:
            target.write_text(content)
            print(f"wrote {target}")

    if args.check and changed:
        sys.exit(1)


if __name__ == "__main__":
    main()