253 lines
9.1 KiB
Python
253 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Render Woodpecker buildx-quartet pipelines from a per-repo manifest.
|
|
|
|
oleks/ci-scripts#2 — one generator, three supported shapes (`kind:` in the
|
|
manifest), because the repos in scope aren't actually one shape:
|
|
|
|
matrix - single build.yaml with a `matrix: {ARCH: [...]}` + manifest.yaml
|
|
combining the per-arch tags (xonsh-image, csi-s3, ...).
|
|
split - separate per-arch files (amd64.yaml, arm64.yaml, ...), no
|
|
manifest step — used for attic-closure "nix run .#publish-*"
|
|
publishes rather than docker buildx (woodpecker-peek, ...).
|
|
single - one container.yaml, one native-arch build (+ optional extra
|
|
components like a grpc image), no matrix, no manifest
|
|
(the-eye, emdash.kotkanagrilli.fi).
|
|
|
|
Usage:
|
|
python3 generator/render.py manifests/<repo>.yaml --out /path/to/repo/.woodpecker
|
|
|
|
Every rendered file gets a "GENERATED by oleks/ci-scripts — do not hand-edit"
|
|
header. Re-running is idempotent: same manifest in, byte-identical files out.
|
|
"""
|
|
import argparse
|
|
import pathlib
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
HERE = pathlib.Path(__file__).parent
|
|
TEMPLATES = HERE / "templates"
|
|
|
|
GENERATED_HEADER = (
|
|
"# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
|
|
"# Source manifest: {manifest_path}\n"
|
|
"# Regenerate: python3 generator/render.py {manifest_path} --out <repo>/.woodpecker\n"
|
|
"\n"
|
|
)
|
|
|
|
|
|
def load_template(*parts):
|
|
return (TEMPLATES / pathlib.Path(*parts)).read_text()
|
|
|
|
|
|
def render_matrix(manifest: dict) -> dict:
|
|
if manifest.get("explicit_clone"):
|
|
return render_matrix_explicit_clone(manifest)
|
|
|
|
arches = manifest["arches"]
|
|
image = manifest["image"]
|
|
script = manifest.get("script", "ci/local.sh")
|
|
buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
|
|
node_selector = manifest.get("node_selector")
|
|
timeout = manifest.get("timeout")
|
|
manifest_arches_env = manifest.get("manifest_arches_env")
|
|
|
|
arch_list = "\n".join(f" - {a}" for a in arches)
|
|
timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
|
|
if node_selector:
|
|
lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items())
|
|
node_selector_block = f" nodeSelector:\n{lines}\n"
|
|
node_selector_top_block = (
|
|
" backend_options:\n kubernetes:\n nodeSelector:\n" + lines + "\n"
|
|
)
|
|
else:
|
|
node_selector_block = ""
|
|
node_selector_top_block = ""
|
|
|
|
manifest_extra_env = ""
|
|
if manifest_arches_env:
|
|
manifest_extra_env = f' MANIFEST_ARCHES: "{manifest_arches_env}"\n'
|
|
if node_selector:
|
|
manifest_extra_env = (
|
|
f' BUILDKIT_ADDR: "tcp://{buildkit_prefix}-{arches[0]}.infra.svc.cluster.local:1234"\n'
|
|
+ manifest_extra_env
|
|
)
|
|
|
|
build = load_template("matrix", "build.yaml.tmpl").format(
|
|
arch_list=arch_list,
|
|
timeout_block=timeout_block,
|
|
script=script,
|
|
image=image,
|
|
buildkit_prefix=buildkit_prefix,
|
|
node_selector_block=node_selector_block,
|
|
)
|
|
mnf = load_template("matrix", "manifest.yaml.tmpl").format(
|
|
script=script,
|
|
image=image,
|
|
manifest_extra_env=manifest_extra_env,
|
|
node_selector_top_block=node_selector_top_block,
|
|
)
|
|
return {"build.yaml": build, "manifest.yaml": mnf}
|
|
|
|
|
|
def render_matrix_explicit_clone(manifest: dict) -> dict:
|
|
"""Variant for repos that keep a Dockerfile + explicit clone + a fixed
|
|
(non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3.
|
|
Selected by `explicit_clone: true` in the manifest."""
|
|
arches = manifest["arches"]
|
|
image = manifest["image"]
|
|
script = manifest.get("script", "ci/local.sh")
|
|
var_name = manifest.get("var_name", "ARCH")
|
|
buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
|
|
git_host = manifest.get("git_host", "git.oleks.space")
|
|
fixed_label = manifest["fixed_label"]
|
|
node_selector = manifest["node_selector"]
|
|
timeout = manifest.get("timeout")
|
|
manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches))
|
|
manifest_buildkit_addr = manifest["manifest_buildkit_addr"]
|
|
build_header = manifest["build_header"].rstrip("\n")
|
|
manifest_header = manifest["manifest_header"].rstrip("\n")
|
|
|
|
arch_lines = [f" - {a}" for a in arches]
|
|
for extra in manifest.get("commented_arches", []):
|
|
arch_lines.append(f" # - {extra}")
|
|
arch_list = "\n".join(arch_lines)
|
|
timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
|
|
|
|
lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items())
|
|
node_selector_block = f" nodeSelector:\n{lines}\n"
|
|
|
|
build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format(
|
|
fixed_label=fixed_label,
|
|
git_host=git_host,
|
|
timeout_block=timeout_block,
|
|
header=build_header,
|
|
var_name=var_name,
|
|
arch_list=arch_list,
|
|
image=image,
|
|
buildkit_prefix=buildkit_prefix,
|
|
script=script,
|
|
node_selector_block=node_selector_block,
|
|
)
|
|
mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format(
|
|
fixed_label=fixed_label,
|
|
script=script,
|
|
header=manifest_header,
|
|
image=image,
|
|
manifest_buildkit_addr=manifest_buildkit_addr,
|
|
manifest_arches_env=manifest_arches_env,
|
|
node_selector_block=node_selector_block,
|
|
)
|
|
return {"build.yaml": build, "manifest.yaml": mnf}
|
|
|
|
|
|
def render_split(manifest: dict) -> dict:
|
|
repo = manifest["repo"]
|
|
git_host = manifest.get("git_host", "git.oleks.space")
|
|
setup_script = manifest.get("setup_script", "ci/setup.sh")
|
|
out = {}
|
|
tmpl = load_template("split", "PERARCH.yaml.tmpl")
|
|
for arch, leg in manifest["legs"].items():
|
|
header = leg["header"].rstrip("\n") + "\n"
|
|
entrypoint_comment = "".join(
|
|
f" # {line}\n" if line else " #\n"
|
|
for line in leg["entrypoint_comment"].rstrip("\n").splitlines()
|
|
)
|
|
out[f"{arch}.yaml"] = tmpl.format(
|
|
repo=repo,
|
|
nix_system=leg["nix_system"],
|
|
arch=arch,
|
|
header=header,
|
|
entrypoint_comment=entrypoint_comment,
|
|
git_host=git_host,
|
|
image=leg["image"],
|
|
setup_script=setup_script,
|
|
app=leg["app"],
|
|
)
|
|
return out
|
|
|
|
|
|
def _comment_block(text, indent):
|
|
if not text:
|
|
return ""
|
|
lines = text.rstrip("\n").splitlines()
|
|
return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)
|
|
|
|
|
|
def render_single(manifest: dict) -> dict:
|
|
arch = manifest["arch"]
|
|
git_host = manifest.get("git_host", "git.oleks.space")
|
|
branches = "[" + ", ".join(manifest["branches"]) + "]"
|
|
buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
|
|
header = manifest["header"].rstrip("\n") + "\n"
|
|
label_lines = _comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}"
|
|
|
|
step_tmpl = load_template("single", "COMPONENT_STEP.tmpl")
|
|
steps = []
|
|
for c in manifest["components"]:
|
|
command = f"./ci/local.sh --arch {arch}"
|
|
if c.get("component"):
|
|
command += f" --component {c['component']}"
|
|
step_comment = _comment_block(c.get("step_comment", ""), " ")
|
|
steps.append(
|
|
step_tmpl.format(
|
|
step_comment=step_comment,
|
|
step_name=c["name"],
|
|
image=manifest["image"],
|
|
buildkit_comment=_comment_block(c.get("buildkit_comment", ""), " "),
|
|
buildkit_addr=buildkit_addr,
|
|
version_comment=_comment_block(c.get("version_comment", ""), " "),
|
|
command=command,
|
|
arch=arch,
|
|
memory=c.get("memory", "4Gi"),
|
|
)
|
|
)
|
|
container = load_template("single", "container.yaml.tmpl").format(
|
|
header=header,
|
|
label_lines=label_lines,
|
|
branches=branches,
|
|
git_host=git_host,
|
|
component_steps="\n".join(steps),
|
|
)
|
|
return {"container.yaml": container}
|
|
|
|
|
|
RENDERERS = {"matrix": render_matrix, "split": render_split, "single": render_single}
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("manifest", type=pathlib.Path)
|
|
ap.add_argument("--out", type=pathlib.Path, required=True, help="target .woodpecker/ dir")
|
|
ap.add_argument("--check", action="store_true", help="don't write, just diff against --out")
|
|
args = ap.parse_args()
|
|
|
|
manifest = yaml.safe_load(args.manifest.read_text())
|
|
kind = manifest["kind"]
|
|
if kind not in RENDERERS:
|
|
sys.exit(f"unknown kind: {kind}")
|
|
|
|
files = RENDERERS[kind](manifest)
|
|
header = GENERATED_HEADER.format(manifest_path=args.manifest)
|
|
|
|
args.out.mkdir(parents=True, exist_ok=True)
|
|
changed = False
|
|
for name, body in files.items():
|
|
target = args.out / name
|
|
content = header + body
|
|
if args.check:
|
|
existing = target.read_text() if target.exists() else ""
|
|
if existing != content:
|
|
changed = True
|
|
print(f"DIFF: {target}")
|
|
else:
|
|
target.write_text(content)
|
|
print(f"wrote {target}")
|
|
|
|
if args.check and changed:
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|