diff --git a/generator/render_thin_fleet.py b/generator/render_thin_fleet.py new file mode 100644 index 0000000..3b25698 --- /dev/null +++ b/generator/render_thin_fleet.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Render thin ci-fleet-publish-based pipelines for the fleet family +(oleks/ci-scripts#16): the 10 identical-fleet repos in manifest.yaml's +`repos:` list and the 5 thin-parity repos in `thin_repos:`. + +This is a PREP/PLANNING script (per team-lead direction on #16/#17): it +renders into a local --out dir for diffing against the current generated +`.woodpecker.yaml` files. It does NOT write into ~/projects/building/ and is +not wired into generate.py's main() — that wiring (and the actual repo +migration) happens once the diffs are reviewed and the tag-scheme question +is settled. + +Behavior-preservation approach: every piece of shell logic that generate.py's +render()/render_thin() used to inline (resolv.conf, nix.conf, attic +lifecycle, redis/sccache env, token remap, doctor gate) either maps to a +PLUGIN_* setting the plugin already implements (nameserver, cores, max_jobs, +cache), or gets folded into the `run:` command as an explicit shell +one-liner/block — the plugin's `publish` flow just execs `bash -lc +"$PLUGIN_RUN"`, so anything that was a `commands:` line before can still be +a line in `run:`. + +Usage: + python3 generator/render_thin_fleet.py --out /path/to/scratch +""" + +import argparse +import pathlib + +import yaml + +HERE = pathlib.Path(__file__).parent.parent +PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1" + +GENERATED_HEADER = ( + "# GENERATED by oleks/ci-scripts — do not hand-edit.\n" + "# Source: oleks/ci-scripts manifest.yaml + generator/render_thin_fleet.py" + " (oleks/ci-scripts#16).\n" + "# Re-render with: python3 generator/render_thin_fleet.py --out \n" + "\n" +) + +NAMESERVER = "169.254.20.10" + + +def render_identical(repo: dict) -> str: + """The 10 `repos:` entries (oleks/ci-scripts#1 family).""" + attic = repo.get("attic", True) + redis_cache = repo.get("redis_cache", False) + cores = repo.get("cores") + max_jobs = repo.get("max_jobs", False) + + run_lines = [] + if redis_cache: + run_lines += [ + "export SCCACHE_REDIS_ENDPOINT=tcp://redis.default.svc.cluster.local:6379", + "export SCCACHE_REDIS_PASSWORD=$REDIS_PASSWORD", + "export CCACHE_REMOTE_STORAGE=redis://redis.default.svc.cluster.local:6379", + "export CCACHE_REMOTE_ONLY=1", + ] + run_lines.append("PUBLISH=1 nix run .#publish") + run_block = "\n".join(f" {line}" for line in run_lines) + + settings_lines = [ + f' run: |\n{run_block}', + f' cache: "{str(attic).lower()}"', + f' nameserver: "{NAMESERVER}"', + ] + if max_jobs: + settings_lines.append(' max_jobs: "true"') + if cores: + settings_lines.append(f' cores: "{cores}"') + settings_block = "\n".join(settings_lines) + + env_lines = [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"] + if attic: + env_lines += [" ATTIC_TOKEN:", " from_secret: attic_token"] + if redis_cache: + env_lines += [" REDIS_PASSWORD:", " from_secret: redis_password"] + env_block = "\n".join(env_lines) + + return f"""{GENERATED_HEADER}labels: + arch: {repo["arch"]} + +when: + - event: tag + ref: "refs/tags/v*" + +steps: + - name: publish + image: {PLUGIN_IMAGE} + environment: +{env_block} + settings: +{settings_block} + backend_options: + kubernetes: + resources: + requests: + memory: {repo["memory_request"]} + limits: + memory: {repo["memory_limit"]} +""" + + +def render_thin_parity(repo: dict) -> str: + """The 5 `thin_repos:` entries (oleks/ci-scripts#10 family).""" + doctor = repo.get("doctor", False) + token_direct = repo.get("token_direct", True) + publish_style = repo.get("publish_style", "publish1") + tags_full = repo.get("tags_full", False) + gitea_clone_token = repo.get("gitea_clone_token", True) + + run_lines = [] + if doctor: + run_lines.append( + "nix run git+https://git.oleks.space/oleks/parity-lib#pipeline-doctor -- . --strict" + ) + if not token_direct: + run_lines.append('export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"') + if publish_style == "publish1": + run_lines.append("PUBLISH=1 nix run .#publish") + else: + run_lines.append("nix run .#publish-s390x -- --publish") + run_block = "\n".join(f" {line}" for line in run_lines) + + settings_lines = [ + f' run: |\n{run_block}', + ' cache: "false"', # oleks/ci-scripts#10: this family never used + # the inline attic accelerator (dead ATTIC_TOKEN in some repos) — + # cache: false preserves that, doesn't silently turn caching on. + f' nameserver: "{NAMESERVER}"', + ] + settings_block = "\n".join(settings_lines) + + env_lines = [] + if gitea_clone_token: + env_lines += [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"] + if token_direct: + env_lines += [" REGISTRY_TOKEN:", " from_secret: registry_token"] + else: + env_lines += [" CI_REGISTRY_TOKEN:", " from_secret: registry_token"] + env_block = "\n".join(env_lines) if env_lines else " {}" + + # oleks/ci-scripts#10 dead_secrets (ATTIC_TOKEN/REDIS_PASSWORD declared + # but never referenced) are deliberately DROPPED here, not carried + # forward — they were unused in the underlying command either way. + + clone_note = "" + if tags_full: + clone_note = ( + "\n# NOTE: original PLUGIN_TAGS=true/PLUGIN_DEPTH=0 (full clone incl. tags)\n" + "# is a `clone:` block override the thin contract doesn't have a settings\n" + "# equivalent for yet — needs resolving before this repo migrates (see\n" + "# migration report).\n" + ) + + return f"""{GENERATED_HEADER}labels: + arch: {repo["arch"]} + +when: + - event: tag + ref: "refs/tags/v*" +{clone_note} +steps: + - name: publish + image: {PLUGIN_IMAGE} + environment: +{env_block} + settings: +{settings_block} + backend_options: + kubernetes: + resources: + requests: + memory: {repo["memory_request"]} + limits: + memory: {repo["memory_limit"]} +""" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", type=pathlib.Path, required=True) + args = ap.parse_args() + args.out.mkdir(parents=True, exist_ok=True) + + manifest = yaml.safe_load((HERE / "manifest.yaml").read_text()) + for repo in manifest["repos"]: + name = pathlib.Path(repo["dir"]).name + target = args.out / f"{name}.woodpecker.yaml" + target.write_text(render_identical(repo)) + print(f"wrote {target}") + for repo in manifest.get("thin_repos", []): + name = pathlib.Path(repo["dir"]).name + target = args.out / f"{name}.woodpecker.yaml" + target.write_text(render_thin_parity(repo)) + print(f"wrote {target}") + + +if __name__ == "__main__": + main() diff --git a/generator/render_thin_quartet.py b/generator/render_thin_quartet.py new file mode 100644 index 0000000..4b812cf --- /dev/null +++ b/generator/render_thin_quartet.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Render thin ci-fleet-publish-based pipelines for the quartet family +(oleks/ci-scripts#17) — PREP/PLANNING script, not wired into anything yet. + +SCOPE NOTE (surfaced to team-lead, not silently assumed): #17 lists 6 repos, +but manifests/*.yaml shows only 2 are actually `kind: matrix` (real buildx +per-arch-build + manifest-consolidate quartets): xonsh-image, csi-s3. The +other 4 are shaped differently and don't map onto the plugin's +buildx-build/buildx-manifest flow at all: + - the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch build, + no manifest step. These want `flow: publish` (single-step, like the + fleet family), not the buildx quartet flow. + - flake-hub, woodpecker-peek: kind: split — separate per-arch files using + `nix run .#publish-*`, no docker buildx involved at all. Same as above, + `flow: publish` per arch file, not buildx-build/buildx-manifest. +This script only renders the 2 true matrix-kind repos. The other 4 need a +render_thin_fleet.py-style treatment instead — flagged as follow-up, not +done here. + +For xonsh-image/csi-s3, migrating to the plugin's buildx-build/buildx-manifest +flow means the plugin's generic `docker buildx build --push ` +(building the repo's own Dockerfile directly) REPLACES what `ci/local.sh +--arch/--manifest` used to do internally. This is a bigger behavioral swap +than the fleet family's migration (which just moves inline shell into +`run:`) — needs explicit confirmation that ci/local.sh's Dockerfile build is +the ONLY thing it does before this ships (see report). + +Usage: + python3 generator/render_thin_quartet.py --out /path/to/scratch +""" + +import argparse +import pathlib + +import yaml + +HERE = pathlib.Path(__file__).parent.parent +PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1" + +GENERATED_HEADER = ( + "# GENERATED by oleks/ci-scripts — do not hand-edit.\n" + "# Source: oleks/ci-scripts manifests/{repo}.yaml + " + "generator/render_thin_quartet.py (oleks/ci-scripts#17).\n" + "# Re-render with: python3 generator/render_thin_quartet.py --out /.woodpecker\n" + "\n" +) + + +def render_build(manifest: dict, arch: str, image_name: str) -> str: + node_selector = manifest.get("node_selector") + backend_options_block = "" + if node_selector: + lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items()) + backend_options_block = ( + " backend_options:\n kubernetes:\n nodeSelector:\n" + lines + "\n" + ) + + return f"""{GENERATED_HEADER}labels: + arch: {arch} + +when: + - event: tag + ref: "refs/tags/v*" + +steps: + - name: build-{arch} + image: {PLUGIN_IMAGE} + environment: + REGISTRY_TOKEN: + from_secret: registry_token + settings: + flow: buildx-build + image_name: "{image_name}" + tag_scheme: "{{version}}-{{arch}}" +{backend_options_block}""" + + +def render_manifest(manifest: dict, arches: list, image_name: str) -> str: + platforms = ",".join(arches) + return f"""{GENERATED_HEADER}when: + - event: tag + ref: "refs/tags/v*" + +depends_on: +{chr(10).join(f" - build-{a}" for a in arches)} + +# oleks/ci-scripts#9: run even if a per-arch build failed — the plugin's +# buildx-manifest flow itself probes the registry for which per-arch tags +# actually landed and builds the manifest from whatever exists. +runs_on: + - success + - failure + +steps: + - name: manifest + image: {PLUGIN_IMAGE} + environment: + REGISTRY_TOKEN: + from_secret: registry_token + settings: + flow: buildx-manifest + image_name: "{image_name}" + platforms: "{platforms}" + tag_scheme: "{{version}}-{{arch}}" +""" + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", type=pathlib.Path, required=True) + args = ap.parse_args() + + manifests_dir = HERE / "manifests" + for path in sorted(manifests_dir.glob("*.yaml")): + manifest = yaml.safe_load(path.read_text()) + if manifest.get("kind") != "matrix": + print(f"skip {manifest['repo']} (kind={manifest.get('kind')}, not a buildx quartet)") + continue + repo = manifest["repo"] + image_name = f"git.oleks.space/oleks/{repo}" + arches = manifest["arches"] + out_dir = args.out / repo + out_dir.mkdir(parents=True, exist_ok=True) + for arch in arches: + target = out_dir / f"build-{arch}.yaml" + target.write_text(render_build(manifest, arch, image_name)) + print(f"wrote {target}") + target = out_dir / "manifest.yaml" + target.write_text(render_manifest(manifest, arches, image_name)) + print(f"wrote {target}") + + +if __name__ == "__main__": + main()