Add thin-pipeline render PREP scripts for #16/#17 (not wired in, no repo pushes)

generator/render_thin_fleet.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""Render thin ci-fleet-publish-based pipelines for the fleet family
+(oleks/ci-scripts#16): the 10 identical-fleet repos in manifest.yaml's
+`repos:` list and the 5 thin-parity repos in `thin_repos:`.
+
+This is a PREP/PLANNING script (per team-lead direction on #16/#17): it
+renders into a local --out dir for diffing against the current generated
+`.woodpecker.yaml` files. It does NOT write into ~/projects/building/ and is
+not wired into generate.py's main() — that wiring (and the actual repo
+migration) happens once the diffs are reviewed and the tag-scheme question
+is settled.
+
+Behavior-preservation approach: every piece of shell logic that generate.py's
+render()/render_thin() used to inline (resolv.conf, nix.conf, attic
+lifecycle, redis/sccache env, token remap, doctor gate) either maps to a
+PLUGIN_* setting the plugin already implements (nameserver, cores, max_jobs,
+cache), or gets folded into the `run:` command as an explicit shell
+one-liner/block — the plugin's `publish` flow just execs `bash -lc
+"$PLUGIN_RUN"`, so anything that was a `commands:` line before can still be
+a line in `run:`.
+
+Usage:
+    python3 generator/render_thin_fleet.py --out /path/to/scratch
+"""
+
+import argparse
+import pathlib
+
+import yaml
+
+HERE = pathlib.Path(__file__).parent.parent
+PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1"
+
+GENERATED_HEADER = (
+    "# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
+    "# Source: oleks/ci-scripts manifest.yaml + generator/render_thin_fleet.py"
+    " (oleks/ci-scripts#16).\n"
+    "# Re-render with: python3 generator/render_thin_fleet.py --out <repo>\n"
+    "\n"
+)
+
+NAMESERVER = "169.254.20.10"
+
+
+def render_identical(repo: dict) -> str:
+    """The 10 `repos:` entries (oleks/ci-scripts#1 family)."""
+    attic = repo.get("attic", True)
+    redis_cache = repo.get("redis_cache", False)
+    cores = repo.get("cores")
+    max_jobs = repo.get("max_jobs", False)
+
+    run_lines = []
+    if redis_cache:
+        run_lines += [
+            "export SCCACHE_REDIS_ENDPOINT=tcp://redis.default.svc.cluster.local:6379",
+            "export SCCACHE_REDIS_PASSWORD=$REDIS_PASSWORD",
+            "export CCACHE_REMOTE_STORAGE=redis://redis.default.svc.cluster.local:6379",
+            "export CCACHE_REMOTE_ONLY=1",
+        ]
+    run_lines.append("PUBLISH=1 nix run .#publish")
+    run_block = "\n".join(f"        {line}" for line in run_lines)
+
+    settings_lines = [
+        f'      run: |\n{run_block}',
+        f'      cache: "{str(attic).lower()}"',
+        f'      nameserver: "{NAMESERVER}"',
+    ]
+    if max_jobs:
+        settings_lines.append('      max_jobs: "true"')
+    if cores:
+        settings_lines.append(f'      cores: "{cores}"')
+    settings_block = "\n".join(settings_lines)
+
+    env_lines = ["      GITEA_CLONE_TOKEN:", "        from_secret: gitea_clone_token"]
+    if attic:
+        env_lines += ["      ATTIC_TOKEN:", "        from_secret: attic_token"]
+    if redis_cache:
+        env_lines += ["      REDIS_PASSWORD:", "        from_secret: redis_password"]
+    env_block = "\n".join(env_lines)
+
+    return f"""{GENERATED_HEADER}labels:
+  arch: {repo["arch"]}
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+steps:
+  - name: publish
+    image: {PLUGIN_IMAGE}
+    environment:
+{env_block}
+    settings:
+{settings_block}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: {repo["memory_request"]}
+          limits:
+            memory: {repo["memory_limit"]}
+"""
+
+
+def render_thin_parity(repo: dict) -> str:
+    """The 5 `thin_repos:` entries (oleks/ci-scripts#10 family)."""
+    doctor = repo.get("doctor", False)
+    token_direct = repo.get("token_direct", True)
+    publish_style = repo.get("publish_style", "publish1")
+    tags_full = repo.get("tags_full", False)
+    gitea_clone_token = repo.get("gitea_clone_token", True)
+
+    run_lines = []
+    if doctor:
+        run_lines.append(
+            "nix run git+https://git.oleks.space/oleks/parity-lib#pipeline-doctor -- . --strict"
+        )
+    if not token_direct:
+        run_lines.append('export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"')
+    if publish_style == "publish1":
+        run_lines.append("PUBLISH=1 nix run .#publish")
+    else:
+        run_lines.append("nix run .#publish-s390x -- --publish")
+    run_block = "\n".join(f"        {line}" for line in run_lines)
+
+    settings_lines = [
+        f'      run: |\n{run_block}',
+        '      cache: "false"',  # oleks/ci-scripts#10: this family never used
+        # the inline attic accelerator (dead ATTIC_TOKEN in some repos) —
+        # cache: false preserves that, doesn't silently turn caching on.
+        f'      nameserver: "{NAMESERVER}"',
+    ]
+    settings_block = "\n".join(settings_lines)
+
+    env_lines = []
+    if gitea_clone_token:
+        env_lines += ["      GITEA_CLONE_TOKEN:", "        from_secret: gitea_clone_token"]
+    if token_direct:
+        env_lines += ["      REGISTRY_TOKEN:", "        from_secret: registry_token"]
+    else:
+        env_lines += ["      CI_REGISTRY_TOKEN:", "        from_secret: registry_token"]
+    env_block = "\n".join(env_lines) if env_lines else "      {}"
+
+    # oleks/ci-scripts#10 dead_secrets (ATTIC_TOKEN/REDIS_PASSWORD declared
+    # but never referenced) are deliberately DROPPED here, not carried
+    # forward — they were unused in the underlying command either way.
+
+    clone_note = ""
+    if tags_full:
+        clone_note = (
+            "\n# NOTE: original PLUGIN_TAGS=true/PLUGIN_DEPTH=0 (full clone incl. tags)\n"
+            "# is a `clone:` block override the thin contract doesn't have a settings\n"
+            "# equivalent for yet — needs resolving before this repo migrates (see\n"
+            "# migration report).\n"
+        )
+
+    return f"""{GENERATED_HEADER}labels:
+  arch: {repo["arch"]}
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+{clone_note}
+steps:
+  - name: publish
+    image: {PLUGIN_IMAGE}
+    environment:
+{env_block}
+    settings:
+{settings_block}
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            memory: {repo["memory_request"]}
+          limits:
+            memory: {repo["memory_limit"]}
+"""
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", type=pathlib.Path, required=True)
+    args = ap.parse_args()
+    args.out.mkdir(parents=True, exist_ok=True)
+
+    manifest = yaml.safe_load((HERE / "manifest.yaml").read_text())
+    for repo in manifest["repos"]:
+        name = pathlib.Path(repo["dir"]).name
+        target = args.out / f"{name}.woodpecker.yaml"
+        target.write_text(render_identical(repo))
+        print(f"wrote {target}")
+    for repo in manifest.get("thin_repos", []):
+        name = pathlib.Path(repo["dir"]).name
+        target = args.out / f"{name}.woodpecker.yaml"
+        target.write_text(render_thin_parity(repo))
+        print(f"wrote {target}")
+
+
+if __name__ == "__main__":
+    main()

generator/render_thin_quartet.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Render thin ci-fleet-publish-based pipelines for the quartet family
+(oleks/ci-scripts#17) — PREP/PLANNING script, not wired into anything yet.
+
+SCOPE NOTE (surfaced to team-lead, not silently assumed): #17 lists 6 repos,
+but manifests/*.yaml shows only 2 are actually `kind: matrix` (real buildx
+per-arch-build + manifest-consolidate quartets): xonsh-image, csi-s3. The
+other 4 are shaped differently and don't map onto the plugin's
+buildx-build/buildx-manifest flow at all:
+  - the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch build,
+    no manifest step. These want `flow: publish` (single-step, like the
+    fleet family), not the buildx quartet flow.
+  - flake-hub, woodpecker-peek: kind: split — separate per-arch files using
+    `nix run .#publish-*`, no docker buildx involved at all. Same as above,
+    `flow: publish` per arch file, not buildx-build/buildx-manifest.
+This script only renders the 2 true matrix-kind repos. The other 4 need a
+render_thin_fleet.py-style treatment instead — flagged as follow-up, not
+done here.
+
+For xonsh-image/csi-s3, migrating to the plugin's buildx-build/buildx-manifest
+flow means the plugin's generic `docker buildx build --push <context>`
+(building the repo's own Dockerfile directly) REPLACES what `ci/local.sh
+--arch/--manifest` used to do internally. This is a bigger behavioral swap
+than the fleet family's migration (which just moves inline shell into
+`run:`) — needs explicit confirmation that ci/local.sh's Dockerfile build is
+the ONLY thing it does before this ships (see report).
+
+Usage:
+    python3 generator/render_thin_quartet.py --out /path/to/scratch
+"""
+
+import argparse
+import pathlib
+
+import yaml
+
+HERE = pathlib.Path(__file__).parent.parent
+PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1"
+
+GENERATED_HEADER = (
+    "# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
+    "# Source: oleks/ci-scripts manifests/{repo}.yaml + "
+    "generator/render_thin_quartet.py (oleks/ci-scripts#17).\n"
+    "# Re-render with: python3 generator/render_thin_quartet.py --out <repo>/.woodpecker\n"
+    "\n"
+)
+
+
+def render_build(manifest: dict, arch: str, image_name: str) -> str:
+    node_selector = manifest.get("node_selector")
+    backend_options_block = ""
+    if node_selector:
+        lines = "\n".join(f"          {k}: {v}" for k, v in node_selector.items())
+        backend_options_block = (
+            "    backend_options:\n      kubernetes:\n        nodeSelector:\n" + lines + "\n"
+        )
+
+    return f"""{GENERATED_HEADER}labels:
+  arch: {arch}
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+steps:
+  - name: build-{arch}
+    image: {PLUGIN_IMAGE}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+    settings:
+      flow: buildx-build
+      image_name: "{image_name}"
+      tag_scheme: "{{version}}-{{arch}}"
+{backend_options_block}"""
+
+
+def render_manifest(manifest: dict, arches: list, image_name: str) -> str:
+    platforms = ",".join(arches)
+    return f"""{GENERATED_HEADER}when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+depends_on:
+{chr(10).join(f"  - build-{a}" for a in arches)}
+
+# oleks/ci-scripts#9: run even if a per-arch build failed — the plugin's
+# buildx-manifest flow itself probes the registry for which per-arch tags
+# actually landed and builds the manifest from whatever exists.
+runs_on:
+  - success
+  - failure
+
+steps:
+  - name: manifest
+    image: {PLUGIN_IMAGE}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+    settings:
+      flow: buildx-manifest
+      image_name: "{image_name}"
+      platforms: "{platforms}"
+      tag_scheme: "{{version}}-{{arch}}"
+"""
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--out", type=pathlib.Path, required=True)
+    args = ap.parse_args()
+
+    manifests_dir = HERE / "manifests"
+    for path in sorted(manifests_dir.glob("*.yaml")):
+        manifest = yaml.safe_load(path.read_text())
+        if manifest.get("kind") != "matrix":
+            print(f"skip {manifest['repo']} (kind={manifest.get('kind')}, not a buildx quartet)")
+            continue
+        repo = manifest["repo"]
+        image_name = f"git.oleks.space/oleks/{repo}"
+        arches = manifest["arches"]
+        out_dir = args.out / repo
+        out_dir.mkdir(parents=True, exist_ok=True)
+        for arch in arches:
+            target = out_dir / f"build-{arch}.yaml"
+            target.write_text(render_build(manifest, arch, image_name))
+            print(f"wrote {target}")
+        target = out_dir / "manifest.yaml"
+        target.write_text(render_manifest(manifest, arches, image_name))
+        print(f"wrote {target}")
+
+
+if __name__ == "__main__":
+    main()