Generator: explicit-clone matrix variant + raw header/comment fields for single kind (oleks/ci-scripts#2)

generator/render.py

@@ -41,6 +41,9 @@ def load_template(*parts):
 
 
 def render_matrix(manifest: dict) -> dict:
+    if manifest.get("explicit_clone"):
+        return render_matrix_explicit_clone(manifest)
+
     arches = manifest["arches"]
     image = manifest["image"]
     script = manifest.get("script", "ci/local.sh")
@@ -87,6 +90,57 @@ def render_matrix(manifest: dict) -> dict:
     return {"build.yaml": build, "manifest.yaml": mnf}
 
 
+def render_matrix_explicit_clone(manifest: dict) -> dict:
+    """Variant for repos that keep a Dockerfile + explicit clone + a fixed
+    (non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3.
+    Selected by `explicit_clone: true` in the manifest."""
+    arches = manifest["arches"]
+    image = manifest["image"]
+    script = manifest.get("script", "ci/local.sh")
+    var_name = manifest.get("var_name", "ARCH")
+    buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
+    git_host = manifest.get("git_host", "git.oleks.space")
+    fixed_label = manifest["fixed_label"]
+    node_selector = manifest["node_selector"]
+    timeout = manifest.get("timeout")
+    manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches))
+    manifest_buildkit_addr = manifest["manifest_buildkit_addr"]
+    build_header = manifest["build_header"].rstrip("\n")
+    manifest_header = manifest["manifest_header"].rstrip("\n")
+
+    arch_lines = [f"    - {a}" for a in arches]
+    for extra in manifest.get("commented_arches", []):
+        arch_lines.append(f"    # - {extra}")
+    arch_list = "\n".join(arch_lines)
+    timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
+
+    lines = "\n".join(f"          {k}: {v}" for k, v in node_selector.items())
+    node_selector_block = f"        nodeSelector:\n{lines}\n"
+
+    build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format(
+        fixed_label=fixed_label,
+        git_host=git_host,
+        timeout_block=timeout_block,
+        header=build_header,
+        var_name=var_name,
+        arch_list=arch_list,
+        image=image,
+        buildkit_prefix=buildkit_prefix,
+        script=script,
+        node_selector_block=node_selector_block,
+    )
+    mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format(
+        fixed_label=fixed_label,
+        script=script,
+        header=manifest_header,
+        image=image,
+        manifest_buildkit_addr=manifest_buildkit_addr,
+        manifest_arches_env=manifest_arches_env,
+        node_selector_block=node_selector_block,
+    )
+    return {"build.yaml": build, "manifest.yaml": mnf}
+
+
 def render_split(manifest: dict) -> dict:
     repo = manifest["repo"]
     git_host = manifest.get("git_host", "git.oleks.space")
@@ -113,12 +167,20 @@ def render_split(manifest: dict) -> dict:
     return out
 
 
+def _comment_block(text, indent):
+    if not text:
+        return ""
+    lines = text.rstrip("\n").splitlines()
+    return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)
+
+
 def render_single(manifest: dict) -> dict:
-    repo = manifest["repo"]
     arch = manifest["arch"]
     git_host = manifest.get("git_host", "git.oleks.space")
     branches = "[" + ", ".join(manifest["branches"]) + "]"
     buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
+    header = manifest["header"].rstrip("\n") + "\n"
+    label_lines = _comment_block(manifest.get("label_comment", ""), "  ") + f"  arch: {arch}"
 
     step_tmpl = load_template("single", "COMPONENT_STEP.tmpl")
     steps = []
@@ -126,19 +188,23 @@ def render_single(manifest: dict) -> dict:
         command = f"./ci/local.sh --arch {arch}"
         if c.get("component"):
             command += f" --component {c['component']}"
+        step_comment = _comment_block(c.get("step_comment", ""), "  ")
         steps.append(
             step_tmpl.format(
+                step_comment=step_comment,
                 step_name=c["name"],
                 image=manifest["image"],
+                buildkit_comment=_comment_block(c.get("buildkit_comment", ""), "      "),
                 buildkit_addr=buildkit_addr,
+                version_comment=_comment_block(c.get("version_comment", ""), "      "),
                 command=command,
                 arch=arch,
                 memory=c.get("memory", "4Gi"),
             )
         )
     container = load_template("single", "container.yaml.tmpl").format(
-        repo=repo,
+        header=header,
-        arch=arch,
+        label_lines=label_lines,
         branches=branches,
         git_host=git_host,
         component_steps="\n".join(steps),

generator/templates/matrix/build-explicit-clone.yaml.tmpl

@@ -0,0 +1,37 @@
+labels:
+  arch: {fixed_label}
+
+clone:
+  - name: clone
+    image: woodpeckerci/plugin-git
+    environment:
+      CI_NETRC_MACHINE: {git_host}
+      CI_NETRC_USERNAME: oleks
+      CI_NETRC_PASSWORD:
+        from_secret: gitea_clone_token
+      PLUGIN_TAGS: "false"
+      PLUGIN_DEPTH: "1"
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+{timeout_block}
+{header}
+matrix:
+  {var_name}:
+{arch_list}
+
+steps:
+  - name: build-${{{var_name}}}
+    image: {image}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "tcp://{buildkit_prefix}-${{{var_name}}}.infra.svc.cluster.local:1234"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - {script} --arch "${{{var_name}}}"
+    backend_options:
+      kubernetes:
+{node_selector_block}

generator/templates/matrix/manifest-explicit-clone.yaml.tmpl

@@ -0,0 +1,37 @@
+labels:
+  arch: {fixed_label}
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+depends_on:
+  - build
+
+# oleks/ci-scripts#9: run even if a per-arch matrix leg failed. Without
+# `runs_on`, Woodpecker skips a dependent workflow outright once any leg of
+# `build` fails, so a lone arch success would silently produce no image at
+# all. `{script} --manifest` itself must probe the registry for which
+# per-arch tags actually landed (not assume a fixed arch list, and not rely
+# on step/pipeline success status) and build the manifest from whatever
+# exists — a partial manifest beats none.
+runs_on:
+  - success
+  - failure
+
+{header}
+steps:
+  - name: manifest
+    image: {image}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "{manifest_buildkit_addr}"
+      MANIFEST_ARCHES: "{manifest_arches_env}"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - {script} --manifest
+    backend_options:
+      kubernetes:
+{node_selector_block}

generator/templates/single/COMPONENT_STEP.tmpl

@@ -1,12 +1,12 @@
-  - name: {step_name}
+{step_comment}  - name: {step_name}
     image: {image}
     environment:
       REGISTRY_TOKEN:
         from_secret: registry_token
-      BUILDKIT_ADDR: {buildkit_addr}
+{buildkit_comment}      BUILDKIT_ADDR: {buildkit_addr}
       PUBLISH: "1"
       BRANCH: "${{CI_COMMIT_BRANCH}}"
-      VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
+{version_comment}      VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
     commands:
       - echo "▸ arch=$(uname -m)"
       - {command}

generator/templates/single/container.yaml.tmpl

@@ -1,10 +1,6 @@
-# Build + push the {repo} container (cluster #196, emmett#44).
+{header}
-#
-# Keep-Dockerfile OCI image build; native single-arch buildkit, no matrix,
-# no manifest step. See {repo}/ci/local.sh for the actual build logic.
-
 labels:
-  arch: {arch}
+{label_lines}
 
 when:
   - event: push

manifests/csi-s3.yaml

@@ -1,10 +1,31 @@
 repo: csi-s3
 kind: matrix
-image: git.oleks.space/oleks/nix-ci
+explicit_clone: true
+image: git.oleks.space/oleks/nix-ci:latest
 script: ci/local.sh
+var_name: TARGET_ARCH
 arches: [amd64, arm64]
+commented_arches:
+  - "s390x  # disabled: no s390x builder available"
 buildkit_prefix: buildkit-rootless
+fixed_label: amd64
 timeout: 240
 node_selector:
   kubernetes.io/hostname: howard2404
 manifest_arches_env: "amd64 arm64"
+manifest_buildkit_addr: "tcp://buildkit-amd64.infra.svc.cluster.local:1234"
+build_header: |
+  # Thin wrapper (cluster #196 / emmett#44): CI runs the SAME ci/local.sh a
+  # developer runs, so CI and local can't drift. csi-s3 keeps its Dockerfile
+  # (s3fs-fuse from Alpine edge + a fetched geesefs binary are not cleanly
+  # Nix-expressible), so this is the keep-Dockerfile escape hatch, not a
+  # parity-lib conversion. The only CI-specific bits are env overrides:
+  #   BUILDKIT_ADDR -> the in-cluster native-arch remote buildkit
+  #   PUBLISH=1     -> actually push (local default is dry-run)
+  # Each native-arch node builds + pushes its own per-arch tags; the manifest
+  # workflow then combines them.
+manifest_header: |
+  # Thin wrapper (cluster #196 / emmett#44): the SAME ci/local.sh combines the
+  # per-arch tags into the multi-arch manifest. MANIFEST_ARCHES tracks which
+  # per-arch tags exist; re-add s390x here and in build.yaml when a builder
+  # returns.

manifests/emdash.kotkanagrilli.fi.yaml

@@ -0,0 +1,31 @@
+repo: emdash.kotkanagrilli.fi
+kind: single
+image: git.oleks.space/oleks/nix-ci:latest-arm64
+arch: arm64
+git_host: git.oleks.space
+branches: [develop, staging, production]
+header: |
+  # Build + push the EmDash app container (cluster #196, emmett#44).
+  #
+  # This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3,
+  # built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so
+  # it stays on `docker buildx` against the in-cluster buildkit. The build logic
+  # lives entirely in ci/local.sh so CI and a local run can't drift; this
+  # pipeline is a thin wrapper that just sets the CI-specific env and invokes it.
+  #
+  # Tagging (handled by ci/local.sh) — every build publishes three refs:
+  #   0.1.<pipeline>  immutable, audit.
+  #   <branch>        floating pointer Flux's ImagePolicy tracks (staging/
+  #                   production) → digest rewritten into the fleet repo → pod
+  #                   rolls. Do not drop.
+  #   <branch>-latest same image; chart image.tag cosmetic fallback.
+  # Only staging/production have an ImagePolicy, so only those move pods.
+label_comment: |
+  kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build
+  natively on arm64 — no cross-compile needed.
+components:
+  - name: build-and-push
+    buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes."
+    version_comment: |
+      Semver-shaped immutable tag, one per build. 0.1 is fixed (no real
+      semver discipline yet); the patch is the monotonic pipeline number.

manifests/the-eye.yaml

@@ -2,8 +2,38 @@ repo: the-eye
 kind: single
 image: git.oleks.space/oleks/nix-ci:latest-arm64
 arch: arm64
+git_host: git.oleks.space
 branches: [develop, staging, production]
+header: |
+  # Build + push the EmDash app container (cluster #196, emmett#44).
+  #
+  # This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3,
+  # built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so
+  # it stays on `docker buildx` against the in-cluster buildkit. The build logic
+  # lives entirely in ci/local.sh so CI and a local run can't drift; this
+  # pipeline is a thin wrapper that just sets the CI-specific env and invokes it.
+  #
+  # Tagging (handled by ci/local.sh) — every build publishes three refs:
+  #   0.1.<pipeline>  immutable, audit.
+  #   <branch>        floating pointer Flux's ImagePolicy tracks (staging/
+  #                   production) → digest rewritten into the fleet repo → pod
+  #                   rolls. Do not drop.
+  #   <branch>-latest same image; chart image.tag cosmetic fallback.
+  # Only staging/production have an ImagePolicy, so only those move pods.
+label_comment: |
+  kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build
+  natively on arm64 — no cross-compile needed.
 components:
   - name: build-and-push
+    buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes."
+    version_comment: |
+      Semver-shaped immutable tag, one per build. 0.1 is fixed (no real
+      semver discipline yet); the patch is the monotonic pipeline number.
   - name: build-and-push-grpc
     component: grpc
+    step_comment: |
+      gRPC WorkspaceService image (#80). Same Dockerfile (the `grpc` target), same
+      builder / registry / auth / tagging logic as the web build above — only
+      --component grpc differs, which suffixes the published refs with -grpc
+      (e.g. staging-grpc). Reuses the immutable per-pipeline $VERSION (0.1.<n>),
+      published as 0.1.<n>-grpc.