Generator: explicit-clone matrix variant + raw header/comment fields for single kind (oleks/ci-scripts#2)

generator/render.py

@@ -41,6 +41,9 @@ def load_template(*parts):
 
 
 def render_matrix(manifest: dict) -> dict:
+    if manifest.get("explicit_clone"):
+        return render_matrix_explicit_clone(manifest)
+
     arches = manifest["arches"]
     image = manifest["image"]
     script = manifest.get("script", "ci/local.sh")
@@ -87,6 +90,57 @@ def render_matrix(manifest: dict) -> dict:
     return {"build.yaml": build, "manifest.yaml": mnf}
 
 
+def render_matrix_explicit_clone(manifest: dict) -> dict:
+    """Variant for repos that keep a Dockerfile + explicit clone + a fixed
+    (non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3.
+    Selected by `explicit_clone: true` in the manifest."""
+    arches = manifest["arches"]
+    image = manifest["image"]
+    script = manifest.get("script", "ci/local.sh")
+    var_name = manifest.get("var_name", "ARCH")
+    buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
+    git_host = manifest.get("git_host", "git.oleks.space")
+    fixed_label = manifest["fixed_label"]
+    node_selector = manifest["node_selector"]
+    timeout = manifest.get("timeout")
+    manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches))
+    manifest_buildkit_addr = manifest["manifest_buildkit_addr"]
+    build_header = manifest["build_header"].rstrip("\n")
+    manifest_header = manifest["manifest_header"].rstrip("\n")
+
+    arch_lines = [f"    - {a}" for a in arches]
+    for extra in manifest.get("commented_arches", []):
+        arch_lines.append(f"    # - {extra}")
+    arch_list = "\n".join(arch_lines)
+    timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
+
+    lines = "\n".join(f"          {k}: {v}" for k, v in node_selector.items())
+    node_selector_block = f"        nodeSelector:\n{lines}\n"
+
+    build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format(
+        fixed_label=fixed_label,
+        git_host=git_host,
+        timeout_block=timeout_block,
+        header=build_header,
+        var_name=var_name,
+        arch_list=arch_list,
+        image=image,
+        buildkit_prefix=buildkit_prefix,
+        script=script,
+        node_selector_block=node_selector_block,
+    )
+    mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format(
+        fixed_label=fixed_label,
+        script=script,
+        header=manifest_header,
+        image=image,
+        manifest_buildkit_addr=manifest_buildkit_addr,
+        manifest_arches_env=manifest_arches_env,
+        node_selector_block=node_selector_block,
+    )
+    return {"build.yaml": build, "manifest.yaml": mnf}
+
+
 def render_split(manifest: dict) -> dict:
     repo = manifest["repo"]
     git_host = manifest.get("git_host", "git.oleks.space")
@@ -113,12 +167,20 @@ def render_split(manifest: dict) -> dict:
     return out
 
 
+def _comment_block(text, indent):
+    if not text:
+        return ""
+    lines = text.rstrip("\n").splitlines()
+    return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)
+
+
 def render_single(manifest: dict) -> dict:
-    repo = manifest["repo"]
     arch = manifest["arch"]
     git_host = manifest.get("git_host", "git.oleks.space")
     branches = "[" + ", ".join(manifest["branches"]) + "]"
     buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
+    header = manifest["header"].rstrip("\n") + "\n"
+    label_lines = _comment_block(manifest.get("label_comment", ""), "  ") + f"  arch: {arch}"
 
     step_tmpl = load_template("single", "COMPONENT_STEP.tmpl")
     steps = []
@@ -126,19 +188,23 @@ def render_single(manifest: dict) -> dict:
         command = f"./ci/local.sh --arch {arch}"
         if c.get("component"):
             command += f" --component {c['component']}"
+        step_comment = _comment_block(c.get("step_comment", ""), "  ")
         steps.append(
             step_tmpl.format(
+                step_comment=step_comment,
                 step_name=c["name"],
                 image=manifest["image"],
+                buildkit_comment=_comment_block(c.get("buildkit_comment", ""), "      "),
                 buildkit_addr=buildkit_addr,
+                version_comment=_comment_block(c.get("version_comment", ""), "      "),
                 command=command,
                 arch=arch,
                 memory=c.get("memory", "4Gi"),
             )
         )
     container = load_template("single", "container.yaml.tmpl").format(
-        repo=repo,
-        arch=arch,
+        header=header,
+        label_lines=label_lines,
         branches=branches,
         git_host=git_host,
         component_steps="\n".join(steps),

generator/templates/matrix/build-explicit-clone.yaml.tmpl

@@ -0,0 +1,37 @@
+labels:
+  arch: {fixed_label}
+
+clone:
+  - name: clone
+    image: woodpeckerci/plugin-git
+    environment:
+      CI_NETRC_MACHINE: {git_host}
+      CI_NETRC_USERNAME: oleks
+      CI_NETRC_PASSWORD:
+        from_secret: gitea_clone_token
+      PLUGIN_TAGS: "false"
+      PLUGIN_DEPTH: "1"
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+{timeout_block}
+{header}
+matrix:
+  {var_name}:
+{arch_list}
+
+steps:
+  - name: build-${{{var_name}}}
+    image: {image}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "tcp://{buildkit_prefix}-${{{var_name}}}.infra.svc.cluster.local:1234"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - {script} --arch "${{{var_name}}}"
+    backend_options:
+      kubernetes:
+{node_selector_block}

generator/templates/matrix/manifest-explicit-clone.yaml.tmpl

@@ -0,0 +1,37 @@
+labels:
+  arch: {fixed_label}
+
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+depends_on:
+  - build
+
+# oleks/ci-scripts#9: run even if a per-arch matrix leg failed. Without
+# `runs_on`, Woodpecker skips a dependent workflow outright once any leg of
+# `build` fails, so a lone arch success would silently produce no image at
+# all. `{script} --manifest` itself must probe the registry for which
+# per-arch tags actually landed (not assume a fixed arch list, and not rely
+# on step/pipeline success status) and build the manifest from whatever
+# exists — a partial manifest beats none.
+runs_on:
+  - success
+  - failure
+
+{header}
+steps:
+  - name: manifest
+    image: {image}
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "{manifest_buildkit_addr}"
+      MANIFEST_ARCHES: "{manifest_arches_env}"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - {script} --manifest
+    backend_options:
+      kubernetes:
+{node_selector_block}

generator/templates/single/COMPONENT_STEP.tmpl

@@ -1,12 +1,12 @@
-  - name: {step_name}
+{step_comment}  - name: {step_name}
     image: {image}
     environment:
       REGISTRY_TOKEN:
         from_secret: registry_token
-      BUILDKIT_ADDR: {buildkit_addr}
+{buildkit_comment}      BUILDKIT_ADDR: {buildkit_addr}
       PUBLISH: "1"
       BRANCH: "${{CI_COMMIT_BRANCH}}"
-      VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
+{version_comment}      VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
     commands:
       - echo "▸ arch=$(uname -m)"
       - {command}

generator/templates/single/container.yaml.tmpl

@@ -1,10 +1,6 @@
-# Build + push the {repo} container (cluster #196, emmett#44).
-#
-# Keep-Dockerfile OCI image build; native single-arch buildkit, no matrix,
-# no manifest step. See {repo}/ci/local.sh for the actual build logic.
-
+{header}
 labels:
-  arch: {arch}
+{label_lines}
 
 when:
   - event: push