Fix stuck-pending pipeline: hardcode per-arch labels instead of unresolved ${ARCH} (oleks/ci-scripts#14, #15)

.woodpecker/build-amd64.yaml

@@ -0,0 +1,36 @@
+# Hand-authored (not generated) for oleks/ci-scripts#14, #15.
+#
+# Deviation from the shared generator's "matrix" kind (which emits one
+# build.yaml with `matrix: {ARCH: [...]}` + `labels: {arch: ${ARCH}}`): a
+# Woodpecker scheduler bug/regression on this cluster's agent version left
+# that unresolved-at-label-matching-time `${ARCH}` literal never matching
+# any agent, so the pipeline sat pending forever with zero dispatch
+# (diagnosed via cluster:ci-agent). Splitting into one file per arch with a
+# hardcoded literal label sidesteps the whole class of bug for this new
+# self-build pipeline. The shared generator's matrix kind still needs an
+# audit across the ~15 repos that use it (see oleks/ci-scripts#12/#13 CI
+# registration + drift-detection work) — not fixed here, flagged on #14.
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+labels:
+  arch: amd64
+
+steps:
+  - name: build-and-push
+    image: git.oleks.space/oleks/nix-ci:latest
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "tcp://buildkit-amd64.infra.svc.cluster.local:1234"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - ci/local.sh --arch amd64
+    backend_options:
+      kubernetes:
+        labels:
+          commit-tag: "${CI_COMMIT_TAG}"
+          pipeline-number: "${CI_PIPELINE_NUMBER}"
+          build-arch: amd64

.woodpecker/build-arm64.yaml

@@ -0,0 +1,27 @@
+# Hand-authored (not generated) for oleks/ci-scripts#14, #15. See
+# build-amd64.yaml for why this is split per-arch instead of using the
+# shared generator's matrix kind (unresolved `${ARCH}` label bug).
+when:
+  - event: tag
+    ref: "refs/tags/v*"
+
+labels:
+  arch: arm64
+
+steps:
+  - name: build-and-push
+    image: git.oleks.space/oleks/nix-ci:latest
+    environment:
+      REGISTRY_TOKEN:
+        from_secret: registry_token
+      BUILDKIT_ADDR: "tcp://buildkit-arm64.infra.svc.cluster.local:1234"
+      PUBLISH: "1"
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - ci/local.sh --arch arm64
+    backend_options:
+      kubernetes:
+        labels:
+          commit-tag: "${CI_COMMIT_TAG}"
+          pipeline-number: "${CI_PIPELINE_NUMBER}"
+          build-arch: arm64

.woodpecker/build.yaml

@@ -1,38 +0,0 @@
-# GENERATED by oleks/ci-scripts — do not hand-edit.
-# Source manifest: manifests/ci-fleet-publish.yaml
-# Regenerate: python3 generator/render.py manifests/ci-fleet-publish.yaml --out <repo>/.woodpecker
-
-matrix:
-  ARCH:
-    - amd64
-    - arm64
-
-when:
-  - event: tag
-    ref: "refs/tags/v*"
-
-labels:
-  arch: ${ARCH}
-
-# Thin wrapper (cluster #196, emmett#44): CI runs the SAME ci/local.sh a
-# developer runs, so CI and local can't drift. CI-specific bits are env
-# overrides only: BUILDKIT_ADDR -> the in-cluster NATIVE per-arch buildkit
-# (no QEMU), PUBLISH=1 -> actually push. Output is unchanged: a per-arch
-# :<version>-<arch> image, later merged into :<version>+:latest by manifest.yaml.
-steps:
-  - name: build-and-push
-    image: git.oleks.space/oleks/nix-ci:latest
-    environment:
-      REGISTRY_TOKEN:
-        from_secret: registry_token
-      BUILDKIT_ADDR: "tcp://buildkit-${ARCH}.infra.svc.cluster.local:1234"
-      PUBLISH: "1"
-    commands:
-      - echo "▸ arch=$(uname -m)"
-      - ci/local.sh --arch "${ARCH}"
-    backend_options:
-      kubernetes:
-        labels:
-          commit-tag: "${CI_COMMIT_TAG}"
-          pipeline-number: "${CI_PIPELINE_NUMBER}"
-          build-arch: "${ARCH}"

.woodpecker/drift.yaml

@@ -0,0 +1,20 @@
+# Drift-detection pipeline (oleks/ci-scripts#13): re-renders every manifest
+# entry and diffs it against each target repo's live HEAD .woodpecker.yaml
+# (fetched over the Gitea API, so this works from a bare ci-scripts clone).
+# On drift it opens/updates a single "CI drift detected" issue in this repo;
+# when drift clears it closes that issue. Runs on a schedule (see cron
+# registration) and can also be triggered manually.
+
+when:
+  - event: cron
+  - event: manual
+
+steps:
+  - name: drift-check
+    image: git.oleks.space/oleks/nix-ci:latest
+    environment:
+      CI_DRIFT_TOKEN:
+        from_secret: ci_drift_token
+    commands:
+      - echo "▸ arch=$(uname -m)"
+      - nix shell nixpkgs#python3 nixpkgs#python3Packages.pyyaml -c python3 drift_check.py

.woodpecker/manifest.yaml

@@ -1,21 +1,21 @@
-# GENERATED by oleks/ci-scripts — do not hand-edit.
-# Source manifest: manifests/ci-fleet-publish.yaml
-# Regenerate: python3 generator/render.py manifests/ci-fleet-publish.yaml --out <repo>/.woodpecker
-
+# Hand-authored (not generated) for oleks/ci-scripts#14, #15 — see
+# build-amd64.yaml for why the arch builds are split per-file instead of a
+# single generated matrix build.yaml.
 when:
   - event: tag
     ref: "refs/tags/v*"
 
 depends_on:
-  - build
+  - build-amd64
+  - build-arm64
 
-# oleks/ci-scripts#9: run even if a per-arch matrix leg failed. Without
-# `runs_on`, Woodpecker skips a dependent workflow outright once any leg of
-# `build` fails, so a lone arm64 success would silently produce no image at
-# all. `ci/local.sh --manifest` itself must probe the registry for which
-# per-arch tags actually landed (not assume a fixed arch list, and not rely
-# on step/pipeline success status) and build the manifest from whatever
-# exists — a partial manifest beats none.
+# oleks/ci-scripts#9: run even if a per-arch build failed. Without
+# `runs_on`, Woodpecker skips a dependent workflow outright once any
+# dependency fails, so a lone arm64 success would silently produce no
+# image at all. `ci/local.sh --manifest` itself must probe the registry
+# for which per-arch tags actually landed (not assume a fixed arch list,
+# and not rely on step/pipeline success status) and build the manifest
+# from whatever exists — a partial manifest beats none.
 runs_on:
   - success
   - failure

drift_check.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Detect drift between manifest.yaml-rendered pipelines and each target
+repo's HEAD `.woodpecker.yaml` (oleks/ci-scripts#13).
+
+Unlike `generate.py --check` (which compares against local `../building/*`
+checkouts), this fetches each repo's live HEAD file over the Gitea API, so it
+works from a fresh CI clone of ci-scripts alone. On drift it opens or updates
+a single "CI drift detected" issue in oleks/ci-scripts; when drift clears, it
+closes that issue if still open.
+"""
+
+import base64
+import json
+import os
+import pathlib
+import sys
+import urllib.error
+import urllib.request
+
+import yaml
+
+import generate
+
+GITEA_HOST = "git.oleks.space"
+OWNER = "oleks"
+ISSUE_TITLE = "CI drift detected"
+HERE = pathlib.Path(__file__).parent
+
+
+def api(method: str, path: str, token: str, body: dict | None = None) -> dict:
+    url = f"https://{GITEA_HOST}/api/v1{path}"
+    data = json.dumps(body).encode() if body is not None else None
+    req = urllib.request.Request(url, data=data, method=method)
+    req.add_header("Authorization", f"token {token}")
+    if data is not None:
+        req.add_header("Content-Type", "application/json")
+    try:
+        with urllib.request.urlopen(req) as resp:
+            raw = resp.read()
+            return json.loads(raw) if raw else {}
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return {}
+        raise
+
+
+def fetch_head_file(repo: str, token: str) -> str:
+    data = api("GET", f"/repos/{OWNER}/{repo}/contents/.woodpecker.yaml", token)
+    if not data or "content" not in data:
+        return ""
+    return base64.b64decode(data["content"]).decode()
+
+
+def find_drift_issue(token: str) -> dict | None:
+    issues = api(
+        "GET",
+        f"/repos/{OWNER}/ci-scripts/issues?state=open&type=issues&q={ISSUE_TITLE.replace(' ', '+')}",
+        token,
+    )
+    for issue in issues or []:
+        if issue.get("title") == ISSUE_TITLE:
+            return issue
+    return None
+
+
+def repo_name(dir_: str) -> str:
+    return dir_.rsplit("/", 1)[-1]
+
+
+def main() -> int:
+    token = os.environ.get("CI_DRIFT_TOKEN")
+    if not token:
+        print("CI_DRIFT_TOKEN not set", file=sys.stderr)
+        return 1
+
+    manifest = yaml.safe_load((HERE / "manifest.yaml").read_text())
+    jobs = [(r, generate.render) for r in manifest["repos"]] + [
+        (r, generate.render_thin) for r in manifest.get("thin_repos", [])
+    ]
+
+    drifted = []
+    for repo, render_fn in jobs:
+        name = repo_name(repo["dir"])
+        expected = render_fn(repo)
+        actual = fetch_head_file(name, token)
+        if actual != expected:
+            drifted.append(name)
+
+    existing_issue = find_drift_issue(token)
+
+    if drifted:
+        body_lines = [
+            "Drift detected between `manifest.yaml`-rendered pipelines and "
+            "the repo's live `.woodpecker.yaml`:",
+            "",
+        ] + [f"- {OWNER}/{name}" for name in sorted(drifted)]
+        body_lines += [
+            "",
+            "Re-render with `python3 generate.py` in oleks/ci-scripts and "
+            "push the update to the drifted repo(s), or reconcile the "
+            "manifest if the hand-edit was intentional.",
+        ]
+        body = "\n".join(body_lines)
+        if existing_issue:
+            api(
+                "PATCH",
+                f"/repos/{OWNER}/ci-scripts/issues/{existing_issue['number']}",
+                token,
+                {"body": body},
+            )
+            print(f"updated issue #{existing_issue['number']}")
+        else:
+            api(
+                "POST",
+                f"/repos/{OWNER}/ci-scripts/issues",
+                token,
+                {"title": ISSUE_TITLE, "body": body},
+            )
+            print("opened drift issue")
+        print(f"drift found in: {', '.join(sorted(drifted))}")
+        return 1
+
+    print("no drift")
+    if existing_issue:
+        api(
+            "PATCH",
+            f"/repos/{OWNER}/ci-scripts/issues/{existing_issue['number']}",
+            token,
+            {"state": "closed", "body": "Drift cleared on next scheduled check."},
+        )
+        print(f"closed issue #{existing_issue['number']}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

manifests/ci-fleet-publish.yaml

@@ -1,6 +0,0 @@
-repo: ci-fleet-publish
-kind: matrix
-image: git.oleks.space/oleks/nix-ci
-script: ci/local.sh
-arches: [amd64, arm64]
-buildkit_prefix: buildkit