diff --git a/generator/render.py b/generator/render.py index 677939f..ba3a02f 100644 --- a/generator/render.py +++ b/generator/render.py @@ -41,6 +41,9 @@ def load_template(*parts): def render_matrix(manifest: dict) -> dict: + if manifest.get("explicit_clone"): + return render_matrix_explicit_clone(manifest) + arches = manifest["arches"] image = manifest["image"] script = manifest.get("script", "ci/local.sh") @@ -87,6 +90,57 @@ def render_matrix(manifest: dict) -> dict: return {"build.yaml": build, "manifest.yaml": mnf} +def render_matrix_explicit_clone(manifest: dict) -> dict: + """Variant for repos that keep a Dockerfile + explicit clone + a fixed + (non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3. + Selected by `explicit_clone: true` in the manifest.""" + arches = manifest["arches"] + image = manifest["image"] + script = manifest.get("script", "ci/local.sh") + var_name = manifest.get("var_name", "ARCH") + buildkit_prefix = manifest.get("buildkit_prefix", "buildkit") + git_host = manifest.get("git_host", "git.oleks.space") + fixed_label = manifest["fixed_label"] + node_selector = manifest["node_selector"] + timeout = manifest.get("timeout") + manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches)) + manifest_buildkit_addr = manifest["manifest_buildkit_addr"] + build_header = manifest["build_header"].rstrip("\n") + manifest_header = manifest["manifest_header"].rstrip("\n") + + arch_lines = [f" - {a}" for a in arches] + for extra in manifest.get("commented_arches", []): + arch_lines.append(f" # - {extra}") + arch_list = "\n".join(arch_lines) + timeout_block = f"\ntimeout: {timeout}\n" if timeout else "" + + lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items()) + node_selector_block = f" nodeSelector:\n{lines}\n" + + build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format( + fixed_label=fixed_label, + git_host=git_host, + timeout_block=timeout_block, + header=build_header, + var_name=var_name, + arch_list=arch_list, + image=image, + buildkit_prefix=buildkit_prefix, + script=script, + node_selector_block=node_selector_block, + ) + mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format( + fixed_label=fixed_label, + script=script, + header=manifest_header, + image=image, + manifest_buildkit_addr=manifest_buildkit_addr, + manifest_arches_env=manifest_arches_env, + node_selector_block=node_selector_block, + ) + return {"build.yaml": build, "manifest.yaml": mnf} + + def render_split(manifest: dict) -> dict: repo = manifest["repo"] git_host = manifest.get("git_host", "git.oleks.space") @@ -113,12 +167,20 @@ def render_split(manifest: dict) -> dict: return out +def _comment_block(text, indent): + if not text: + return "" + lines = text.rstrip("\n").splitlines() + return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines) + + def render_single(manifest: dict) -> dict: - repo = manifest["repo"] arch = manifest["arch"] git_host = manifest.get("git_host", "git.oleks.space") branches = "[" + ", ".join(manifest["branches"]) + "]" buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234" + header = manifest["header"].rstrip("\n") + "\n" + label_lines = _comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}" step_tmpl = load_template("single", "COMPONENT_STEP.tmpl") steps = [] @@ -126,19 +188,23 @@ def render_single(manifest: dict) -> dict: command = f"./ci/local.sh --arch {arch}" if c.get("component"): command += f" --component {c['component']}" + step_comment = _comment_block(c.get("step_comment", ""), " ") steps.append( step_tmpl.format( + step_comment=step_comment, step_name=c["name"], image=manifest["image"], + buildkit_comment=_comment_block(c.get("buildkit_comment", ""), " "), buildkit_addr=buildkit_addr, + version_comment=_comment_block(c.get("version_comment", ""), " "), command=command, arch=arch, memory=c.get("memory", "4Gi"), ) ) container = load_template("single", "container.yaml.tmpl").format( - repo=repo, - arch=arch, + header=header, + label_lines=label_lines, branches=branches, git_host=git_host, component_steps="\n".join(steps), diff --git a/generator/templates/matrix/build-explicit-clone.yaml.tmpl b/generator/templates/matrix/build-explicit-clone.yaml.tmpl new file mode 100644 index 0000000..6cc0d1b --- /dev/null +++ b/generator/templates/matrix/build-explicit-clone.yaml.tmpl @@ -0,0 +1,37 @@ +labels: + arch: {fixed_label} + +clone: + - name: clone + image: woodpeckerci/plugin-git + environment: + CI_NETRC_MACHINE: {git_host} + CI_NETRC_USERNAME: oleks + CI_NETRC_PASSWORD: + from_secret: gitea_clone_token + PLUGIN_TAGS: "false" + PLUGIN_DEPTH: "1" + +when: + - event: tag + ref: "refs/tags/v*" +{timeout_block} +{header} +matrix: + {var_name}: +{arch_list} + +steps: + - name: build-${{{var_name}}} + image: {image} + environment: + REGISTRY_TOKEN: + from_secret: registry_token + BUILDKIT_ADDR: "tcp://{buildkit_prefix}-${{{var_name}}}.infra.svc.cluster.local:1234" + PUBLISH: "1" + commands: + - echo "▸ arch=$(uname -m)" + - {script} --arch "${{{var_name}}}" + backend_options: + kubernetes: +{node_selector_block} \ No newline at end of file diff --git a/generator/templates/matrix/manifest-explicit-clone.yaml.tmpl b/generator/templates/matrix/manifest-explicit-clone.yaml.tmpl new file mode 100644 index 0000000..47fc9a8 --- /dev/null +++ b/generator/templates/matrix/manifest-explicit-clone.yaml.tmpl @@ -0,0 +1,37 @@ +labels: + arch: {fixed_label} + +when: + - event: tag + ref: "refs/tags/v*" + +depends_on: + - build + +# oleks/ci-scripts#9: run even if a per-arch matrix leg failed. Without +# `runs_on`, Woodpecker skips a dependent workflow outright once any leg of +# `build` fails, so a lone arch success would silently produce no image at +# all. `{script} --manifest` itself must probe the registry for which +# per-arch tags actually landed (not assume a fixed arch list, and not rely +# on step/pipeline success status) and build the manifest from whatever +# exists — a partial manifest beats none. +runs_on: + - success + - failure + +{header} +steps: + - name: manifest + image: {image} + environment: + REGISTRY_TOKEN: + from_secret: registry_token + BUILDKIT_ADDR: "{manifest_buildkit_addr}" + MANIFEST_ARCHES: "{manifest_arches_env}" + PUBLISH: "1" + commands: + - echo "▸ arch=$(uname -m)" + - {script} --manifest + backend_options: + kubernetes: +{node_selector_block} \ No newline at end of file diff --git a/generator/templates/single/COMPONENT_STEP.tmpl b/generator/templates/single/COMPONENT_STEP.tmpl index d71cdc5..8b47345 100644 --- a/generator/templates/single/COMPONENT_STEP.tmpl +++ b/generator/templates/single/COMPONENT_STEP.tmpl @@ -1,12 +1,12 @@ - - name: {step_name} +{step_comment} - name: {step_name} image: {image} environment: REGISTRY_TOKEN: from_secret: registry_token - BUILDKIT_ADDR: {buildkit_addr} +{buildkit_comment} BUILDKIT_ADDR: {buildkit_addr} PUBLISH: "1" BRANCH: "${{CI_COMMIT_BRANCH}}" - VERSION: "0.1.${{CI_PIPELINE_NUMBER}}" +{version_comment} VERSION: "0.1.${{CI_PIPELINE_NUMBER}}" commands: - echo "▸ arch=$(uname -m)" - {command} diff --git a/generator/templates/single/container.yaml.tmpl b/generator/templates/single/container.yaml.tmpl index c30fc3f..512a300 100644 --- a/generator/templates/single/container.yaml.tmpl +++ b/generator/templates/single/container.yaml.tmpl @@ -1,10 +1,6 @@ -# Build + push the {repo} container (cluster #196, emmett#44). -# -# Keep-Dockerfile OCI image build; native single-arch buildkit, no matrix, -# no manifest step. See {repo}/ci/local.sh for the actual build logic. - +{header} labels: - arch: {arch} +{label_lines} when: - event: push diff --git a/manifests/csi-s3.yaml b/manifests/csi-s3.yaml index b7fb96c..0ec90a2 100644 --- a/manifests/csi-s3.yaml +++ b/manifests/csi-s3.yaml @@ -1,10 +1,31 @@ repo: csi-s3 kind: matrix -image: git.oleks.space/oleks/nix-ci +explicit_clone: true +image: git.oleks.space/oleks/nix-ci:latest script: ci/local.sh +var_name: TARGET_ARCH arches: [amd64, arm64] +commented_arches: + - "s390x # disabled: no s390x builder available" buildkit_prefix: buildkit-rootless +fixed_label: amd64 timeout: 240 node_selector: kubernetes.io/hostname: howard2404 manifest_arches_env: "amd64 arm64" +manifest_buildkit_addr: "tcp://buildkit-amd64.infra.svc.cluster.local:1234" +build_header: | + # Thin wrapper (cluster #196 / emmett#44): CI runs the SAME ci/local.sh a + # developer runs, so CI and local can't drift. csi-s3 keeps its Dockerfile + # (s3fs-fuse from Alpine edge + a fetched geesefs binary are not cleanly + # Nix-expressible), so this is the keep-Dockerfile escape hatch, not a + # parity-lib conversion. The only CI-specific bits are env overrides: + # BUILDKIT_ADDR -> the in-cluster native-arch remote buildkit + # PUBLISH=1 -> actually push (local default is dry-run) + # Each native-arch node builds + pushes its own per-arch tags; the manifest + # workflow then combines them. +manifest_header: | + # Thin wrapper (cluster #196 / emmett#44): the SAME ci/local.sh combines the + # per-arch tags into the multi-arch manifest. MANIFEST_ARCHES tracks which + # per-arch tags exist; re-add s390x here and in build.yaml when a builder + # returns. diff --git a/manifests/emdash.kotkanagrilli.fi.yaml b/manifests/emdash.kotkanagrilli.fi.yaml new file mode 100644 index 0000000..96a26d1 --- /dev/null +++ b/manifests/emdash.kotkanagrilli.fi.yaml @@ -0,0 +1,31 @@ +repo: emdash.kotkanagrilli.fi +kind: single +image: git.oleks.space/oleks/nix-ci:latest-arm64 +arch: arm64 +git_host: git.oleks.space +branches: [develop, staging, production] +header: | + # Build + push the EmDash app container (cluster #196, emmett#44). + # + # This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3, + # built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so + # it stays on `docker buildx` against the in-cluster buildkit. The build logic + # lives entirely in ci/local.sh so CI and a local run can't drift; this + # pipeline is a thin wrapper that just sets the CI-specific env and invokes it. + # + # Tagging (handled by ci/local.sh) — every build publishes three refs: + # 0.1. immutable, audit. + # floating pointer Flux's ImagePolicy tracks (staging/ + # production) → digest rewritten into the fleet repo → pod + # rolls. Do not drop. + # -latest same image; chart image.tag cosmetic fallback. + # Only staging/production have an ImagePolicy, so only those move pods. +label_comment: | + kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build + natively on arm64 — no cross-compile needed. +components: + - name: build-and-push + buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes." + version_comment: | + Semver-shaped immutable tag, one per build. 0.1 is fixed (no real + semver discipline yet); the patch is the monotonic pipeline number. diff --git a/manifests/the-eye.yaml b/manifests/the-eye.yaml index 16b3dd3..643001b 100644 --- a/manifests/the-eye.yaml +++ b/manifests/the-eye.yaml @@ -2,8 +2,38 @@ repo: the-eye kind: single image: git.oleks.space/oleks/nix-ci:latest-arm64 arch: arm64 +git_host: git.oleks.space branches: [develop, staging, production] +header: | + # Build + push the EmDash app container (cluster #196, emmett#44). + # + # This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3, + # built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so + # it stays on `docker buildx` against the in-cluster buildkit. The build logic + # lives entirely in ci/local.sh so CI and a local run can't drift; this + # pipeline is a thin wrapper that just sets the CI-specific env and invokes it. + # + # Tagging (handled by ci/local.sh) — every build publishes three refs: + # 0.1. immutable, audit. + # floating pointer Flux's ImagePolicy tracks (staging/ + # production) → digest rewritten into the fleet repo → pod + # rolls. Do not drop. + # -latest same image; chart image.tag cosmetic fallback. + # Only staging/production have an ImagePolicy, so only those move pods. +label_comment: | + kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build + natively on arm64 — no cross-compile needed. components: - name: build-and-push + buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes." + version_comment: | + Semver-shaped immutable tag, one per build. 0.1 is fixed (no real + semver discipline yet); the patch is the monotonic pipeline number. - name: build-and-push-grpc component: grpc + step_comment: | + gRPC WorkspaceService image (#80). Same Dockerfile (the `grpc` target), same + builder / registry / auth / tagging logic as the web build above — only + --component grpc differs, which suffixes the published refs with -grpc + (e.g. staging-grpc). Reuses the immutable per-pipeline $VERSION (0.1.), + published as 0.1.-grpc.