oleks/ci-scripts
1
0
Fork 0
Code Issues 8 Pull Requests Actions Packages Projects Releases Wiki Activity

Generator: explicit-clone matrix variant + raw header/comment fields for single kind (oleks/ci-scripts#2)

Browse Source
This commit is contained in:
repo-worker
2026-07-04 19:30:12 +03:00
parent 7fbf3431ad
commit db971954d6
8 changed files with 231 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
generator/render.py
+69 -3
View File
@@ -41,6 +41,9 @@ def load_template(*parts):
def render_matrix(manifest: dict) -> dict:
if manifest.get("explicit_clone"):
return render_matrix_explicit_clone(manifest)
arches = manifest["arches"]
image = manifest["image"]
script = manifest.get("script", "ci/local.sh")
@@ -87,6 +90,57 @@ def render_matrix(manifest: dict) -> dict:
return {"build.yaml": build, "manifest.yaml": mnf}
def render_matrix_explicit_clone(manifest: dict) -> dict:
"""Variant for repos that keep a Dockerfile + explicit clone + a fixed
(non-templated) `labels: arch:` and pin a specific node, e.g. csi-s3.
Selected by `explicit_clone: true` in the manifest."""
arches = manifest["arches"]
image = manifest["image"]
script = manifest.get("script", "ci/local.sh")
var_name = manifest.get("var_name", "ARCH")
buildkit_prefix = manifest.get("buildkit_prefix", "buildkit")
git_host = manifest.get("git_host", "git.oleks.space")
fixed_label = manifest["fixed_label"]
node_selector = manifest["node_selector"]
timeout = manifest.get("timeout")
manifest_arches_env = manifest.get("manifest_arches_env", " ".join(arches))
manifest_buildkit_addr = manifest["manifest_buildkit_addr"]
build_header = manifest["build_header"].rstrip("\n")
manifest_header = manifest["manifest_header"].rstrip("\n")
arch_lines = [f" - {a}" for a in arches]
for extra in manifest.get("commented_arches", []):
arch_lines.append(f" # - {extra}")
arch_list = "\n".join(arch_lines)
timeout_block = f"\ntimeout: {timeout}\n" if timeout else ""
lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items())
node_selector_block = f" nodeSelector:\n{lines}\n"
build = load_template("matrix", "build-explicit-clone.yaml.tmpl").format(
fixed_label=fixed_label,
git_host=git_host,
timeout_block=timeout_block,
header=build_header,
var_name=var_name,
arch_list=arch_list,
image=image,
buildkit_prefix=buildkit_prefix,
script=script,
node_selector_block=node_selector_block,
)
mnf = load_template("matrix", "manifest-explicit-clone.yaml.tmpl").format(
fixed_label=fixed_label,
script=script,
header=manifest_header,
image=image,
manifest_buildkit_addr=manifest_buildkit_addr,
manifest_arches_env=manifest_arches_env,
node_selector_block=node_selector_block,
)
return {"build.yaml": build, "manifest.yaml": mnf}
def render_split(manifest: dict) -> dict:
repo = manifest["repo"]
git_host = manifest.get("git_host", "git.oleks.space")
@@ -113,12 +167,20 @@ def render_split(manifest: dict) -> dict:
return out
def _comment_block(text, indent):
if not text:
return ""
lines = text.rstrip("\n").splitlines()
return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)
def render_single(manifest: dict) -> dict:
repo = manifest["repo"]
arch = manifest["arch"]
git_host = manifest.get("git_host", "git.oleks.space")
branches = "[" + ", ".join(manifest["branches"]) + "]"
buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
header = manifest["header"].rstrip("\n") + "\n"
label_lines = _comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}"
step_tmpl = load_template("single", "COMPONENT_STEP.tmpl")
steps = []
@@ -126,19 +188,23 @@ def render_single(manifest: dict) -> dict:
command = f"./ci/local.sh --arch {arch}"
if c.get("component"):
command += f" --component {c['component']}"
step_comment = _comment_block(c.get("step_comment", ""), " ")
steps.append(
step_tmpl.format(
step_comment=step_comment,
step_name=c["name"],
image=manifest["image"],
buildkit_comment=_comment_block(c.get("buildkit_comment", ""), " "),
buildkit_addr=buildkit_addr,
version_comment=_comment_block(c.get("version_comment", ""), " "),
command=command,
arch=arch,
memory=c.get("memory", "4Gi"),
)
)
container = load_template("single", "container.yaml.tmpl").format(
repo=repo,
arch=arch,
header=header,
label_lines=label_lines,
branches=branches,
git_host=git_host,
component_steps="\n".join(steps),
generator/templates/matrix/build-explicit-clone.yaml.tmpl
+37
View File
@@ -0,0 +1,37 @@
labels:
arch: {fixed_label}
clone:
- name: clone
image: woodpeckerci/plugin-git
environment:
CI_NETRC_MACHINE: {git_host}
CI_NETRC_USERNAME: oleks
CI_NETRC_PASSWORD:
from_secret: gitea_clone_token
PLUGIN_TAGS: "false"
PLUGIN_DEPTH: "1"
when:
- event: tag
ref: "refs/tags/v*"
{timeout_block}
{header}
matrix:
{var_name}:
{arch_list}
steps:
- name: build-${{{var_name}}}
image: {image}
environment:
REGISTRY_TOKEN:
from_secret: registry_token
BUILDKIT_ADDR: "tcp://{buildkit_prefix}-${{{var_name}}}.infra.svc.cluster.local:1234"
PUBLISH: "1"
commands:
- echo "▸ arch=$(uname -m)"
- {script} --arch "${{{var_name}}}"
backend_options:
kubernetes:
{node_selector_block}
generator/templates/matrix/manifest-explicit-clone.yaml.tmpl
+37
View File
@@ -0,0 +1,37 @@
labels:
arch: {fixed_label}
when:
- event: tag
ref: "refs/tags/v*"
depends_on:
- build
# oleks/ci-scripts#9: run even if a per-arch matrix leg failed. Without
# `runs_on`, Woodpecker skips a dependent workflow outright once any leg of
# `build` fails, so a lone arch success would silently produce no image at
# all. `{script} --manifest` itself must probe the registry for which
# per-arch tags actually landed (not assume a fixed arch list, and not rely
# on step/pipeline success status) and build the manifest from whatever
# exists — a partial manifest beats none.
runs_on:
- success
- failure
{header}
steps:
- name: manifest
image: {image}
environment:
REGISTRY_TOKEN:
from_secret: registry_token
BUILDKIT_ADDR: "{manifest_buildkit_addr}"
MANIFEST_ARCHES: "{manifest_arches_env}"
PUBLISH: "1"
commands:
- echo "▸ arch=$(uname -m)"
- {script} --manifest
backend_options:
kubernetes:
{node_selector_block}
generator/templates/single/COMPONENT_STEP.tmpl
+3 -3
View File
@@ -1,12 +1,12 @@
- name: {step_name}
{step_comment} - name: {step_name}
image: {image}
environment:
REGISTRY_TOKEN:
from_secret: registry_token
BUILDKIT_ADDR: {buildkit_addr}
{buildkit_comment} BUILDKIT_ADDR: {buildkit_addr}
PUBLISH: "1"
BRANCH: "${{CI_COMMIT_BRANCH}}"
VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
{version_comment} VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
commands:
- echo "▸ arch=$(uname -m)"
- {command}
generator/templates/single/container.yaml.tmpl
+2 -6
View File
@@ -1,10 +1,6 @@
# Build + push the {repo} container (cluster #196, emmett#44).
#
# Keep-Dockerfile OCI image build; native single-arch buildkit, no matrix,
# no manifest step. See {repo}/ci/local.sh for the actual build logic.
{header}
labels:
arch: {arch}
{label_lines}
when:
- event: push
manifests/csi-s3.yaml
+22 -1
View File
@@ -1,10 +1,31 @@
repo: csi-s3
kind: matrix
image: git.oleks.space/oleks/nix-ci
explicit_clone: true
image: git.oleks.space/oleks/nix-ci:latest
script: ci/local.sh
var_name: TARGET_ARCH
arches: [amd64, arm64]
commented_arches:
- "s390x # disabled: no s390x builder available"
buildkit_prefix: buildkit-rootless
fixed_label: amd64
timeout: 240
node_selector:
kubernetes.io/hostname: howard2404
manifest_arches_env: "amd64 arm64"
manifest_buildkit_addr: "tcp://buildkit-amd64.infra.svc.cluster.local:1234"
build_header: |
# Thin wrapper (cluster #196 / emmett#44): CI runs the SAME ci/local.sh a
# developer runs, so CI and local can't drift. csi-s3 keeps its Dockerfile
# (s3fs-fuse from Alpine edge + a fetched geesefs binary are not cleanly
# Nix-expressible), so this is the keep-Dockerfile escape hatch, not a
# parity-lib conversion. The only CI-specific bits are env overrides:
# BUILDKIT_ADDR -> the in-cluster native-arch remote buildkit
# PUBLISH=1 -> actually push (local default is dry-run)
# Each native-arch node builds + pushes its own per-arch tags; the manifest
# workflow then combines them.
manifest_header: |
# Thin wrapper (cluster #196 / emmett#44): the SAME ci/local.sh combines the
# per-arch tags into the multi-arch manifest. MANIFEST_ARCHES tracks which
# per-arch tags exist; re-add s390x here and in build.yaml when a builder
# returns.
manifests/emdash.kotkanagrilli.fi.yaml
+31
View File
@@ -0,0 +1,31 @@
repo: emdash.kotkanagrilli.fi
kind: single
image: git.oleks.space/oleks/nix-ci:latest-arm64
arch: arm64
git_host: git.oleks.space
branches: [develop, staging, production]
header: |
# Build + push the EmDash app container (cluster #196, emmett#44).
#
# This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3,
# built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so
# it stays on `docker buildx` against the in-cluster buildkit. The build logic
# lives entirely in ci/local.sh so CI and a local run can't drift; this
# pipeline is a thin wrapper that just sets the CI-specific env and invokes it.
#
# Tagging (handled by ci/local.sh) — every build publishes three refs:
# 0.1.<pipeline> immutable, audit.
# <branch> floating pointer Flux's ImagePolicy tracks (staging/
# production) → digest rewritten into the fleet repo → pod
# rolls. Do not drop.
# <branch>-latest same image; chart image.tag cosmetic fallback.
# Only staging/production have an ImagePolicy, so only those move pods.
label_comment: |
kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build
natively on arm64 — no cross-compile needed.
components:
- name: build-and-push
buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes."
version_comment: |
Semver-shaped immutable tag, one per build. 0.1 is fixed (no real
semver discipline yet); the patch is the monotonic pipeline number.
manifests/the-eye.yaml
+30
View File
@@ -2,8 +2,38 @@ repo: the-eye
kind: single
image: git.oleks.space/oleks/nix-ci:latest-arm64
arch: arm64
git_host: git.oleks.space
branches: [develop, staging, production]
header: |
# Build + push the EmDash app container (cluster #196, emmett#44).
#
# This is a keep-Dockerfile OCI web app (Astro/Node + native better-sqlite3,
# built from app/package-lock.json) — it is NOT Nix-expressible on emmett, so
# it stays on `docker buildx` against the in-cluster buildkit. The build logic
# lives entirely in ci/local.sh so CI and a local run can't drift; this
# pipeline is a thin wrapper that just sets the CI-specific env and invokes it.
#
# Tagging (handled by ci/local.sh) — every build publishes three refs:
# 0.1.<pipeline> immutable, audit.
# <branch> floating pointer Flux's ImagePolicy tracks (staging/
# production) → digest rewritten into the fleet repo → pod
# rolls. Do not drop.
# <branch>-latest same image; chart image.tag cosmetic fallback.
# Only staging/production have an ImagePolicy, so only those move pods.
label_comment: |
kotkan (the deploy target) is an Oracle Ampere arm64 host, so we build
natively on arm64 — no cross-compile needed.
components:
- name: build-and-push
buildkit_comment: "CI builds natively on the in-cluster arm64 buildkit and publishes."
version_comment: |
Semver-shaped immutable tag, one per build. 0.1 is fixed (no real
semver discipline yet); the patch is the monotonic pipeline number.
- name: build-and-push-grpc
component: grpc
step_comment: |
gRPC WorkspaceService image (#80). Same Dockerfile (the `grpc` target), same
builder / registry / auth / tagging logic as the web build above — only
--component grpc differs, which suffixes the published refs with -grpc
(e.g. staging-grpc). Reuses the immutable per-pipeline $VERSION (0.1.<n>),
published as 0.1.<n>-grpc.