Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 44d6ba9c8d | |||
| 38bb59e3d0 | |||
| bf36991833 | |||
| d1f8297c46 |
+38
-2
@@ -29,6 +29,13 @@
|
||||
# manifest step) using the SAME plugin image. Split into `buildx-build` /
|
||||
# `buildx-manifest` instead; wiki updated to match.
|
||||
#
|
||||
# Version resolution for buildx-build/buildx-manifest (oleks/ci-scripts#21,
|
||||
# see resolve_version() below): defaults to the raw CI_COMMIT_TAG (or an
|
||||
# explicit `version:` setting, or "latest") — unchanged from before. A repo
|
||||
# whose own build script normalizes tags (strips a leading "v" and a
|
||||
# trailing "-<build-number>") can opt in with `tag_strip_v: "true"` to
|
||||
# reproduce that exactly; default is off.
|
||||
#
|
||||
# See wiki Spec/Modern-CI-Target and Spec/Fleet Publish Plugin System.
|
||||
set -euo pipefail
|
||||
|
||||
@@ -67,6 +74,33 @@ render_tag() {
|
||||
echo "$scheme"
|
||||
}
|
||||
|
||||
resolve_version() {
|
||||
# resolve_version — the string substituted for {version} in tag_scheme,
|
||||
# used by both buildx_build_flow and buildx_manifest_flow so the two
|
||||
# stay in lockstep (oleks/ci-scripts#21).
|
||||
#
|
||||
# Precedence: an explicit `version:` setting (PLUGIN_VERSION) wins over
|
||||
# the tag the pipeline triggered on (CI_COMMIT_TAG), else "latest".
|
||||
# Nothing currently sets PLUGIN_VERSION (audited across every repo using
|
||||
# this plugin), so this is a no-op precedence change for every existing
|
||||
# consumer today.
|
||||
#
|
||||
# `tag_strip_v: "true"` (PLUGIN_TAG_STRIP_V) additionally strips a
|
||||
# leading "v" and a trailing "-<build-number>" suffix, e.g. "v0.22.7-4"
|
||||
# -> "0.22.7" — the normalization xonsh-image's (and csi-s3's) ci/local.sh
|
||||
# already does before tagging. Default is "false": preserves the raw
|
||||
# tag verbatim, which is what ci-scripts' own self-build (ci/local.sh,
|
||||
# this plugin's own v1.0.x tags) and every other current consumer of
|
||||
# this plugin already expect. Opt in per-repo via `tag_strip_v: "true"`
|
||||
# in `settings:` — do not flip the default.
|
||||
local version="${PLUGIN_VERSION:-${CI_COMMIT_TAG:-latest}}"
|
||||
if [ "${PLUGIN_TAG_STRIP_V:-false}" = "true" ]; then
|
||||
version="${version#v}"
|
||||
version="${version%-[0-9]*}"
|
||||
fi
|
||||
printf '%s' "$version"
|
||||
}
|
||||
|
||||
# --- attic watch-store lifecycle: explicit non-masking shutdown, never `|| true` ---
|
||||
ATTIC_PID=""
|
||||
|
||||
@@ -130,7 +164,8 @@ buildx_build_flow() {
|
||||
if [ -n "${PLUGIN_TAG_SCHEME:-}" ]; then
|
||||
tag_scheme="${PLUGIN_TAG_SCHEME}"
|
||||
fi
|
||||
local version="${CI_COMMIT_TAG:-${PLUGIN_VERSION:-latest}}"
|
||||
local version
|
||||
version=$(resolve_version)
|
||||
local tag
|
||||
tag=$(render_tag "$tag_scheme" "$version" "$arch")
|
||||
local context="${PLUGIN_CONTEXT:-.}"
|
||||
@@ -151,7 +186,8 @@ buildx_manifest_flow() {
|
||||
if [ -n "${PLUGIN_TAG_SCHEME:-}" ]; then
|
||||
tag_scheme="${PLUGIN_TAG_SCHEME}"
|
||||
fi
|
||||
local version="${CI_COMMIT_TAG:-${PLUGIN_VERSION:-latest}}"
|
||||
local version
|
||||
version=$(resolve_version)
|
||||
local dest_tags="${PLUGIN_DEST_TAGS:-${version},latest}"
|
||||
|
||||
IFS=',' read -ra platforms <<<"$PLUGIN_PLATFORMS"
|
||||
|
||||
+33
-1
@@ -19,6 +19,38 @@ HEADER = """# GENERATED by oleks/ci-scripts — do not hand-edit.
|
||||
"""
|
||||
|
||||
|
||||
def is_no_build(repo: dict) -> bool:
|
||||
"""s390x = no-build: s390x hardware is currently unavailable, so those
|
||||
wheels are not built/published (the repo code is kept as a cross-compile
|
||||
reference only). Any manifest entry may opt in/out with an explicit
|
||||
`no_build:` flag; otherwise entries under `s390x/` default to no-build.
|
||||
See the ci-scripts wiki Spec/* pages and memory feedback-no-s390x-builds."""
|
||||
return repo.get("no_build", str(repo.get("dir", "")).startswith("s390x/"))
|
||||
|
||||
|
||||
def render_no_build(repo: dict) -> str:
|
||||
"""Render a neutralized pipeline that does NOT build. Uses `event: manual`
|
||||
so tag pushes never auto-trigger a build; a manual run just prints why."""
|
||||
arch = repo.get("arch", "amd64")
|
||||
return f"""{HEADER}# s390x = NO-BUILD: s390x hardware is currently unavailable, so this wheel is
|
||||
# not built or published — the repo is kept as a cross-compile reference only.
|
||||
# Re-enable by setting `no_build: false` on this repo in manifest.yaml.
|
||||
labels:
|
||||
arch: {arch}
|
||||
|
||||
when:
|
||||
- event: manual
|
||||
|
||||
steps:
|
||||
- name: build-disabled
|
||||
image: alpine:3
|
||||
commands:
|
||||
- echo "s390x build disabled — hardware currently unavailable."
|
||||
- echo "Repo kept as a cross-compile reference only; not built/published."
|
||||
- echo "See oleks/ci-scripts wiki Spec/* and memory feedback-no-s390x-builds."
|
||||
"""
|
||||
|
||||
|
||||
def render(repo: dict) -> str:
|
||||
arch = repo["arch"]
|
||||
mem_req = repo["memory_request"]
|
||||
@@ -246,7 +278,7 @@ def main():
|
||||
]
|
||||
for repo, render_fn in jobs:
|
||||
target = BUILDING / repo["dir"] / ".woodpecker.yaml"
|
||||
rendered = render_fn(repo)
|
||||
rendered = render_no_build(repo) if is_no_build(repo) else render_fn(repo)
|
||||
if check_only:
|
||||
current = target.read_text() if target.exists() else ""
|
||||
if current != rendered:
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render thin ci-fleet-publish-based pipelines for the fleet family
|
||||
(oleks/ci-scripts#16): the 10 identical-fleet repos in manifest.yaml's
|
||||
`repos:` list and the 5 thin-parity repos in `thin_repos:`.
|
||||
|
||||
This is a PREP/PLANNING script (per team-lead direction on #16/#17): it
|
||||
renders into a local --out dir for diffing against the current generated
|
||||
`.woodpecker.yaml` files. It does NOT write into ~/projects/building/ and is
|
||||
not wired into generate.py's main() — that wiring (and the actual repo
|
||||
migration) happens once the diffs are reviewed and the tag-scheme question
|
||||
is settled.
|
||||
|
||||
Behavior-preservation approach: every piece of shell logic that generate.py's
|
||||
render()/render_thin() used to inline (resolv.conf, nix.conf, attic
|
||||
lifecycle, redis/sccache env, token remap, doctor gate) either maps to a
|
||||
PLUGIN_* setting the plugin already implements (nameserver, cores, max_jobs,
|
||||
cache), or gets folded into the `run:` command as an explicit shell
|
||||
one-liner/block — the plugin's `publish` flow just execs `bash -lc
|
||||
"$PLUGIN_RUN"`, so anything that was a `commands:` line before can still be
|
||||
a line in `run:`.
|
||||
|
||||
Usage:
|
||||
python3 generator/render_thin_fleet.py --out /path/to/scratch
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
|
||||
import yaml
|
||||
|
||||
HERE = pathlib.Path(__file__).parent.parent
|
||||
PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1"
|
||||
|
||||
GENERATED_HEADER = (
|
||||
"# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
|
||||
"# Source: oleks/ci-scripts manifest.yaml + generator/render_thin_fleet.py"
|
||||
" (oleks/ci-scripts#16).\n"
|
||||
"# Re-render with: python3 generator/render_thin_fleet.py --out <repo>\n"
|
||||
"\n"
|
||||
)
|
||||
|
||||
NAMESERVER = "169.254.20.10"
|
||||
|
||||
|
||||
def render_identical(repo: dict) -> str:
|
||||
"""The 10 `repos:` entries (oleks/ci-scripts#1 family)."""
|
||||
attic = repo.get("attic", True)
|
||||
redis_cache = repo.get("redis_cache", False)
|
||||
cores = repo.get("cores")
|
||||
max_jobs = repo.get("max_jobs", False)
|
||||
|
||||
run_lines = []
|
||||
if redis_cache:
|
||||
run_lines += [
|
||||
"export SCCACHE_REDIS_ENDPOINT=tcp://redis.default.svc.cluster.local:6379",
|
||||
"export SCCACHE_REDIS_PASSWORD=$REDIS_PASSWORD",
|
||||
"export CCACHE_REMOTE_STORAGE=redis://redis.default.svc.cluster.local:6379",
|
||||
"export CCACHE_REMOTE_ONLY=1",
|
||||
]
|
||||
run_lines.append("PUBLISH=1 nix run .#publish")
|
||||
run_block = "\n".join(f" {line}" for line in run_lines)
|
||||
|
||||
settings_lines = [
|
||||
f' run: |\n{run_block}',
|
||||
f' cache: "{str(attic).lower()}"',
|
||||
f' nameserver: "{NAMESERVER}"',
|
||||
]
|
||||
if max_jobs:
|
||||
settings_lines.append(' max_jobs: "true"')
|
||||
if cores:
|
||||
settings_lines.append(f' cores: "{cores}"')
|
||||
settings_block = "\n".join(settings_lines)
|
||||
|
||||
env_lines = [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"]
|
||||
if attic:
|
||||
env_lines += [" ATTIC_TOKEN:", " from_secret: attic_token"]
|
||||
if redis_cache:
|
||||
env_lines += [" REDIS_PASSWORD:", " from_secret: redis_password"]
|
||||
env_block = "\n".join(env_lines)
|
||||
|
||||
return f"""{GENERATED_HEADER}labels:
|
||||
arch: {repo["arch"]}
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: "refs/tags/v*"
|
||||
|
||||
steps:
|
||||
- name: publish
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
{env_block}
|
||||
settings:
|
||||
{settings_block}
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: {repo["memory_request"]}
|
||||
limits:
|
||||
memory: {repo["memory_limit"]}
|
||||
"""
|
||||
|
||||
|
||||
def render_thin_parity(repo: dict) -> str:
|
||||
"""The 5 `thin_repos:` entries (oleks/ci-scripts#10 family)."""
|
||||
doctor = repo.get("doctor", False)
|
||||
token_direct = repo.get("token_direct", True)
|
||||
publish_style = repo.get("publish_style", "publish1")
|
||||
tags_full = repo.get("tags_full", False)
|
||||
gitea_clone_token = repo.get("gitea_clone_token", True)
|
||||
|
||||
run_lines = []
|
||||
if doctor:
|
||||
run_lines.append(
|
||||
"nix run git+https://git.oleks.space/oleks/parity-lib#pipeline-doctor -- . --strict"
|
||||
)
|
||||
if not token_direct:
|
||||
run_lines.append('export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"')
|
||||
if publish_style == "publish1":
|
||||
run_lines.append("PUBLISH=1 nix run .#publish")
|
||||
else:
|
||||
run_lines.append("nix run .#publish-s390x -- --publish")
|
||||
run_block = "\n".join(f" {line}" for line in run_lines)
|
||||
|
||||
settings_lines = [
|
||||
f' run: |\n{run_block}',
|
||||
' cache: "false"', # oleks/ci-scripts#10: this family never used
|
||||
# the inline attic accelerator (dead ATTIC_TOKEN in some repos) —
|
||||
# cache: false preserves that, doesn't silently turn caching on.
|
||||
f' nameserver: "{NAMESERVER}"',
|
||||
]
|
||||
settings_block = "\n".join(settings_lines)
|
||||
|
||||
env_lines = []
|
||||
if gitea_clone_token:
|
||||
env_lines += [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"]
|
||||
if token_direct:
|
||||
env_lines += [" REGISTRY_TOKEN:", " from_secret: registry_token"]
|
||||
else:
|
||||
env_lines += [" CI_REGISTRY_TOKEN:", " from_secret: registry_token"]
|
||||
env_block = "\n".join(env_lines) if env_lines else " {}"
|
||||
|
||||
# oleks/ci-scripts#10 dead_secrets (ATTIC_TOKEN/REDIS_PASSWORD declared
|
||||
# but never referenced) are deliberately DROPPED here, not carried
|
||||
# forward — they were unused in the underlying command either way.
|
||||
|
||||
clone_note = ""
|
||||
if tags_full:
|
||||
clone_note = (
|
||||
"\n# NOTE: original PLUGIN_TAGS=true/PLUGIN_DEPTH=0 (full clone incl. tags)\n"
|
||||
"# is a `clone:` block override the thin contract doesn't have a settings\n"
|
||||
"# equivalent for yet — needs resolving before this repo migrates (see\n"
|
||||
"# migration report).\n"
|
||||
)
|
||||
|
||||
return f"""{GENERATED_HEADER}labels:
|
||||
arch: {repo["arch"]}
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: "refs/tags/v*"
|
||||
{clone_note}
|
||||
steps:
|
||||
- name: publish
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
{env_block}
|
||||
settings:
|
||||
{settings_block}
|
||||
backend_options:
|
||||
kubernetes:
|
||||
resources:
|
||||
requests:
|
||||
memory: {repo["memory_request"]}
|
||||
limits:
|
||||
memory: {repo["memory_limit"]}
|
||||
"""
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", type=pathlib.Path, required=True)
|
||||
args = ap.parse_args()
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
manifest = yaml.safe_load((HERE / "manifest.yaml").read_text())
|
||||
for repo in manifest["repos"]:
|
||||
name = pathlib.Path(repo["dir"]).name
|
||||
target = args.out / f"{name}.woodpecker.yaml"
|
||||
target.write_text(render_identical(repo))
|
||||
print(f"wrote {target}")
|
||||
for repo in manifest.get("thin_repos", []):
|
||||
name = pathlib.Path(repo["dir"]).name
|
||||
target = args.out / f"{name}.woodpecker.yaml"
|
||||
target.write_text(render_thin_parity(repo))
|
||||
print(f"wrote {target}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,348 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render thin ci-fleet-publish-based pipelines for the quartet family
|
||||
(oleks/ci-scripts#17, #19) — PREP/PLANNING script, not wired into anything yet.
|
||||
|
||||
SHAPES (from manifests/*.yaml, `kind:`):
|
||||
- xonsh-image, csi-s3: kind: matrix — real buildx per-arch-build +
|
||||
manifest-consolidate quartets. Maps onto the plugin's
|
||||
buildx-build/buildx-manifest flow. (oleks/ci-scripts#17, done first.)
|
||||
- the-eye, emdash.kotkanagrilli.fi: kind: single — one native-arch
|
||||
`docker buildx` build via `ci/local.sh`, no manifest step (the-eye also
|
||||
has a second `grpc` component, same shape, second step in the same
|
||||
file). Maps onto the plugin's `flow: publish` — same single-step shape
|
||||
render_thin_fleet.py uses for the fleet family.
|
||||
- flake-hub, woodpecker-peek: kind: split — separate per-arch files
|
||||
(amd64.yaml/arm64.yaml), `nix run .#publish[-<arch>]` against parity-lib's
|
||||
attic-closure archetype, no docker buildx at all. Also maps onto
|
||||
`flow: publish`, one thin step per arch file — per-arch file layout is
|
||||
kept (NOT collapsed into one matrix file): the two arches build against
|
||||
different node-bound PVCs and were already split for that reason (see
|
||||
manifests/flake-hub.yaml header), a constraint the thin migration doesn't
|
||||
change.
|
||||
|
||||
(oleks/ci-scripts#19 implements the single/split renderers below; the matrix
|
||||
renderer was already done for #17.)
|
||||
|
||||
For xonsh-image/csi-s3, migrating to the plugin's buildx-build/buildx-manifest
|
||||
flow means the plugin's generic `docker buildx build --push <context>`
|
||||
(building the repo's own Dockerfile directly) REPLACES what `ci/local.sh
|
||||
--arch/--manifest` used to do internally. This is a bigger behavioral swap
|
||||
than the fleet family's migration (which just moves inline shell into
|
||||
`run:`) — needs explicit confirmation that ci/local.sh's Dockerfile build is
|
||||
the ONLY thing it does before this ships (see report).
|
||||
|
||||
`kind: matrix` manifests must set `image_name:` explicitly, verified against
|
||||
the live Gitea package registry (oleks/ci-scripts#22). It is NEVER inferred
|
||||
from the manifest's `repo:`/dir field — `xonsh-image`'s repo field and its
|
||||
actually-published image (`oleks/xonsh`, confirmed live) genuinely differ;
|
||||
inferring one from the other silently pushes a migrated pipeline to a
|
||||
different, wrong (and possibly empty) image path.
|
||||
|
||||
For the-eye/emdash (single) and flake-hub/woodpecker-peek (split), the
|
||||
existing pipelines already invoke a repo-owned script (`ci/local.sh`,
|
||||
`nix run .#publish-*`) as their entire build logic — same shape as the fleet
|
||||
family's `run:` migration, not the bigger buildx-quartet swap above.
|
||||
|
||||
Behavior-preservation notes (single/split):
|
||||
- The manual `echo "▸ arch=$(uname -m)"` command line is dropped from
|
||||
`run:` — entrypoint.sh already prints that banner unconditionally before
|
||||
any flow runs (line ~35), so keeping it in `run:` too would just double
|
||||
it. Same convention render_thin_fleet.py already uses.
|
||||
- `clone:` step overrides are dropped, matching the precedent already set
|
||||
for the fleet family (render_thin_fleet.py) and the matrix renderer
|
||||
below: Woodpecker's implicit clone is judged equivalent for the
|
||||
PLUGIN_TAGS=false/PLUGIN_DEPTH=1 shape all four of these repos use. None
|
||||
of the 4 use the `tags_full` (full-history) shape render_thin_fleet.py
|
||||
flagged as an unresolved gap — this repo set doesn't hit that gap.
|
||||
- `backend_options.kubernetes.nodeSelector`/`resources`/`labels` (the
|
||||
per-pod tracing labels: commit-branch/commit-sha/commit-tag/
|
||||
pipeline-number) are NOT manifest-parametrized inputs today — render.py's
|
||||
single/split templates hardcode them per component/leg. They are
|
||||
preserved here as the same fixed shape, still populated from the
|
||||
manifest's `arch`/leg key.
|
||||
- ATTIC_TOKEN handling: split legs already have `attic_token` wired as a
|
||||
plain `environment:` secret (consumed by nixos-ci-entrypoint's own
|
||||
ATTIC_TOKEN handling today). The plugin's `attic_start`/`attic_stop`
|
||||
(entrypoint.sh) already replaces that lifecycle when `cache: "true"` —
|
||||
same mapping render_thin_fleet.py uses for the identical-repos family.
|
||||
the-eye/emdash never reference ATTIC_TOKEN today, so they get
|
||||
`cache: "false"`, not silently turned on.
|
||||
|
||||
Usage:
|
||||
python3 generator/render_thin_quartet.py --out /path/to/scratch
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import pathlib
|
||||
|
||||
import yaml
|
||||
|
||||
HERE = pathlib.Path(__file__).parent.parent
|
||||
PLUGIN_IMAGE = "git.oleks.space/oleks/ci-fleet-publish:v1"
|
||||
|
||||
GENERATED_HEADER = (
|
||||
"# GENERATED by oleks/ci-scripts — do not hand-edit.\n"
|
||||
"# Source: oleks/ci-scripts manifests/{repo}.yaml + "
|
||||
"generator/render_thin_quartet.py (oleks/ci-scripts#17, #19).\n"
|
||||
"# Re-render with: python3 generator/render_thin_quartet.py --out <repo>/.woodpecker\n"
|
||||
"\n"
|
||||
)
|
||||
|
||||
|
||||
def render_build(manifest: dict, arch: str, image_name: str) -> str:
|
||||
node_selector = manifest.get("node_selector")
|
||||
backend_options_block = ""
|
||||
if node_selector:
|
||||
lines = "\n".join(f" {k}: {v}" for k, v in node_selector.items())
|
||||
backend_options_block = (
|
||||
" backend_options:\n kubernetes:\n nodeSelector:\n"
|
||||
+ lines
|
||||
+ "\n"
|
||||
)
|
||||
# oleks/ci-scripts#21: opt-in only — reproduces a repo's own ci/local.sh
|
||||
# tag normalization (strip leading "v" + trailing "-N"). Default off
|
||||
# (entrypoint.sh's resolve_version() default matches today's raw-tag
|
||||
# behavior), set per-manifest via `tag_strip_v: true`.
|
||||
tag_strip_v_line = (
|
||||
' tag_strip_v: "true"\n' if manifest.get("tag_strip_v") else ""
|
||||
)
|
||||
|
||||
return f"""{GENERATED_HEADER}labels:
|
||||
arch: {arch}
|
||||
|
||||
when:
|
||||
- event: tag
|
||||
ref: "refs/tags/v*"
|
||||
|
||||
steps:
|
||||
- name: build-{arch}
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
REGISTRY_TOKEN:
|
||||
from_secret: registry_token
|
||||
settings:
|
||||
flow: buildx-build
|
||||
image_name: "{image_name}"
|
||||
tag_scheme: "{{version}}-{{arch}}"
|
||||
{tag_strip_v_line}{backend_options_block}"""
|
||||
|
||||
|
||||
def render_manifest(manifest: dict, arches: list, image_name: str) -> str:
|
||||
platforms = ",".join(arches)
|
||||
tag_strip_v_line = (
|
||||
' tag_strip_v: "true"\n' if manifest.get("tag_strip_v") else ""
|
||||
)
|
||||
return f"""{GENERATED_HEADER}when:
|
||||
- event: tag
|
||||
ref: "refs/tags/v*"
|
||||
|
||||
depends_on:
|
||||
{chr(10).join(f" - build-{a}" for a in arches)}
|
||||
|
||||
# oleks/ci-scripts#9: run even if a per-arch build failed — the plugin's
|
||||
# buildx-manifest flow itself probes the registry for which per-arch tags
|
||||
# actually landed and builds the manifest from whatever exists.
|
||||
runs_on:
|
||||
- success
|
||||
- failure
|
||||
|
||||
steps:
|
||||
- name: manifest
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
REGISTRY_TOKEN:
|
||||
from_secret: registry_token
|
||||
settings:
|
||||
flow: buildx-manifest
|
||||
image_name: "{image_name}"
|
||||
platforms: "{platforms}"
|
||||
tag_scheme: "{{version}}-{{arch}}"
|
||||
{tag_strip_v_line}"""
|
||||
|
||||
|
||||
def _comment_block(text: str, indent: str) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
lines = text.rstrip("\n").splitlines()
|
||||
return "".join(f"{indent}# {line}\n" if line else f"{indent}#\n" for line in lines)
|
||||
|
||||
|
||||
def render_single_step(component: dict, arch: str) -> str:
|
||||
"""One thin `flow: publish` step for a `kind: single` component
|
||||
(the-eye's build-and-push / build-and-push-grpc, emdash's build-and-push).
|
||||
Mirrors templates/single/COMPONENT_STEP.tmpl's fixed env/backend_options
|
||||
shape, swapping the `commands:` block for a `settings: {{run, flow, cache}}`.
|
||||
"""
|
||||
command = f"./ci/local.sh --arch {arch}"
|
||||
if component.get("component"):
|
||||
command += f" --component {component['component']}"
|
||||
step_comment = _comment_block(component.get("step_comment", ""), " ")
|
||||
buildkit_comment = _comment_block(component.get("buildkit_comment", ""), " ")
|
||||
version_comment = _comment_block(component.get("version_comment", ""), " ")
|
||||
memory = component.get("memory", "4Gi")
|
||||
buildkit_addr = f"tcp://buildkit-{arch}.infra.svc.cluster.local:1234"
|
||||
|
||||
return f"""{step_comment} - name: {component["name"]}
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
REGISTRY_TOKEN:
|
||||
from_secret: registry_token
|
||||
{buildkit_comment} BUILDKIT_ADDR: {buildkit_addr}
|
||||
PUBLISH: "1"
|
||||
BRANCH: "${{CI_COMMIT_BRANCH}}"
|
||||
{version_comment} VERSION: "0.1.${{CI_PIPELINE_NUMBER}}"
|
||||
settings:
|
||||
run: |
|
||||
{command}
|
||||
flow: publish
|
||||
cache: "false"
|
||||
backend_options:
|
||||
kubernetes:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: {arch}
|
||||
resources:
|
||||
requests:
|
||||
memory: {memory}
|
||||
limits:
|
||||
memory: {memory}
|
||||
labels:
|
||||
commit-branch: "${{CI_COMMIT_BRANCH}}"
|
||||
commit-sha: "${{CI_COMMIT_SHA}}"
|
||||
pipeline-number: "${{CI_PIPELINE_NUMBER}}"
|
||||
"""
|
||||
|
||||
|
||||
def render_single(manifest: dict) -> dict:
|
||||
"""`kind: single` (the-eye, emdash.kotkanagrilli.fi) → one file,
|
||||
one-or-more thin `flow: publish` steps (the-eye has a second `grpc`
|
||||
component alongside the web build)."""
|
||||
arch = manifest["arch"]
|
||||
branches = "[" + ", ".join(manifest["branches"]) + "]"
|
||||
header = manifest["header"].rstrip("\n") + "\n"
|
||||
label_lines = (
|
||||
_comment_block(manifest.get("label_comment", ""), " ") + f" arch: {arch}"
|
||||
)
|
||||
|
||||
steps = "\n".join(
|
||||
render_single_step(component, arch) for component in manifest["components"]
|
||||
)
|
||||
|
||||
body = f"""{header}
|
||||
labels:
|
||||
{label_lines}
|
||||
|
||||
when:
|
||||
- event: push
|
||||
branch: {branches}
|
||||
|
||||
steps:
|
||||
{steps}"""
|
||||
return {"container.yaml": GENERATED_HEADER.format(repo=manifest["repo"]) + body}
|
||||
|
||||
|
||||
def render_split_leg(manifest: dict, arch: str, leg: dict) -> str:
|
||||
"""One thin `flow: publish` file for a `kind: split` leg (flake-hub's
|
||||
amd64.yaml/arm64.yaml, woodpecker-peek's amd64.yaml/arm64.yaml)."""
|
||||
setup_script = manifest.get("setup_script", "ci/setup.sh")
|
||||
header = leg["header"].rstrip("\n") + "\n"
|
||||
entrypoint_comment = _comment_block(leg.get("entrypoint_comment", ""), " ")
|
||||
tag_event_block = (
|
||||
"\n - event: tag\n" if manifest.get("when_tag_event", True) else "\n"
|
||||
)
|
||||
publish_flag = " -- --publish" if leg.get("publish_flag", True) else ""
|
||||
|
||||
run_lines = f" sh {setup_script}\n"
|
||||
if entrypoint_comment:
|
||||
run_lines += entrypoint_comment
|
||||
run_lines += f" PUBLISH=1 nix run .#{leg['app']}{publish_flag}\n"
|
||||
|
||||
return f"""{GENERATED_HEADER.format(repo=manifest["repo"])}{header}
|
||||
when:
|
||||
- event: push
|
||||
branch: main{tag_event_block}
|
||||
steps:
|
||||
- name: build-{arch}
|
||||
image: {PLUGIN_IMAGE}
|
||||
environment:
|
||||
ATTIC_TOKEN:
|
||||
from_secret: attic_token
|
||||
GITEA_CLONE_TOKEN:
|
||||
from_secret: gitea_clone_token
|
||||
settings:
|
||||
run: |
|
||||
{run_lines} flow: publish
|
||||
cache: "true"
|
||||
backend_options:
|
||||
kubernetes:
|
||||
nodeSelector:
|
||||
kubernetes.io/arch: {arch}
|
||||
labels:
|
||||
commit-tag: "${{CI_COMMIT_TAG}}"
|
||||
commit-branch: "${{CI_COMMIT_BRANCH}}"
|
||||
pipeline-number: "${{CI_PIPELINE_NUMBER}}"
|
||||
"""
|
||||
|
||||
|
||||
def render_split(manifest: dict) -> dict:
|
||||
"""`kind: split` (flake-hub, woodpecker-peek) → one file per arch leg,
|
||||
kept separate (never collapsed into a single matrix file — the two arches
|
||||
build against different node-bound PVCs, see manifests/flake-hub.yaml)."""
|
||||
return {
|
||||
f"{arch}.yaml": render_split_leg(manifest, arch, leg)
|
||||
for arch, leg in manifest["legs"].items()
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", type=pathlib.Path, required=True)
|
||||
args = ap.parse_args()
|
||||
|
||||
manifests_dir = HERE / "manifests"
|
||||
for path in sorted(manifests_dir.glob("*.yaml")):
|
||||
manifest = yaml.safe_load(path.read_text())
|
||||
kind = manifest.get("kind")
|
||||
repo = manifest["repo"]
|
||||
out_dir = args.out / repo
|
||||
|
||||
if kind == "matrix":
|
||||
# oleks/ci-scripts#22: image_name must be explicit in the
|
||||
# manifest, never inferred from the repo/dir name — the two
|
||||
# silently diverge for xonsh-image (repo field "xonsh-image",
|
||||
# actually-published image "oleks/xonsh") and inference would
|
||||
# have pushed a migrated pipeline to an empty, wrong image path.
|
||||
image_name = manifest.get("image_name")
|
||||
if not image_name:
|
||||
raise SystemExit(
|
||||
f"manifest for {repo!r} (kind=matrix) is missing required "
|
||||
"'image_name' field (oleks/ci-scripts#22: must be explicit, "
|
||||
"verified against the live package registry — do not infer "
|
||||
"it from the repo/dir name)"
|
||||
)
|
||||
arches = manifest["arches"]
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for arch in arches:
|
||||
target = out_dir / f"build-{arch}.yaml"
|
||||
target.write_text(render_build(manifest, arch, image_name))
|
||||
print(f"wrote {target}")
|
||||
target = out_dir / "manifest.yaml"
|
||||
target.write_text(render_manifest(manifest, arches, image_name))
|
||||
print(f"wrote {target}")
|
||||
elif kind == "single":
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name, body in render_single(manifest).items():
|
||||
target = out_dir / name
|
||||
target.write_text(body)
|
||||
print(f"wrote {target}")
|
||||
elif kind == "split":
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name, body in render_split(manifest).items():
|
||||
target = out_dir / name
|
||||
target.write_text(body)
|
||||
print(f"wrote {target}")
|
||||
else:
|
||||
print(f"skip {repo} (kind={kind}, unknown)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -22,9 +22,18 @@
|
||||
# cores - Nix `cores` setting written into /etc/nix/nix.conf
|
||||
# max_jobs - whether to also cap `max-jobs = 1` (small/serial builds)
|
||||
# attic - whether to run the inline `attic watch-store` accelerator
|
||||
# no_build - if true, render a NEUTRALIZED pipeline (event: manual, no
|
||||
# build/publish). s390x = no-build: s390x hardware is
|
||||
# currently UNAVAILABLE, so those wheels are not built —
|
||||
# the repo code is kept as a cross-compile reference only.
|
||||
# Entries under s390x/ default to no_build even without the
|
||||
# flag (generate.py is_no_build). Re-enable by setting
|
||||
# no_build: false. arm64 builds are unaffected (arm64 hw is
|
||||
# available). See wiki Spec/* + memory feedback-no-s390x-builds.
|
||||
|
||||
repos:
|
||||
- dir: s390x/lxml-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 2Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -32,6 +41,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/pillow-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 6Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -39,6 +49,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/psycopg-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 6Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -46,6 +57,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/psycopg2-binary-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 6Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -53,6 +65,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/tiktoken-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 6Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -61,6 +74,7 @@ repos:
|
||||
attic: true
|
||||
redis_cache: true
|
||||
- dir: s390x/duckdb-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -68,6 +82,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/faiss-cpu-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -75,6 +90,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/numpy-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -82,6 +98,7 @@ repos:
|
||||
max_jobs: true
|
||||
attic: true
|
||||
- dir: s390x/pymupdf-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 50Gi
|
||||
@@ -89,6 +106,7 @@ repos:
|
||||
max_jobs: false
|
||||
attic: true
|
||||
- dir: s390x/markupsafe-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 2Gi
|
||||
memory_limit: 6Gi
|
||||
@@ -126,6 +144,7 @@ repos:
|
||||
# bespoke hand-maintained file; flagged alongside fastuuid.
|
||||
thin_repos:
|
||||
- dir: s390x/asyncpg-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -136,6 +155,7 @@ thin_repos:
|
||||
dead_secrets: []
|
||||
gitea_clone_token: false
|
||||
- dir: s390x/cryptography-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -147,6 +167,7 @@ thin_repos:
|
||||
tags_full: false
|
||||
dead_secrets: [attic_token, redis_password]
|
||||
- dir: s390x/lightningcss-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
@@ -156,6 +177,7 @@ thin_repos:
|
||||
tags_full: true
|
||||
dead_secrets: [attic_token, redis_password]
|
||||
- dir: s390x/nextjs-swc-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 50Gi
|
||||
@@ -165,6 +187,7 @@ thin_repos:
|
||||
tags_full: true
|
||||
dead_secrets: [attic_token, redis_password]
|
||||
- dir: s390x/rollup-s390x
|
||||
no_build: true
|
||||
arch: amd64
|
||||
memory_request: 8Gi
|
||||
memory_limit: 32Gi
|
||||
|
||||
@@ -3,6 +3,15 @@ kind: matrix
|
||||
explicit_clone: true
|
||||
image: git.oleks.space/oleks/nix-ci:latest
|
||||
script: ci/local.sh
|
||||
# oleks/ci-scripts#22: matches ci/local.sh's IMAGE_REPO default
|
||||
# ("csi-s3-driver"), NOT the `repo:` field above. Unlike xonsh-image there is
|
||||
# no live package under this name yet to cross-check (csi-s3's --publish path
|
||||
# has never run in CI: no pipeline history, no registry package) — this is
|
||||
# the intended target per the script, not a confirmed-live value. csi-s3
|
||||
# remains EXCLUDED from the #17 migration regardless (oleks/ci-scripts#20:
|
||||
# too bespoke — build-args, dual per-arch tags, skip-if-published, dropped
|
||||
# clone/timeout overrides); this field is added for manifest correctness only.
|
||||
image_name: git.oleks.space/oleks/csi-s3-driver
|
||||
var_name: TARGET_ARCH
|
||||
arches: [amd64, arm64]
|
||||
commented_arches:
|
||||
|
||||
@@ -4,3 +4,14 @@ image: git.oleks.space/oleks/nix-ci
|
||||
script: ci/local.sh
|
||||
arches: [amd64, arm64]
|
||||
buildkit_prefix: buildkit
|
||||
# oleks/ci-scripts#22: the PUBLISHED image is "oleks/xonsh", not
|
||||
# "oleks/xonsh-image" — ci/local.sh's IMAGE_REPO default is "xonsh", and the
|
||||
# live Gitea package registry confirms it (tags 0.22.7, 0.22.7-amd64,
|
||||
# 0.22.7-arm64, latest all under package "xonsh"). Do not derive this from
|
||||
# the `repo:` field above.
|
||||
image_name: git.oleks.space/oleks/xonsh
|
||||
# oleks/ci-scripts#21: ci/local.sh strips a leading "v" (and a trailing
|
||||
# "-N" build suffix) from CI_COMMIT_TAG before tagging — live published tags
|
||||
# are "0.22.7"/"0.22.7-<arch>", not "v0.22.7-<arch>". Reproduces that exactly
|
||||
# via entrypoint.sh's opt-in resolve_version() normalization.
|
||||
tag_strip_v: true
|
||||
|
||||
Reference in New Issue
Block a user