From d3c4c8e585aebdf23ea4a9ad397cfdee02cf694f Mon Sep 17 00:00:00 2001 From: repo-worker Date: Sat, 4 Jul 2026 19:33:34 +0300 Subject: [PATCH] Add thin-parity-lib-publish template + disposition of #10's 9 repos --- README.md | 46 +++++++++++++++---- generate.py | 121 +++++++++++++++++++++++++++++++++++++++++++++++++- manifest.yaml | 98 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 245 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 415452e..009c7ce 100644 --- a/README.md +++ b/README.md @@ -13,15 +13,10 @@ s390x cross-build repos that share the "single build-and-publish step, inline Generated files carry a `GENERATED by oleks/ci-scripts — do not hand-edit` header; edit `manifest.yaml` and re-run `python3 generate.py` instead. -Not every s390x/arm64 repo fits this template — some (`asyncpg`, -`cryptography`, `fastuuid`, `lightningcss`, `nextjs-swc`, `rollup`, -`onnxruntime`, `scikit-learn`, `scipy`) run a genuinely different "thin -parity-lib publish" pipeline with per-repo Redis sccache wiring and a -pipeline-doctor gate — those differences are load-bearing, not copy-paste -drift, and were deliberately left un-templated (see the scope note atop -`manifest.yaml`). `geesefs-s390x`, `sentry-cli-s390x`, `attic-client-s390x`, -`angie-arm64`, `devpi-arm64`, and `mempalace` are bespoke pipelines (OCI image -publish, binary builds, multi-file workflows) and are also out of scope. +`geesefs-s390x`, `sentry-cli-s390x`, `attic-client-s390x`, `angie-arm64`, +`devpi-arm64`, and `mempalace` are bespoke pipelines (OCI image publish, +binary builds, multi-file workflows) and are out of scope for either +template. Verify with: @@ -30,6 +25,39 @@ python3 generate.py --check # fails if any rendered file is stale woodpecker-cli lint /.woodpecker.yaml ``` +## thin-parity-lib-publish template (oleks/ci-scripts#10) + +The same `generate.py` also renders a second, genuinely different shape via +`render_thin()` + the `thin_repos:` list in `manifest.yaml`: no inline attic +accelerator, publish is a pure `nix run .#publish[-s390x]` front door, +optionally gated by a `pipeline-doctor --strict` step first. Covers +`asyncpg`, `cryptography`, `lightningcss`, `nextjs-swc`, `rollup`-s390x. + +Real per-repo variance parameterized (not papered over): `doctor` gate +presence, `token_direct` (`REGISTRY_TOKEN` straight from secret vs. +`CI_REGISTRY_TOKEN` + explicit remap), `publish_style` (`PUBLISH=1 nix run +.#publish` vs. `nix run .#publish-s390x -- --publish`), `tags_full` (full +clone with tags vs. shallow), and `dead_secrets` (some repos declare +`ATTIC_TOKEN`/`REDIS_PASSWORD` but never reference them — preserved as-is, +not this issue's scope to clean up). + +Excluded after re-audit, not forced: + +- `fastuuid-s390x` — declares `CI_REGISTRY_TOKEN` but has **no** `export + REGISTRY_TOKEN=...` remap before `PUBLISH=1 nix run .#publish` + (`cryptography-s390x`, the only other `CI_REGISTRY_TOKEN` user, does have + the remap). This looks like a real latent bug — publish would run with + `REGISTRY_TOKEN` unset. Flagged, not silently fixed via templating. +- `onnxruntime-s390x` — same `CI_REGISTRY_TOKEN`-without-remap pattern as + `fastuuid-s390x`, and it's actually attic+redis shaped (single step) like + the `pypi-wheel-attic` family above, not this one. Left hand-maintained. +- `scikit-learn-s390x`, `scipy-s390x` — attic+redis shaped like the + `pypi-wheel-attic` family, but their Redis setup writes exports to a file + via a heredoc that's never sourced elsewhere in the step, unlike + `tiktoken-s390x`'s direct-export style. Whether that heredoc form actually + does anything meaningful isn't confirmed; normalizing it to direct exports + would be a real behavior change, not a safe refactor. Left hand-maintained. + ## Node-pinning convention (oleks/ci-scripts#5) Woodpecker agents advertise `custom_labels` of `{arch, worker}` (e.g. diff --git a/generate.py b/generate.py index c5df91f..9e49365 100644 --- a/generate.py +++ b/generate.py @@ -120,13 +120,130 @@ steps: {attic_teardown}""" +SECRET_ENV = { + "attic_token": ("ATTIC_TOKEN", "attic_token"), + "redis_password": ("REDIS_PASSWORD", "redis_password"), +} + + +def render_thin(repo: dict) -> str: + """Render the "thin parity-lib publish" family (oleks/ci-scripts#10): + no inline attic accelerator, publish is a pure `nix run .#publish[-s390x]` + front door, optionally gated by a pipeline-doctor step first.""" + arch = repo["arch"] + mem_req = repo["memory_request"] + mem_lim = repo["memory_limit"] + cores = repo.get("cores") + max_jobs = repo.get("max_jobs", False) + doctor = repo.get("doctor", False) + token_direct = repo.get("token_direct", True) + publish_style = repo.get("publish_style", "publish1") + tags_full = repo.get("tags_full", False) + dead_secrets = repo.get("dead_secrets", []) + gitea_clone_token = repo.get("gitea_clone_token", True) + + plugin_tags = "true" if tags_full else "false" + plugin_depth = "0" if tags_full else "1" + + doctor_block = "" + if doctor: + doctor_block = f""" # ENFORCEMENT gate (cluster #193): run parity-lib's pipeline-doctor --strict + # against THIS repo BEFORE publishing. Read-only, no registry contact, no + # token. The doctor's non-zero exit fails the pipeline on any parity-contract + # violation (or, in --strict, any warning), so a drifted repo never publishes. + - name: doctor + image: git.oleks.space/oleks/nix-ci-ccpp-rust:latest + commands: + - echo "▸ arch=$(uname -m)" + - nix run git+https://git.oleks.space/oleks/parity-lib#pipeline-doctor -- . --strict + +""" + + env_lines = [] + if gitea_clone_token: + env_lines += [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"] + if token_direct: + env_lines += [" REGISTRY_TOKEN:", " from_secret: registry_token"] + else: + env_lines += [" CI_REGISTRY_TOKEN:", " from_secret: registry_token"] + for dead in dead_secrets: + name, secret = SECRET_ENV[dead] + env_lines += [f" {name}:", f" from_secret: {secret}"] + env_block = "\n".join(env_lines) + + nix_conf_lines = [] + if max_jobs: + nix_conf_lines.append(' - echo "max-jobs = 1" >> /etc/nix/nix.conf') + if cores: + nix_conf_lines.append(f' - echo "cores = {cores}" >> /etc/nix/nix.conf') + nix_conf_lines.append(' - echo "sandbox = false" >> /etc/nix/nix.conf') + nix_conf_block = "\n".join(nix_conf_lines) + + token_remap = "" + if not token_direct: + token_remap = ( + " # Thin parity-lib publish: build + publish all built versions on a v* tag.\n" + " # The mkPyPiWheelPublishMulti app handles staging, idempotent 409-skip\n" + " # uploads, and token resolution (cluster #192/#197, emmett#44).\n" + ' - export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"\n' + ) + + if publish_style == "publish1": + publish_cmd = " - PUBLISH=1 nix run .#publish" + else: + publish_cmd = ( + " # Same front door as local; --publish flips it out of dry-run.\n" + " - nix run .#publish-s390x -- --publish" + ) + + return f"""{HEADER}labels: + arch: {arch} + +clone: + - name: clone + image: woodpeckerci/plugin-git + environment: + CI_NETRC_MACHINE: git.oleks.space + CI_NETRC_USERNAME: oleks + CI_NETRC_PASSWORD: + from_secret: gitea_clone_token + PLUGIN_TAGS: "{plugin_tags}" + PLUGIN_DEPTH: "{plugin_depth}" + +when: + - event: tag + ref: "refs/tags/v*" +steps: +{doctor_block} - name: publish + image: git.oleks.space/oleks/nix-ci-ccpp-rust:latest + backend_options: + kubernetes: + resources: + requests: + memory: {mem_req} + limits: + memory: {mem_lim} + environment: +{env_block} + commands: + - echo "▸ arch=$(uname -m)" + - echo "nameserver 169.254.20.10" > /etc/resolv.conf && echo "options ndots:1" >> /etc/resolv.conf +{nix_conf_block} + - nixos-ci-entrypoint true +{token_remap}{publish_cmd} +""" + + def main(): manifest = yaml.safe_load((HERE / "manifest.yaml").read_text()) check_only = "--check" in sys.argv changed = [] - for repo in manifest["repos"]: + jobs = [(r, render) for r in manifest["repos"]] + [ + (r, render_thin) for r in manifest.get("thin_repos", []) + ] + for repo, render_fn in jobs: target = BUILDING / repo["dir"] / ".woodpecker.yaml" - rendered = render(repo) + rendered = render_fn(repo) if check_only: current = target.read_text() if target.exists() else "" if current != rendered: diff --git a/manifest.yaml b/manifest.yaml index 67cd8b7..e518db6 100644 --- a/manifest.yaml +++ b/manifest.yaml @@ -2,15 +2,17 @@ # # Scope note (oleks/ci-scripts#1): only repos that share the SAME pipeline # shape (single build-and-publish step, inline `attic watch-store` accelerator -# optional, PLUGIN_TAGS=false/PLUGIN_DEPTH=1) are templated here. A second, -# genuinely different family (asyncpg/cryptography/fastuuid/lightningcss/ -# nextjs-swc/rollup/onnxruntime/scikit-learn/scipy — "thin parity-lib publish" -# with per-repo Redis sccache and a pipeline-doctor gate) was deliberately left -# out: those differ in real, load-bearing ways (redis cache wiring, doctor -# step, CI_REGISTRY_TOKEN vs REGISTRY_TOKEN), not accidental copy-paste drift. -# Forcing them into this template would either lose functionality or bloat the -# manifest schema for a family that isn't actually duplicated. Tracked as -# follow-up scope, not done here. +# optional, PLUGIN_TAGS=false/PLUGIN_DEPTH=1) are templated here. +# +# A second family ("thin parity-lib publish": no inline attic, pure +# `nix run .#publish[-s390x]` front door) is templated separately below under +# `thin_repos:` — see oleks/ci-scripts#10 for the re-audit that found which of +# the originally-suspected 9 repos actually belong there, and which don't +# (scikit-learn-s390x/scipy-s390x turned out to be attic+redis shaped like +# THIS family, but their Redis setup writes to a file via heredoc rather than +# exporting directly — a real behavioral difference from tiktoken-s390x's +# style, not confirmed safe to normalize, so left hand-maintained rather than +# forced into either template). # # Fields: # dir - path relative to ~/projects/building/ @@ -93,3 +95,81 @@ repos: cores: 2 max_jobs: true attic: false + +# Second family (oleks/ci-scripts#10): "thin parity-lib publish" — no inline +# attic accelerator, publish is a pure `nix run .#publish[-s390x]` front door. +# Real variance across this family (verified per-repo, not assumed): +# doctor - whether a `pipeline-doctor --strict` gate step runs first +# token_direct - true: REGISTRY_TOKEN straight from secret +# false: CI_REGISTRY_TOKEN from secret + explicit +# `export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"` remap +# publish_style - "publish1": `PUBLISH=1 nix run .#publish` +# "publish-arch": `nix run .#publish-s390x -- --publish` +# tags_full - true: PLUGIN_TAGS=true/PLUGIN_DEPTH=0 (full clone, tags) +# false: PLUGIN_TAGS=false/PLUGIN_DEPTH=1 +# dead_secrets - ATTIC_TOKEN/REDIS_PASSWORD declared in `environment:` but +# never referenced in `commands:` on some repos (pre-existing +# dead config, preserved as-is here — not this issue's scope +# to clean up) +# +# Excluded after re-audit (documented, not forced): +# - fastuuid-s390x: declares CI_REGISTRY_TOKEN but has NO +# `export REGISTRY_TOKEN=...` remap before `PUBLISH=1 nix run .#publish` +# (cryptography-s390x, the only other CI_REGISTRY_TOKEN user, DOES have +# the remap). This looks like a real latent bug — publish would run with +# REGISTRY_TOKEN unset. Not silently "fixed" via templating; flagged +# separately, left untouched. +# - onnxruntime-s390x: same CI_REGISTRY_TOKEN-without-remap pattern as +# fastuuid, AND it's actually attic+redis shaped (single step) like the +# family above, not this one — a template would have to either paper over +# the missing remap or diverge from every other repo's shape. Left as a +# bespoke hand-maintained file; flagged alongside fastuuid. +thin_repos: + - dir: s390x/asyncpg-s390x + arch: amd64 + memory_request: 8Gi + memory_limit: 32Gi + doctor: true + token_direct: true + publish_style: publish1 + tags_full: true + dead_secrets: [] + gitea_clone_token: false + - dir: s390x/cryptography-s390x + arch: amd64 + memory_request: 8Gi + memory_limit: 32Gi + cores: 4 + max_jobs: true + doctor: false + token_direct: false + publish_style: publish1 + tags_full: false + dead_secrets: [attic_token, redis_password] + - dir: s390x/lightningcss-s390x + arch: amd64 + memory_request: 8Gi + memory_limit: 32Gi + doctor: false + token_direct: true + publish_style: publish-arch + tags_full: true + dead_secrets: [attic_token, redis_password] + - dir: s390x/nextjs-swc-s390x + arch: amd64 + memory_request: 8Gi + memory_limit: 50Gi + doctor: false + token_direct: true + publish_style: publish-arch + tags_full: true + dead_secrets: [attic_token, redis_password] + - dir: s390x/rollup-s390x + arch: amd64 + memory_request: 8Gi + memory_limit: 32Gi + doctor: false + token_direct: true + publish_style: publish-arch + tags_full: true + dead_secrets: [attic_token, redis_password]