265 lines
8.6 KiB
Python
265 lines
8.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Render .woodpecker.yaml for the pypi-wheel-attic fleet family.
|
|
|
|
GENERATED files carry a "do not hand-edit" header; edit manifest.yaml instead
|
|
and re-run this script. See manifest.yaml for scope notes (oleks/ci-scripts#1).
|
|
"""
|
|
|
|
import pathlib
|
|
import sys
|
|
|
|
import yaml
|
|
|
|
HERE = pathlib.Path(__file__).parent
|
|
BUILDING = HERE.parent / "building"
|
|
|
|
HEADER = """# GENERATED by oleks/ci-scripts — do not hand-edit.
|
|
# Source: oleks/ci-scripts manifest.yaml + generate.py (oleks/ci-scripts#1).
|
|
# Re-render with: python3 generate.py
|
|
"""
|
|
|
|
|
|
def render(repo: dict) -> str:
|
|
arch = repo["arch"]
|
|
mem_req = repo["memory_request"]
|
|
mem_lim = repo["memory_limit"]
|
|
cores = repo.get("cores")
|
|
max_jobs = repo.get("max_jobs", False)
|
|
attic = repo.get("attic", True)
|
|
redis_cache = repo.get("redis_cache", False)
|
|
|
|
nix_conf_lines = []
|
|
if max_jobs:
|
|
nix_conf_lines.append(' - echo "max-jobs = 1" >> /etc/nix/nix.conf')
|
|
if cores:
|
|
nix_conf_lines.append(f' - echo "cores = {cores}" >> /etc/nix/nix.conf')
|
|
nix_conf_lines.append(' - echo "sandbox = false" >> /etc/nix/nix.conf')
|
|
nix_conf_block = "\n".join(nix_conf_lines)
|
|
|
|
env_lines = [
|
|
" GITEA_CLONE_TOKEN:",
|
|
" from_secret: gitea_clone_token",
|
|
" REGISTRY_TOKEN:",
|
|
" from_secret: registry_token",
|
|
]
|
|
if attic:
|
|
env_lines += [
|
|
" ATTIC_TOKEN:",
|
|
" from_secret: attic_token",
|
|
]
|
|
if redis_cache:
|
|
env_lines += [
|
|
" REDIS_PASSWORD:",
|
|
" from_secret: redis_password",
|
|
]
|
|
env_block = "\n".join(env_lines)
|
|
|
|
redis_block = ""
|
|
if redis_cache:
|
|
redis_block = (
|
|
" # Set up sccache/ccache with Redis for the cross build.\n"
|
|
" - export SCCACHE_REDIS_ENDPOINT=tcp://redis.default.svc.cluster.local:6379\n"
|
|
" - export SCCACHE_REDIS_PASSWORD=$REDIS_PASSWORD\n"
|
|
" - export CCACHE_REMOTE_STORAGE=redis://redis.default.svc.cluster.local:6379\n"
|
|
" - export CCACHE_REMOTE_ONLY=1\n"
|
|
)
|
|
|
|
if attic:
|
|
attic_block = (
|
|
" # Push built paths to the binary cache while we build.\n"
|
|
' - attic login ci https://nix-cache-upload.oleks.space "$ATTIC_TOKEN"\n'
|
|
" - attic watch-store attic-infra-cache-k3s-1 &\n"
|
|
" - ATTIC_PID=$!\n"
|
|
" - sleep 2\n"
|
|
)
|
|
attic_teardown = (
|
|
' - if ! kill $ATTIC_PID 2>/dev/null; then echo "attic watch-store already exited"; fi\n'
|
|
' - if ! wait $ATTIC_PID 2>/dev/null; then echo "attic watch-store exited non-zero after kill (expected)"; fi\n'
|
|
)
|
|
else:
|
|
attic_block = ""
|
|
attic_teardown = ""
|
|
|
|
return f"""{HEADER}labels:
|
|
arch: {arch}
|
|
|
|
clone:
|
|
- name: clone
|
|
image: woodpeckerci/plugin-git
|
|
environment:
|
|
CI_NETRC_MACHINE: git.oleks.space
|
|
CI_NETRC_USERNAME: oleks
|
|
CI_NETRC_PASSWORD:
|
|
from_secret: gitea_clone_token
|
|
PLUGIN_TAGS: "false"
|
|
PLUGIN_DEPTH: "1"
|
|
|
|
when:
|
|
- event: tag
|
|
ref: "refs/tags/v*"
|
|
|
|
steps:
|
|
- name: build-and-publish
|
|
image: git.oleks.space/oleks/nix-ci-ccpp-rust:latest
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: {mem_req}
|
|
limits:
|
|
memory: {mem_lim}
|
|
environment:
|
|
{env_block}
|
|
commands:
|
|
- echo "▸ arch=$(uname -m)"
|
|
- echo "nameserver 169.254.20.10" > /etc/resolv.conf && echo "options ndots:1" >> /etc/resolv.conf
|
|
{nix_conf_block}
|
|
- nixos-ci-entrypoint true
|
|
{redis_block}{attic_block} # Thin parity entrypoint: build + publish all s390x wheels (CI mutates).
|
|
- PUBLISH=1 nix run .#publish
|
|
{attic_teardown}"""
|
|
|
|
|
|
SECRET_ENV = {
|
|
"attic_token": ("ATTIC_TOKEN", "attic_token"),
|
|
"redis_password": ("REDIS_PASSWORD", "redis_password"),
|
|
}
|
|
|
|
|
|
def render_thin(repo: dict) -> str:
|
|
"""Render the "thin parity-lib publish" family (oleks/ci-scripts#10):
|
|
no inline attic accelerator, publish is a pure `nix run .#publish[-s390x]`
|
|
front door, optionally gated by a pipeline-doctor step first."""
|
|
arch = repo["arch"]
|
|
mem_req = repo["memory_request"]
|
|
mem_lim = repo["memory_limit"]
|
|
cores = repo.get("cores")
|
|
max_jobs = repo.get("max_jobs", False)
|
|
doctor = repo.get("doctor", False)
|
|
token_direct = repo.get("token_direct", True)
|
|
publish_style = repo.get("publish_style", "publish1")
|
|
tags_full = repo.get("tags_full", False)
|
|
dead_secrets = repo.get("dead_secrets", [])
|
|
gitea_clone_token = repo.get("gitea_clone_token", True)
|
|
|
|
plugin_tags = "true" if tags_full else "false"
|
|
plugin_depth = "0" if tags_full else "1"
|
|
|
|
doctor_block = ""
|
|
if doctor:
|
|
doctor_block = f""" # ENFORCEMENT gate (cluster #193): run parity-lib's pipeline-doctor --strict
|
|
# against THIS repo BEFORE publishing. Read-only, no registry contact, no
|
|
# token. The doctor's non-zero exit fails the pipeline on any parity-contract
|
|
# violation (or, in --strict, any warning), so a drifted repo never publishes.
|
|
- name: doctor
|
|
image: git.oleks.space/oleks/nix-ci-ccpp-rust:latest
|
|
commands:
|
|
- echo "▸ arch=$(uname -m)"
|
|
- nix run git+https://git.oleks.space/oleks/parity-lib#pipeline-doctor -- . --strict
|
|
|
|
"""
|
|
|
|
env_lines = []
|
|
if gitea_clone_token:
|
|
env_lines += [" GITEA_CLONE_TOKEN:", " from_secret: gitea_clone_token"]
|
|
if token_direct:
|
|
env_lines += [" REGISTRY_TOKEN:", " from_secret: registry_token"]
|
|
else:
|
|
env_lines += [" CI_REGISTRY_TOKEN:", " from_secret: registry_token"]
|
|
for dead in dead_secrets:
|
|
name, secret = SECRET_ENV[dead]
|
|
env_lines += [f" {name}:", f" from_secret: {secret}"]
|
|
env_block = "\n".join(env_lines)
|
|
|
|
nix_conf_lines = []
|
|
if max_jobs:
|
|
nix_conf_lines.append(' - echo "max-jobs = 1" >> /etc/nix/nix.conf')
|
|
if cores:
|
|
nix_conf_lines.append(f' - echo "cores = {cores}" >> /etc/nix/nix.conf')
|
|
nix_conf_lines.append(' - echo "sandbox = false" >> /etc/nix/nix.conf')
|
|
nix_conf_block = "\n".join(nix_conf_lines)
|
|
|
|
token_remap = ""
|
|
if not token_direct:
|
|
token_remap = (
|
|
" # Thin parity-lib publish: build + publish all built versions on a v* tag.\n"
|
|
" # The mkPyPiWheelPublishMulti app handles staging, idempotent 409-skip\n"
|
|
" # uploads, and token resolution (cluster #192/#197, emmett#44).\n"
|
|
' - export REGISTRY_TOKEN="$CI_REGISTRY_TOKEN"\n'
|
|
)
|
|
|
|
if publish_style == "publish1":
|
|
publish_cmd = " - PUBLISH=1 nix run .#publish"
|
|
else:
|
|
publish_cmd = (
|
|
" # Same front door as local; --publish flips it out of dry-run.\n"
|
|
" - nix run .#publish-s390x -- --publish"
|
|
)
|
|
|
|
return f"""{HEADER}labels:
|
|
arch: {arch}
|
|
|
|
clone:
|
|
- name: clone
|
|
image: woodpeckerci/plugin-git
|
|
environment:
|
|
CI_NETRC_MACHINE: git.oleks.space
|
|
CI_NETRC_USERNAME: oleks
|
|
CI_NETRC_PASSWORD:
|
|
from_secret: gitea_clone_token
|
|
PLUGIN_TAGS: "{plugin_tags}"
|
|
PLUGIN_DEPTH: "{plugin_depth}"
|
|
|
|
when:
|
|
- event: tag
|
|
ref: "refs/tags/v*"
|
|
steps:
|
|
{doctor_block} - name: publish
|
|
image: git.oleks.space/oleks/nix-ci-ccpp-rust:latest
|
|
backend_options:
|
|
kubernetes:
|
|
resources:
|
|
requests:
|
|
memory: {mem_req}
|
|
limits:
|
|
memory: {mem_lim}
|
|
environment:
|
|
{env_block}
|
|
commands:
|
|
- echo "▸ arch=$(uname -m)"
|
|
- echo "nameserver 169.254.20.10" > /etc/resolv.conf && echo "options ndots:1" >> /etc/resolv.conf
|
|
{nix_conf_block}
|
|
- nixos-ci-entrypoint true
|
|
{token_remap}{publish_cmd}
|
|
"""
|
|
|
|
|
|
def main():
|
|
manifest = yaml.safe_load((HERE / "manifest.yaml").read_text())
|
|
check_only = "--check" in sys.argv
|
|
changed = []
|
|
jobs = [(r, render) for r in manifest["repos"]] + [
|
|
(r, render_thin) for r in manifest.get("thin_repos", [])
|
|
]
|
|
for repo, render_fn in jobs:
|
|
target = BUILDING / repo["dir"] / ".woodpecker.yaml"
|
|
rendered = render_fn(repo)
|
|
if check_only:
|
|
current = target.read_text() if target.exists() else ""
|
|
if current != rendered:
|
|
changed.append(str(target))
|
|
else:
|
|
target.write_text(rendered)
|
|
print(f"wrote {target}")
|
|
if check_only:
|
|
if changed:
|
|
print("NOT up to date:")
|
|
for c in changed:
|
|
print(f" {c}")
|
|
sys.exit(1)
|
|
print("all rendered files up to date")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|