ci: local-pipeline parity via stage/publish flake apps (cluster#192, emmett#44)
attic-closure archetype: no parity-lib builder exists for attic pushes, so
wrap the existing per-arch package build in ci/publish.py (woodpecker-peek
pattern) and expose `nix run .#{stage,publish}-amd64` + `.#publish`.
Two-halves rule: STAGE nix-builds every package in the arch list into the
local store (emmett-buildable); PUBLISH additionally attic-pushes each
closure. Local runs DRY-RUN unless --push/PUBLISH=1; CI sets PUBLISH=1.
The .woodpecker/{amd64,arm64}.yaml now call the same ci/publish.py so CI
and local runs can't drift. arm64 stays node-bound (no emmett cross path),
so it has no local-parity app. ci/build.py becomes a forwarding shim.
This commit is contained in:
Oleks
+13
-81
@@ -1,88 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build all flake-hub packages and push to attic."""
|
||||
"""Deprecated shim — superseded by ci/publish.py (emmett#44, cluster#192).
|
||||
|
||||
Kept so any stale reference keeps working. Forwards to publish.py with the same
|
||||
arch arg and forces a real push (PUBLISH=1), matching the old always-push
|
||||
behaviour. New callers should use ci/publish.py (dry-run by default) or the
|
||||
`nix run .#publish-amd64` flake app.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def run(cmd):
|
||||
print(f"+ {cmd}", flush=True)
|
||||
r = subprocess.run(cmd, shell=True)
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
|
||||
|
||||
def info(cmd):
|
||||
"""Like run(), but tolerant of failure — for non-load-bearing diagnostics."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
subprocess.run(cmd, shell=True)
|
||||
|
||||
|
||||
def build(cmd):
|
||||
"""Run a `nix build`, streaming stderr live; return stdout (the out path)."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True)
|
||||
out, _ = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
sys.exit(proc.returncode)
|
||||
return out.strip()
|
||||
|
||||
|
||||
ARCH = sys.argv[1]
|
||||
ATTIC_CACHE = "attic-infra-cache-k3s-1"
|
||||
ATTIC_SERVER = "https://nix-cache-upload.oleks.space"
|
||||
ATTIC_TOKEN = os.environ["ATTIC_TOKEN"]
|
||||
|
||||
print(f"=== Building flake-hub packages for {ARCH} ===")
|
||||
|
||||
# Environment context for log readers
|
||||
info("nix --version")
|
||||
info("uname -a")
|
||||
info("df -h /nix 2>/dev/null || df -h /")
|
||||
info("cat /proc/meminfo | head -3")
|
||||
|
||||
# Setup attic
|
||||
attic = (
|
||||
build(
|
||||
"nix build --inputs-from . nixpkgs#attic-client --print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
+ "/bin/attic"
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
env = dict(os.environ, PUBLISH="1")
|
||||
sys.exit(
|
||||
subprocess.run(
|
||||
[sys.executable, os.path.join(here, "publish.py"), *sys.argv[1:]], env=env
|
||||
).returncode
|
||||
)
|
||||
run(f"'{attic}' login ci {ATTIC_SERVER} '{ATTIC_TOKEN}'")
|
||||
|
||||
# Packages per arch
|
||||
packages = ["hello-world", "geesefs", "xonsh"]
|
||||
# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's
|
||||
# default builds for Linux/Darwin; we cache the Linux native arches).
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["woodpecker-peek"]
|
||||
# mcp-chrome: cache the proven-green wasm-simd worker only.
|
||||
# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's
|
||||
# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome
|
||||
# issue #1 close comment); CI would just go red on it.
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["mcp-chrome-wasm-simd"]
|
||||
# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which
|
||||
# transitively builds liberation-fonts; fontforge segfaults while generating
|
||||
# the .ttf files (pipeline #40). Package definitions stay in the flake for
|
||||
# local builds — re-enable here once upstream fontforge is fixed.
|
||||
# if ARCH == "x86_64-linux":
|
||||
# packages += ["google-antigravity", "google-antigravity-no-fhs"]
|
||||
if ARCH == "s390x-linux":
|
||||
packages += ["attic-client"]
|
||||
# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite
|
||||
# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3
|
||||
# compiles from source (~5-8 min cold) on the first push after a rev bump.
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["gitea-local-fork"]
|
||||
|
||||
print("Building packages...")
|
||||
for pkg in packages:
|
||||
print(f"--- {pkg} ---")
|
||||
out = build(
|
||||
f"nix build '.#packages.{ARCH}.{pkg}' --print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
run(f"'{attic}' push {ATTIC_CACHE} {out}")
|
||||
|
||||
print(f"Build completed for {ARCH}")
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build all flake-hub packages for one arch and (optionally) push to attic.
|
||||
|
||||
Single source of truth for both CI and local runs — invoked identically by the
|
||||
`publish-<arch>` flake apps and by .woodpecker/amd64.yaml, so the two can't drift
|
||||
(emmett#44, cluster#192, attic-closure archetype).
|
||||
|
||||
STAGE vs PUBLISH (emmett#44 two-halves rule):
|
||||
- STAGE = `nix build` every package in this arch's list into the local /nix
|
||||
store. No network publish. This half is fully cluster-independent
|
||||
and is what runs on emmett.
|
||||
- PUBLISH = STAGE, then `attic push` each closure to the binary cache. The
|
||||
cache lives next to the cluster, so this half shares fate with it.
|
||||
|
||||
DRY-RUN by default: a local run only STAGES and prints the pushes it *would* do.
|
||||
Pushing requires an explicit opt-in:
|
||||
PUBLISH=1 (env) or --push (flag).
|
||||
CI sets PUBLISH=1 so the pipeline actually publishes.
|
||||
|
||||
Token: the attic cache token is $ATTIC_TOKEN. We never read or print its value.
|
||||
Resolution order (local convenience): $ATTIC_TOKEN, else
|
||||
`pass infra/attic/ci_token` if `pass` is available. Named hard-fail otherwise.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ATTIC_CACHE = "attic-infra-cache-k3s-1"
|
||||
ATTIC_SERVER = "https://nix-cache-upload.oleks.space"
|
||||
|
||||
|
||||
def run(cmd, env=None):
|
||||
print(f"+ {cmd}", flush=True)
|
||||
r = subprocess.run(cmd, shell=True, env=env)
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
|
||||
|
||||
def info(cmd):
|
||||
"""Like run(), but tolerant of failure — non-load-bearing diagnostics."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
subprocess.run(cmd, shell=True)
|
||||
|
||||
|
||||
def build(cmd):
|
||||
"""Run a `nix build`, streaming stderr live; return stdout (the out path)."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True)
|
||||
out, _ = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
sys.exit(proc.returncode)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def resolve_token():
|
||||
"""Return the attic token without ever printing it. Named hard-fail."""
|
||||
tok = os.environ.get("ATTIC_TOKEN")
|
||||
if tok:
|
||||
return tok
|
||||
if shutil.which("pass"):
|
||||
r = subprocess.run(
|
||||
["pass", "infra/attic/ci_token"], capture_output=True, text=True
|
||||
)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout.strip().splitlines()[0]
|
||||
sys.exit(
|
||||
"ERROR: no attic token. Set $ATTIC_TOKEN or store it at "
|
||||
"`pass infra/attic/ci_token`. (Refusing to push without a token.)"
|
||||
)
|
||||
|
||||
|
||||
def packages_for(arch):
|
||||
"""The attic-warmed package set for one arch (verbatim from the old build.py)."""
|
||||
packages = ["hello-world", "geesefs", "xonsh"]
|
||||
# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's
|
||||
# default builds for Linux/Darwin; we cache the Linux native arches).
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["woodpecker-peek"]
|
||||
# mcp-chrome: cache the proven-green wasm-simd worker only.
|
||||
# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's
|
||||
# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome
|
||||
# issue #1 close comment); CI would just go red on it.
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["mcp-chrome-wasm-simd"]
|
||||
# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which
|
||||
# transitively builds liberation-fonts; fontforge segfaults while generating
|
||||
# the .ttf files (pipeline #40). Package definitions stay in the flake for
|
||||
# local builds — re-enable here once upstream fontforge is fixed.
|
||||
# if arch == "x86_64-linux":
|
||||
# packages += ["google-antigravity", "google-antigravity-no-fhs"]
|
||||
if arch == "s390x-linux":
|
||||
packages += ["attic-client"]
|
||||
# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite
|
||||
# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3
|
||||
# compiles from source (~5-8 min cold) on the first push after a rev bump.
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["gitea-local-fork"]
|
||||
return packages
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
push = os.environ.get("PUBLISH") == "1" or "--push" in args
|
||||
args = [a for a in args if a != "--push"]
|
||||
if "--help" in args or "-h" in args or not args:
|
||||
print(__doc__)
|
||||
print("usage: publish.py <arch> [--push] (e.g. publish.py x86_64-linux)")
|
||||
sys.exit(0 if args and args[0] in ("--help", "-h") else (0 if args else 2))
|
||||
arch = args[0]
|
||||
|
||||
mode = "PUBLISH" if push else "STAGE (dry-run)"
|
||||
print(f"=== flake-hub :: {arch} :: {mode} ===", flush=True)
|
||||
|
||||
# Environment context for log readers.
|
||||
info("nix --version")
|
||||
info("uname -a")
|
||||
info("df -h /nix 2>/dev/null || df -h /")
|
||||
info("cat /proc/meminfo | head -3")
|
||||
|
||||
packages = packages_for(arch)
|
||||
|
||||
# STAGE: build every package into the local store, collecting out paths.
|
||||
staged = []
|
||||
print("Staging packages...", flush=True)
|
||||
for pkg in packages:
|
||||
print(f"--- {pkg} ---", flush=True)
|
||||
out = build(
|
||||
f"nix build '.#packages.{arch}.{pkg}' "
|
||||
"--print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
staged.append((pkg, out))
|
||||
|
||||
if not push:
|
||||
print("DRY-RUN: would push the following closures:", flush=True)
|
||||
for pkg, out in staged:
|
||||
print(f" attic push {ATTIC_CACHE} {out} ({pkg})", flush=True)
|
||||
print(
|
||||
"Re-run with --push (or PUBLISH=1) to actually publish.", flush=True
|
||||
)
|
||||
return
|
||||
|
||||
# PUBLISH: build attic-client out of nixpkgs, log in, push every closure.
|
||||
# Token-bearing steps below MUST NOT run under shell tracing — resolve_token
|
||||
# returns the secret in-process and we never echo it.
|
||||
token = resolve_token()
|
||||
attic = (
|
||||
build(
|
||||
"nix build --inputs-from . nixpkgs#attic-client "
|
||||
"--print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
+ "/bin/attic"
|
||||
)
|
||||
# Pass the token via argv of a child without printing it: list form so it
|
||||
# never lands in our `+ ...` trace.
|
||||
print(f"+ {attic} login ci {ATTIC_SERVER} <token-hidden>", flush=True)
|
||||
r = subprocess.run([attic, "login", "ci", ATTIC_SERVER, token])
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
for pkg, out in staged:
|
||||
print(f"--- push {pkg} ---", flush=True)
|
||||
run(f"'{attic}' push {ATTIC_CACHE} {out}")
|
||||
print(f"published {arch} closures to {ATTIC_CACHE}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user