diff --git a/.woodpecker/amd64.yaml b/.woodpecker/amd64.yaml index 0d1932d..b846921 100644 --- a/.woodpecker/amd64.yaml +++ b/.woodpecker/amd64.yaml @@ -36,4 +36,6 @@ steps: pipeline-number: "${CI_PIPELINE_NUMBER}" commands: - sh ci/setup.sh - - python3 ci/build.py x86_64-linux + # Same entrypoint as a local `nix run .#publish-amd64 -- --push`. + # PUBLISH=1 makes the shared script actually push (local runs dry-run). + - PUBLISH=1 python3 ci/publish.py x86_64-linux diff --git a/.woodpecker/arm64.yaml b/.woodpecker/arm64.yaml index 365a95c..dfcab43 100644 --- a/.woodpecker/arm64.yaml +++ b/.woodpecker/arm64.yaml @@ -34,4 +34,8 @@ steps: pipeline-number: "${CI_PIPELINE_NUMBER}" commands: - sh ci/setup.sh - - python3 ci/build.py aarch64-linux + # NODE-BOUND LEG (emmett#44, cluster#192): aarch64-linux can't be built on + # emmett (linux/amd64) and these native packages have no cross path, so this + # leg has no local-parity flake app — it must run on an aarch64 node (the + # arch nodeSelector above). Same shared entrypoint as amd64, only arch differs. + - PUBLISH=1 python3 ci/publish.py aarch64-linux diff --git a/ci/build.py b/ci/build.py index 419d087..b49b351 100644 --- a/ci/build.py +++ b/ci/build.py @@ -1,88 +1,20 @@ #!/usr/bin/env python3 -"""Build all flake-hub packages and push to attic.""" +"""Deprecated shim — superseded by ci/publish.py (emmett#44, cluster#192). + +Kept so any stale reference keeps working. Forwards to publish.py with the same +arch arg and forces a real push (PUBLISH=1), matching the old always-push +behaviour. New callers should use ci/publish.py (dry-run by default) or the +`nix run .#publish-amd64` flake app. +""" import os import subprocess import sys - -def run(cmd): - print(f"+ {cmd}", flush=True) - r = subprocess.run(cmd, shell=True) - if r.returncode != 0: - sys.exit(r.returncode) - - -def info(cmd): - """Like run(), but tolerant of failure — for non-load-bearing diagnostics.""" - print(f"+ {cmd}", flush=True) - subprocess.run(cmd, shell=True) - - -def build(cmd): - """Run a `nix build`, streaming stderr live; return stdout (the out path).""" - print(f"+ {cmd}", flush=True) - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True) - out, _ = proc.communicate() - if proc.returncode != 0: - sys.exit(proc.returncode) - return out.strip() - - -ARCH = sys.argv[1] -ATTIC_CACHE = "attic-infra-cache-k3s-1" -ATTIC_SERVER = "https://nix-cache-upload.oleks.space" -ATTIC_TOKEN = os.environ["ATTIC_TOKEN"] - -print(f"=== Building flake-hub packages for {ARCH} ===") - -# Environment context for log readers -info("nix --version") -info("uname -a") -info("df -h /nix 2>/dev/null || df -h /") -info("cat /proc/meminfo | head -3") - -# Setup attic -attic = ( - build( - "nix build --inputs-from . nixpkgs#attic-client --print-build-logs --print-out-paths --no-link" - ) - + "/bin/attic" +here = os.path.dirname(os.path.abspath(__file__)) +env = dict(os.environ, PUBLISH="1") +sys.exit( + subprocess.run( + [sys.executable, os.path.join(here, "publish.py"), *sys.argv[1:]], env=env + ).returncode ) -run(f"'{attic}' login ci {ATTIC_SERVER} '{ATTIC_TOKEN}'") - -# Packages per arch -packages = ["hello-world", "geesefs", "xonsh"] -# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's -# default builds for Linux/Darwin; we cache the Linux native arches). -if ARCH in ("x86_64-linux", "aarch64-linux"): - packages += ["woodpecker-peek"] -# mcp-chrome: cache the proven-green wasm-simd worker only. -# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's -# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome -# issue #1 close comment); CI would just go red on it. -if ARCH in ("x86_64-linux", "aarch64-linux"): - packages += ["mcp-chrome-wasm-simd"] -# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which -# transitively builds liberation-fonts; fontforge segfaults while generating -# the .ttf files (pipeline #40). Package definitions stay in the flake for -# local builds — re-enable here once upstream fontforge is fixed. -# if ARCH == "x86_64-linux": -# packages += ["google-antigravity", "google-antigravity-no-fhs"] -if ARCH == "s390x-linux": - packages += ["attic-client"] -# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite -# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3 -# compiles from source (~5-8 min cold) on the first push after a rev bump. -if ARCH in ("x86_64-linux", "aarch64-linux"): - packages += ["gitea-local-fork"] - -print("Building packages...") -for pkg in packages: - print(f"--- {pkg} ---") - out = build( - f"nix build '.#packages.{ARCH}.{pkg}' --print-build-logs --print-out-paths --no-link" - ) - run(f"'{attic}' push {ATTIC_CACHE} {out}") - -print(f"Build completed for {ARCH}") diff --git a/ci/publish.py b/ci/publish.py new file mode 100644 index 0000000..1197ce2 --- /dev/null +++ b/ci/publish.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Build all flake-hub packages for one arch and (optionally) push to attic. + +Single source of truth for both CI and local runs — invoked identically by the +`publish-` flake apps and by .woodpecker/amd64.yaml, so the two can't drift +(emmett#44, cluster#192, attic-closure archetype). + +STAGE vs PUBLISH (emmett#44 two-halves rule): + - STAGE = `nix build` every package in this arch's list into the local /nix + store. No network publish. This half is fully cluster-independent + and is what runs on emmett. + - PUBLISH = STAGE, then `attic push` each closure to the binary cache. The + cache lives next to the cluster, so this half shares fate with it. + +DRY-RUN by default: a local run only STAGES and prints the pushes it *would* do. +Pushing requires an explicit opt-in: + PUBLISH=1 (env) or --push (flag). +CI sets PUBLISH=1 so the pipeline actually publishes. + +Token: the attic cache token is $ATTIC_TOKEN. We never read or print its value. +Resolution order (local convenience): $ATTIC_TOKEN, else +`pass infra/attic/ci_token` if `pass` is available. Named hard-fail otherwise. +""" + +import os +import shutil +import subprocess +import sys + +ATTIC_CACHE = "attic-infra-cache-k3s-1" +ATTIC_SERVER = "https://nix-cache-upload.oleks.space" + + +def run(cmd, env=None): + print(f"+ {cmd}", flush=True) + r = subprocess.run(cmd, shell=True, env=env) + if r.returncode != 0: + sys.exit(r.returncode) + + +def info(cmd): + """Like run(), but tolerant of failure — non-load-bearing diagnostics.""" + print(f"+ {cmd}", flush=True) + subprocess.run(cmd, shell=True) + + +def build(cmd): + """Run a `nix build`, streaming stderr live; return stdout (the out path).""" + print(f"+ {cmd}", flush=True) + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True) + out, _ = proc.communicate() + if proc.returncode != 0: + sys.exit(proc.returncode) + return out.strip() + + +def resolve_token(): + """Return the attic token without ever printing it. Named hard-fail.""" + tok = os.environ.get("ATTIC_TOKEN") + if tok: + return tok + if shutil.which("pass"): + r = subprocess.run( + ["pass", "infra/attic/ci_token"], capture_output=True, text=True + ) + if r.returncode == 0 and r.stdout.strip(): + return r.stdout.strip().splitlines()[0] + sys.exit( + "ERROR: no attic token. Set $ATTIC_TOKEN or store it at " + "`pass infra/attic/ci_token`. (Refusing to push without a token.)" + ) + + +def packages_for(arch): + """The attic-warmed package set for one arch (verbatim from the old build.py).""" + packages = ["hello-world", "geesefs", "xonsh"] + # woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's + # default builds for Linux/Darwin; we cache the Linux native arches). + if arch in ("x86_64-linux", "aarch64-linux"): + packages += ["woodpecker-peek"] + # mcp-chrome: cache the proven-green wasm-simd worker only. + # mcp-chrome-extension is exposed in the flake but NOT built in CI — it's + # KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome + # issue #1 close comment); CI would just go red on it. + if arch in ("x86_64-linux", "aarch64-linux"): + packages += ["mcp-chrome-wasm-simd"] + # google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which + # transitively builds liberation-fonts; fontforge segfaults while generating + # the .ttf files (pipeline #40). Package definitions stay in the flake for + # local builds — re-enable here once upstream fontforge is fixed. + # if arch == "x86_64-linux": + # packages += ["google-antigravity", "google-antigravity-no-fhs"] + if arch == "s390x-linux": + packages += ["attic-client"] + # gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite + # and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3 + # compiles from source (~5-8 min cold) on the first push after a rev bump. + if arch in ("x86_64-linux", "aarch64-linux"): + packages += ["gitea-local-fork"] + return packages + + +def main(): + args = sys.argv[1:] + push = os.environ.get("PUBLISH") == "1" or "--push" in args + args = [a for a in args if a != "--push"] + if "--help" in args or "-h" in args or not args: + print(__doc__) + print("usage: publish.py [--push] (e.g. publish.py x86_64-linux)") + sys.exit(0 if args and args[0] in ("--help", "-h") else (0 if args else 2)) + arch = args[0] + + mode = "PUBLISH" if push else "STAGE (dry-run)" + print(f"=== flake-hub :: {arch} :: {mode} ===", flush=True) + + # Environment context for log readers. + info("nix --version") + info("uname -a") + info("df -h /nix 2>/dev/null || df -h /") + info("cat /proc/meminfo | head -3") + + packages = packages_for(arch) + + # STAGE: build every package into the local store, collecting out paths. + staged = [] + print("Staging packages...", flush=True) + for pkg in packages: + print(f"--- {pkg} ---", flush=True) + out = build( + f"nix build '.#packages.{arch}.{pkg}' " + "--print-build-logs --print-out-paths --no-link" + ) + staged.append((pkg, out)) + + if not push: + print("DRY-RUN: would push the following closures:", flush=True) + for pkg, out in staged: + print(f" attic push {ATTIC_CACHE} {out} ({pkg})", flush=True) + print( + "Re-run with --push (or PUBLISH=1) to actually publish.", flush=True + ) + return + + # PUBLISH: build attic-client out of nixpkgs, log in, push every closure. + # Token-bearing steps below MUST NOT run under shell tracing — resolve_token + # returns the secret in-process and we never echo it. + token = resolve_token() + attic = ( + build( + "nix build --inputs-from . nixpkgs#attic-client " + "--print-build-logs --print-out-paths --no-link" + ) + + "/bin/attic" + ) + # Pass the token via argv of a child without printing it: list form so it + # never lands in our `+ ...` trace. + print(f"+ {attic} login ci {ATTIC_SERVER} ", flush=True) + r = subprocess.run([attic, "login", "ci", ATTIC_SERVER, token]) + if r.returncode != 0: + sys.exit(r.returncode) + for pkg, out in staged: + print(f"--- push {pkg} ---", flush=True) + run(f"'{attic}' push {ATTIC_CACHE} {out}") + print(f"published {arch} closures to {ATTIC_CACHE}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/flake.nix b/flake.nix index eaa3c9e..1ea0479 100644 --- a/flake.nix +++ b/flake.nix @@ -320,5 +320,98 @@ gcc15-fixes = import ./overlays/gcc15-fixes.nix; hyprspace = hyprspaceOverlay; }; + + # `nix run .#` — local-parity entrypoints (emmett#44, cluster#192, + # attic-closure archetype). parity-lib has NO attic builder, so this is a + # thin wrap of ci/publish.py (the woodpecker-peek pattern), NOT a parity + # builder conversion. The app IS the shared code: .woodpecker/amd64.yaml + # runs the exact same `ci/publish.py x86_64-linux`, so CI and a local run + # cannot drift. + # + # TWO HALVES (emmett#44): STAGE `nix build`s every package in the arch's + # list into the local /nix store (cluster-independent, runs on emmett); + # PUBLISH additionally `attic push`es each closure to the cache that lives + # next to the cluster. Local runs DRY-RUN (stage + show the pushes) unless + # `--push`/PUBLISH=1. + # + # nix run .#stage-amd64 stage x86_64-linux closures, no publish + # nix run .#publish-amd64 same, then push if `--push` given + # nix run .#publish-amd64 -- --push actually push to attic + # nix run .#publish all locally-buildable arches (amd64) + # + # arm64 BLOCKER: aarch64-linux cannot be built on emmett (linux/amd64) and + # there is no cross path for these native packages, so the arm64 leg MUST + # run on an aarch64 node (.woodpecker/arm64.yaml). It is intentionally + # absent from the apps below. + apps = nixpkgs.lib.genAttrs buildSystems ( + system: + let + pkgs = import nixpkgs { inherit system; }; + mkApp = + { + name, + arch, + defaultPush ? false, + }: + let + prog = pkgs.writeShellApplication { + inherit name; + runtimeInputs = [ + pkgs.python3 + pkgs.git + pkgs.nix + ]; + text = '' + ${pkgs.lib.optionalString defaultPush "export PUBLISH=\"\${PUBLISH:-1}\""} + exec python3 ci/publish.py ${arch} "$@" + ''; + }; + in + { + type = "app"; + program = "${prog}/bin/${name}"; + meta.description = + "flake-hub ${arch} — stage closures then attic push " + + "(dry-run unless --push/PUBLISH=1)"; + }; + in + { + # x86_64-linux: native on emmett. + stage-amd64 = mkApp { + name = "stage-amd64"; + arch = "x86_64-linux"; + }; + publish-amd64 = mkApp { + name = "publish-amd64"; + arch = "x86_64-linux"; + }; + + # `publish` = every locally-buildable arch + the arm64 blocker note. + # On emmett that is amd64 only; arm64 is node-bound (see comment above). + publish = { + type = "app"; + program = + let + p = pkgs.writeShellApplication { + name = "publish"; + runtimeInputs = [ + pkgs.python3 + pkgs.git + pkgs.nix + ]; + text = '' + echo "publish: amd64 is the only emmett-buildable arch; arm64 is" + echo " node-bound (run on an aarch64 node via" + echo " .woodpecker/arm64.yaml)." + exec python3 ci/publish.py x86_64-linux "$@" + ''; + }; + in + "${p}/bin/publish"; + meta.description = + "flake-hub — publish all locally-buildable arches (amd64; arm64 node-bound)"; + }; + } + ); }; }