From b797aefb284d2159f3703bc0fe262070bdc4230e Mon Sep 17 00:00:00 2001 From: Oleks Date: Wed, 3 Jun 2026 10:58:06 +0300 Subject: [PATCH] refactor(ci): retrofit onto parity-lib mkAtticClosurePublish (#200) Replace the bespoke ci/publish.py attic-push logic with parity-lib's mkAtticClosurePublish builder (attic-closure archetype, cluster#104, emmett#44). Adds the parity input (locked at d265a79) and wires the per-arch package closures through builders.mkAtticClosurePublish, with the endpoint (nix-cache-upload.oleks.space) and passEntry (infra/attic/ci_token) overridden so the attic push is byte-for-byte the pre-parity behaviour. .woodpecker/{amd64,arm64}.yaml thinned to PUBLISH=1 nix run .#publish / .#publish-aarch64-linux so CI and a local run share one audited impl. Dead ci/publish.py + ci/build.py removed. pipeline-doctor: 9 passed / 0 failed / 0 warned. --- .woodpecker/amd64.yaml | 8 +- .woodpecker/arm64.yaml | 6 +- ci/build.py | 20 ----- ci/publish.py | 166 ----------------------------------------- flake.lock | 137 ++++++++++++++++++++++++++++++++++ flake.nix | 159 ++++++++++++++++++++------------------- 6 files changed, 224 insertions(+), 272 deletions(-) delete mode 100644 ci/build.py delete mode 100644 ci/publish.py diff --git a/.woodpecker/amd64.yaml b/.woodpecker/amd64.yaml index 1c6063e..14a619b 100644 --- a/.woodpecker/amd64.yaml +++ b/.woodpecker/amd64.yaml @@ -37,6 +37,8 @@ steps: commands: - echo "▸ arch=$(uname -m)" - sh ci/setup.sh - # Same entrypoint as a local `nix run .#publish-amd64 -- --push`. - # PUBLISH=1 makes the shared script actually push (local runs dry-run). - - PUBLISH=1 python3 ci/publish.py x86_64-linux + # Same front door as a local `nix run .#publish -- --publish`. The app + # is parity-lib's mkAtticClosurePublish (attic-closure archetype); CI and + # local share one audited impl. PUBLISH=1 makes it actually push (local + # runs dry-run). + - PUBLISH=1 nix run .#publish diff --git a/.woodpecker/arm64.yaml b/.woodpecker/arm64.yaml index 4c26195..8c1cb21 100644 --- a/.woodpecker/arm64.yaml +++ b/.woodpecker/arm64.yaml @@ -37,6 +37,6 @@ steps: - sh ci/setup.sh # NODE-BOUND LEG (emmett#44, cluster#192): aarch64-linux can't be built on # emmett (linux/amd64) and these native packages have no cross path, so this - # leg has no local-parity flake app — it must run on an aarch64 node (the - # arch nodeSelector above). Same shared entrypoint as amd64, only arch differs. - - PUBLISH=1 python3 ci/publish.py aarch64-linux + # leg must run on an aarch64 node (the arch nodeSelector above). Same + # parity-lib attic-closure front door as amd64, only the arch app differs. + - PUBLISH=1 nix run .#publish-aarch64-linux diff --git a/ci/build.py b/ci/build.py deleted file mode 100644 index b49b351..0000000 --- a/ci/build.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 -"""Deprecated shim — superseded by ci/publish.py (emmett#44, cluster#192). - -Kept so any stale reference keeps working. Forwards to publish.py with the same -arch arg and forces a real push (PUBLISH=1), matching the old always-push -behaviour. New callers should use ci/publish.py (dry-run by default) or the -`nix run .#publish-amd64` flake app. -""" - -import os -import subprocess -import sys - -here = os.path.dirname(os.path.abspath(__file__)) -env = dict(os.environ, PUBLISH="1") -sys.exit( - subprocess.run( - [sys.executable, os.path.join(here, "publish.py"), *sys.argv[1:]], env=env - ).returncode -) diff --git a/ci/publish.py b/ci/publish.py deleted file mode 100644 index 474dcf4..0000000 --- a/ci/publish.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""Build all flake-hub packages for one arch and (optionally) push to attic. - -Single source of truth for both CI and local runs — invoked identically by the -`publish-` flake apps and by .woodpecker/amd64.yaml, so the two can't drift -(emmett#44, cluster#192, attic-closure archetype). - -STAGE vs PUBLISH (emmett#44 two-halves rule): - - STAGE = `nix build` every package in this arch's list into the local /nix - store. No network publish. This half is fully cluster-independent - and is what runs on emmett. - - PUBLISH = STAGE, then `attic push` each closure to the binary cache. The - cache lives next to the cluster, so this half shares fate with it. - -DRY-RUN by default: a local run only STAGES and prints the pushes it *would* do. -Pushing requires an explicit opt-in: - PUBLISH=1 (env) or --push (flag). -CI sets PUBLISH=1 so the pipeline actually publishes. - -Token: the attic cache token is $ATTIC_TOKEN. We never read or print its value. -Resolution order (local convenience): $ATTIC_TOKEN, else -`pass infra/attic/ci_token` if `pass` is available. Named hard-fail otherwise. -""" - -import os -import shutil -import subprocess -import sys - -ATTIC_CACHE = "attic-infra-cache-k3s-1" -ATTIC_SERVER = "https://nix-cache-upload.oleks.space" - - -def run(cmd, env=None): - print(f"+ {cmd}", flush=True) - r = subprocess.run(cmd, shell=True, env=env) - if r.returncode != 0: - sys.exit(r.returncode) - - -def info(cmd): - """Like run(), but tolerant of failure — non-load-bearing diagnostics.""" - print(f"+ {cmd}", flush=True) - subprocess.run(cmd, shell=True) - - -def build(cmd): - """Run a `nix build`, streaming stderr live; return stdout (the out path).""" - print(f"+ {cmd}", flush=True) - proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True) - out, _ = proc.communicate() - if proc.returncode != 0: - sys.exit(proc.returncode) - return out.strip() - - -def resolve_token(): - """Return the attic token without ever printing it. Named hard-fail.""" - tok = os.environ.get("ATTIC_TOKEN") - if tok: - return tok - if shutil.which("pass"): - r = subprocess.run( - ["pass", "infra/attic/ci_token"], capture_output=True, text=True - ) - if r.returncode == 0 and r.stdout.strip(): - return r.stdout.strip().splitlines()[0] - sys.exit( - "ERROR: no attic token. Set $ATTIC_TOKEN or store it at " - "`pass infra/attic/ci_token`. (Refusing to push without a token.)" - ) - - -def packages_for(arch): - """The attic-warmed package set for one arch (verbatim from the old build.py).""" - packages = ["hello-world", "geesefs", "xonsh"] - # woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's - # default builds for Linux/Darwin; we cache the Linux native arches). - if arch in ("x86_64-linux", "aarch64-linux"): - packages += ["woodpecker-peek"] - # mcp-chrome: cache the proven-green wasm-simd worker only. - # mcp-chrome-extension is exposed in the flake but NOT built in CI — it's - # KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome - # issue #1 close comment); CI would just go red on it. - if arch in ("x86_64-linux", "aarch64-linux"): - packages += ["mcp-chrome-wasm-simd"] - # google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which - # transitively builds liberation-fonts; fontforge segfaults while generating - # the .ttf files (pipeline #40). Package definitions stay in the flake for - # local builds — re-enable here once upstream fontforge is fixed. - # if arch == "x86_64-linux": - # packages += ["google-antigravity", "google-antigravity-no-fhs"] - if arch == "s390x-linux": - packages += ["attic-client"] - # gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite - # and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3 - # compiles from source (~5-8 min cold) on the first push after a rev bump. - if arch in ("x86_64-linux", "aarch64-linux"): - packages += ["gitea-local-fork"] - return packages - - -def main(): - args = sys.argv[1:] - push = os.environ.get("PUBLISH") == "1" or "--push" in args - args = [a for a in args if a != "--push"] - if "--help" in args or "-h" in args or not args: - print(__doc__) - print("usage: publish.py [--push] (e.g. publish.py x86_64-linux)") - sys.exit(0 if args and args[0] in ("--help", "-h") else (0 if args else 2)) - arch = args[0] - - mode = "PUBLISH" if push else "STAGE (dry-run)" - print(f"=== flake-hub :: {arch} :: {mode} ===", flush=True) - - # Environment context for log readers. - info("nix --version") - info("uname -a") - info("df -h /nix 2>/dev/null || df -h /") - info("cat /proc/meminfo | head -3") - - packages = packages_for(arch) - - # STAGE: build every package into the local store, collecting out paths. - staged = [] - print("Staging packages...", flush=True) - for pkg in packages: - print(f"--- {pkg} ---", flush=True) - out = build( - f"nix build '.#packages.{arch}.{pkg}' " - "--print-build-logs --print-out-paths --no-link" - ) - staged.append((pkg, out)) - - if not push: - print("DRY-RUN: would push the following closures:", flush=True) - for pkg, out in staged: - print(f" attic push {ATTIC_CACHE} {out} ({pkg})", flush=True) - print("Re-run with --push (or PUBLISH=1) to actually publish.", flush=True) - return - - # PUBLISH: build attic-client out of nixpkgs, log in, push every closure. - # Token-bearing steps below MUST NOT run under shell tracing — resolve_token - # returns the secret in-process and we never echo it. - token = resolve_token() - attic = ( - build( - "nix build --inputs-from . nixpkgs#attic-client " - "--print-build-logs --print-out-paths --no-link" - ) - + "/bin/attic" - ) - # Pass the token via argv of a child without printing it: list form so it - # never lands in our `+ ...` trace. - print(f"+ {attic} login ci {ATTIC_SERVER} ", flush=True) - r = subprocess.run([attic, "login", "ci", ATTIC_SERVER, token]) - if r.returncode != 0: - sys.exit(r.returncode) - for pkg, out in staged: - print(f"--- push {pkg} ---", flush=True) - run(f"'{attic}' push {ATTIC_CACHE} {out}") - print(f"published {arch} closures to {ATTIC_CACHE}", flush=True) - - -if __name__ == "__main__": - main() diff --git a/flake.lock b/flake.lock index 128e403..f6c0f86 100644 --- a/flake.lock +++ b/flake.lock @@ -93,6 +93,87 @@ "type": "github" } }, + "flake-utils_5": { + "inputs": { + "systems": "systems_5" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "fleet": { + "inputs": { + "nixpkgs": "nixpkgs_2", + "nixpkgs-armer": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-bim": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-ci": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-emmett": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-howard": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-mermaid": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-mermaid-gpu": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-micron": [ + "parity", + "fleet", + "nixpkgs" + ], + "nixpkgs-projects": [ + "parity", + "fleet", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1779533061, + "narHash": "sha256-orWNYXtYURhEj3X4+xGMAhaEcKRvwXqTtJ8x2jV/M+Q=", + "ref": "refs/heads/main", + "rev": "b818e345ec4470e4b3e335bd2f864183c512116d", + "revCount": 13, + "type": "git", + "url": "https://git.oleks.space/oleks/fleet-pins" + }, + "original": { + "type": "git", + "url": "https://git.oleks.space/oleks/fleet-pins" + } + }, "fleet-pins": { "inputs": { "nixpkgs": "nixpkgs", @@ -220,6 +301,46 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1777268161, + "narHash": "sha256-bxrdOn8SCOv8tN4JbTF/TXq7kjo9ag4M+C8yzzIRYbE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76", + "type": "github" + } + }, + "parity": { + "inputs": { + "flake-utils": "flake-utils_5", + "fleet": "fleet", + "nixpkgs": [ + "parity", + "fleet", + "nixpkgs-ci" + ] + }, + "locked": { + "lastModified": 1780472546, + "narHash": "sha256-cx+821qtyNZNe9t3ab8qImmeg/rxVzPpZIS45SflrI0=", + "ref": "refs/heads/main", + "rev": "d265a79ddb84b297364e6cc3638c9f6b5dc583d7", + "revCount": 10, + "type": "git", + "url": "https://git.oleks.space/oleks/parity-lib" + }, + "original": { + "type": "git", + "url": "https://git.oleks.space/oleks/parity-lib" + } + }, "root": { "inputs": { "antigravity-nix": "antigravity-nix", @@ -232,6 +353,7 @@ "fleet-pins", "nixpkgs-projects" ], + "parity": "parity", "stalewood": "stalewood", "woodpecker-peek": "woodpecker-peek" } @@ -316,6 +438,21 @@ "type": "github" } }, + "systems_5": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, "woodpecker-peek": { "inputs": { "nixpkgs": [ diff --git a/flake.nix b/flake.nix index 8c62c16..366de64 100644 --- a/flake.nix +++ b/flake.nix @@ -6,6 +6,12 @@ nixpkgs.follows = "fleet-pins/nixpkgs-projects"; flake-utils.url = "github:numtide/flake-utils"; + # Shared per-archetype parity publish-app builders (cluster#104, emmett#44). + # flake-hub is an ATTIC-CLOSURE repo: it builds package closures and warms + # the Attic cache with them — there is NO registry artifact. + # mkAtticClosurePublish makes that archetype explicit to pipeline-doctor. + parity.url = "git+https://git.oleks.space/oleks/parity-lib"; + # Hyprspace source; no flake.nix in the repo so we consume it as raw src. # Pin tracks the last v0.52-compatible commit of Hyprspace. hyprspace = { @@ -58,6 +64,7 @@ nixpkgs, fleet-pins, flake-utils, + parity, hyprspace, antigravity-nix, nix-deps, @@ -322,94 +329,86 @@ }; # `nix run .#` — local-parity entrypoints (emmett#44, cluster#192, - # attic-closure archetype). parity-lib has NO attic builder, so this is a - # thin wrap of ci/publish.py (the woodpecker-peek pattern), NOT a parity - # builder conversion. The app IS the shared code: .woodpecker/amd64.yaml - # runs the exact same `ci/publish.py x86_64-linux`, so CI and a local run - # cannot drift. + # ATTIC-CLOSURE archetype, cluster#104). The stage/publish/push-staged + # apps come straight from parity-lib's mkAtticClosurePublish, so CI and a + # local run share one audited implementation and cannot drift. There is NO + # registry artifact — these apps build the package closures and warm the + # Attic cache with them. # # TWO HALVES (emmett#44): STAGE `nix build`s every package in the arch's # list into the local /nix store (cluster-independent, runs on emmett); # PUBLISH additionally `attic push`es each closure to the cache that lives # next to the cluster. Local runs DRY-RUN (stage + show the pushes) unless - # `--push`/PUBLISH=1. + # `--publish`/PUBLISH=1. # - # nix run .#stage-amd64 stage x86_64-linux closures, no publish - # nix run .#publish-amd64 same, then push if `--push` given - # nix run .#publish-amd64 -- --push actually push to attic - # nix run .#publish all locally-buildable arches (amd64) + # nix run .#stage-x86_64-linux stage amd64 closures, no publish + # nix run .#publish stage amd64 then push if PUBLISH=1 + # nix run .#publish -- --publish actually push to attic + # nix run .#push-staged replay already-staged amd64 paths # - # arm64 BLOCKER: aarch64-linux cannot be built on emmett (linux/amd64) and - # there is no cross path for these native packages, so the arm64 leg MUST - # run on an aarch64 node (.woodpecker/arm64.yaml). It is intentionally - # absent from the apps below. - apps = nixpkgs.lib.genAttrs buildSystems ( - system: + # arm64 leg: aarch64-linux cannot be built on emmett (linux/amd64) and the + # native packages have no cross path, so it MUST run on an aarch64 node + # (.woodpecker/arm64.yaml runs `PUBLISH=1 nix run .#publish-aarch64-linux`). + # + # The package set per arch mirrors flake-hub's CI warm list (was + # ci/publish.py packages_for): the always-on core plus arch-conditional + # extras. Cross-only/known-broken targets stay out (see comments). + apps = let - pkgs = import nixpkgs { inherit system; }; - mkApp = - { - name, - arch, - defaultPush ? false, - }: - let - prog = pkgs.writeShellApplication { - inherit name; - runtimeInputs = [ - pkgs.python3 - pkgs.git - pkgs.nix - ]; - text = '' - ${pkgs.lib.optionalString defaultPush "export PUBLISH=\"\${PUBLISH:-1}\""} - exec python3 ci/publish.py ${arch} "$@" - ''; - }; - in - { - type = "app"; - program = "${prog}/bin/${name}"; - meta.description = - "flake-hub ${arch} — stage closures then attic push " + "(dry-run unless --push/PUBLISH=1)"; - }; - in - { - # x86_64-linux: native on emmett. - stage-amd64 = mkApp { - name = "stage-amd64"; - arch = "x86_64-linux"; - }; - publish-amd64 = mkApp { - name = "publish-amd64"; - arch = "x86_64-linux"; - }; + # The Attic cache token lives at `pass infra/attic/ci_token` and the + # cache is reached via the armer hairpin endpoint — preserve both so + # the push is byte-for-byte the pre-parity behaviour. + atticEndpoint = "https://nix-cache-upload.oleks.space"; + atticPass = "infra/attic/ci_token"; - # `publish` = every locally-buildable arch + the arm64 blocker note. - # On emmett that is amd64 only; arm64 is node-bound (see comment above). - publish = { - type = "app"; - program = - let - p = pkgs.writeShellApplication { - name = "publish"; - runtimeInputs = [ - pkgs.python3 - pkgs.git - pkgs.nix - ]; - text = '' - echo "publish: amd64 is the only emmett-buildable arch; arm64 is" - echo " node-bound (run on an aarch64 node via" - echo " .woodpecker/arm64.yaml)." - exec python3 ci/publish.py x86_64-linux "$@" - ''; - }; - in - "${p}/bin/publish"; - meta.description = "flake-hub — publish all locally-buildable arches (amd64; arm64 node-bound)"; - }; - } - ); + # Package names to warm into Attic for a given native arch. Mirrors the + # old ci/publish.py `packages_for`. Native arches only (amd64/arm64); + # mcp-chrome-extension stays OUT (known-broken under nix-daemon at this + # pin, see oleks/mcp-chrome #1); antigravity stays OUT (fontforge + # segfault, pipeline #40). + packageNamesFor = + arch: + [ + "hello-world" + "geesefs" + "xonsh" + ] + ++ nixpkgs.lib.optionals (arch == "x86_64-linux" || arch == "aarch64-linux") [ + "woodpecker-peek" + "mcp-chrome-wasm-simd" + "gitea-local-fork" + ]; + + drvsFor = arch: map (n: self.packages.${arch}.${n}) (packageNamesFor arch); + + buildersFor = arch: parity.lib.mkParityBuilders (import nixpkgs { system = arch; }); + + atticAppsFor = + arch: + (buildersFor arch).mkAtticClosurePublish { + drvs = drvsFor arch; + inherit arch; + endpoint = atticEndpoint; + passEntry = atticPass; + }; + + amd64Apps = atticAppsFor "x86_64-linux"; + arm64Apps = atticAppsFor "aarch64-linux"; + in + nixpkgs.lib.genAttrs buildSystems (system: + # amd64 is the emmett-buildable arch: expose its stage/publish/ + # push-staged plus the top-level `publish`. Also surface the arm64 + # stage/publish under their arch-suffixed names for the node-bound + # CI leg (.woodpecker/arm64.yaml). + { + inherit (amd64Apps) + "stage-x86_64-linux" + "publish-x86_64-linux" + "publish" + "push-staged" + ; + "stage-aarch64-linux" = arm64Apps."stage-aarch64-linux"; + "publish-aarch64-linux" = arm64Apps."publish-aarch64-linux"; + }); }; }