oleks/flake-hub
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(ci): retrofit onto parity-lib mkAtticClosurePublish (#200)
ci/woodpecker/push/amd64 Pipeline failed
Details
ci/woodpecker/push/arm64 Pipeline was successful
Details

Browse Source 
Replace the bespoke ci/publish.py attic-push logic with parity-lib's
mkAtticClosurePublish builder (attic-closure archetype, cluster#104,
emmett#44). Adds the parity input (locked at d265a79) and wires the
per-arch package closures through builders.mkAtticClosurePublish, with
the endpoint (nix-cache-upload.oleks.space) and passEntry
(infra/attic/ci_token) overridden so the attic push is byte-for-byte the
pre-parity behaviour.

.woodpecker/{amd64,arm64}.yaml thinned to PUBLISH=1 nix run .#publish /
.#publish-aarch64-linux so CI and a local run share one audited impl.
Dead ci/publish.py + ci/build.py removed.

pipeline-doctor: 9 passed / 0 failed / 0 warned.
This commit is contained in:
Oleks
2026-06-03 10:58:06 +03:00
parent e8050f9dfd
commit b797aefb28
6 changed files with 224 additions and 272 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.woodpecker/amd64.yaml
+5 -3
View File
@@ -37,6 +37,8 @@ steps:
commands:
- echo "▸ arch=$(uname -m)"
- sh ci/setup.sh
# Same entrypoint as a local `nix run .#publish-amd64 -- --push`.
# PUBLISH=1 makes the shared script actually push (local runs dry-run).
- PUBLISH=1 python3 ci/publish.py x86_64-linux
# Same front door as a local `nix run .#publish -- --publish`. The app
# is parity-lib's mkAtticClosurePublish (attic-closure archetype); CI and
# local share one audited impl. PUBLISH=1 makes it actually push (local
# runs dry-run).
- PUBLISH=1 nix run .#publish
.woodpecker/arm64.yaml
+3 -3
View File
@@ -37,6 +37,6 @@ steps:
- sh ci/setup.sh
# NODE-BOUND LEG (emmett#44, cluster#192): aarch64-linux can't be built on
# emmett (linux/amd64) and these native packages have no cross path, so this
# leg has no local-parity flake app — it must run on an aarch64 node (the
# arch nodeSelector above). Same shared entrypoint as amd64, only arch differs.
- PUBLISH=1 python3 ci/publish.py aarch64-linux
# leg must run on an aarch64 node (the arch nodeSelector above). Same
# parity-lib attic-closure front door as amd64, only the arch app differs.
- PUBLISH=1 nix run .#publish-aarch64-linux
ci/build.py
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env python3
"""Deprecated shim — superseded by ci/publish.py (emmett#44, cluster#192).
Kept so any stale reference keeps working. Forwards to publish.py with the same
arch arg and forces a real push (PUBLISH=1), matching the old always-push
behaviour. New callers should use ci/publish.py (dry-run by default) or the
`nix run .#publish-amd64` flake app.
"""
import os
import subprocess
import sys
here = os.path.dirname(os.path.abspath(__file__))
env = dict(os.environ, PUBLISH="1")
sys.exit(
subprocess.run(
[sys.executable, os.path.join(here, "publish.py"), *sys.argv[1:]], env=env
).returncode
)
ci/publish.py
-166
View File
@@ -1,166 +0,0 @@
#!/usr/bin/env python3
"""Build all flake-hub packages for one arch and (optionally) push to attic.
Single source of truth for both CI and local runs — invoked identically by the
`publish-<arch>` flake apps and by .woodpecker/amd64.yaml, so the two can't drift
(emmett#44, cluster#192, attic-closure archetype).
STAGE vs PUBLISH (emmett#44 two-halves rule):
- STAGE = `nix build` every package in this arch's list into the local /nix
store. No network publish. This half is fully cluster-independent
and is what runs on emmett.
- PUBLISH = STAGE, then `attic push` each closure to the binary cache. The
cache lives next to the cluster, so this half shares fate with it.
DRY-RUN by default: a local run only STAGES and prints the pushes it *would* do.
Pushing requires an explicit opt-in:
PUBLISH=1 (env) or --push (flag).
CI sets PUBLISH=1 so the pipeline actually publishes.
Token: the attic cache token is $ATTIC_TOKEN. We never read or print its value.
Resolution order (local convenience): $ATTIC_TOKEN, else
`pass infra/attic/ci_token` if `pass` is available. Named hard-fail otherwise.
"""
import os
import shutil
import subprocess
import sys
ATTIC_CACHE = "attic-infra-cache-k3s-1"
ATTIC_SERVER = "https://nix-cache-upload.oleks.space"
def run(cmd, env=None):
print(f"+ {cmd}", flush=True)
r = subprocess.run(cmd, shell=True, env=env)
if r.returncode != 0:
sys.exit(r.returncode)
def info(cmd):
"""Like run(), but tolerant of failure — non-load-bearing diagnostics."""
print(f"+ {cmd}", flush=True)
subprocess.run(cmd, shell=True)
def build(cmd):
"""Run a `nix build`, streaming stderr live; return stdout (the out path)."""
print(f"+ {cmd}", flush=True)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True)
out, _ = proc.communicate()
if proc.returncode != 0:
sys.exit(proc.returncode)
return out.strip()
def resolve_token():
"""Return the attic token without ever printing it. Named hard-fail."""
tok = os.environ.get("ATTIC_TOKEN")
if tok:
return tok
if shutil.which("pass"):
r = subprocess.run(
["pass", "infra/attic/ci_token"], capture_output=True, text=True
)
if r.returncode == 0 and r.stdout.strip():
return r.stdout.strip().splitlines()[0]
sys.exit(
"ERROR: no attic token. Set $ATTIC_TOKEN or store it at "
"`pass infra/attic/ci_token`. (Refusing to push without a token.)"
)
def packages_for(arch):
"""The attic-warmed package set for one arch (verbatim from the old build.py)."""
packages = ["hello-world", "geesefs", "xonsh"]
# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's
# default builds for Linux/Darwin; we cache the Linux native arches).
if arch in ("x86_64-linux", "aarch64-linux"):
packages += ["woodpecker-peek"]
# mcp-chrome: cache the proven-green wasm-simd worker only.
# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's
# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome
# issue #1 close comment); CI would just go red on it.
if arch in ("x86_64-linux", "aarch64-linux"):
packages += ["mcp-chrome-wasm-simd"]
# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which
# transitively builds liberation-fonts; fontforge segfaults while generating
# the .ttf files (pipeline #40). Package definitions stay in the flake for
# local builds — re-enable here once upstream fontforge is fixed.
# if arch == "x86_64-linux":
# packages += ["google-antigravity", "google-antigravity-no-fhs"]
if arch == "s390x-linux":
packages += ["attic-client"]
# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite
# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3
# compiles from source (~5-8 min cold) on the first push after a rev bump.
if arch in ("x86_64-linux", "aarch64-linux"):
packages += ["gitea-local-fork"]
return packages
def main():
args = sys.argv[1:]
push = os.environ.get("PUBLISH") == "1" or "--push" in args
args = [a for a in args if a != "--push"]
if "--help" in args or "-h" in args or not args:
print(__doc__)
print("usage: publish.py <arch> [--push] (e.g. publish.py x86_64-linux)")
sys.exit(0 if args and args[0] in ("--help", "-h") else (0 if args else 2))
arch = args[0]
mode = "PUBLISH" if push else "STAGE (dry-run)"
print(f"=== flake-hub :: {arch} :: {mode} ===", flush=True)
# Environment context for log readers.
info("nix --version")
info("uname -a")
info("df -h /nix 2>/dev/null || df -h /")
info("cat /proc/meminfo | head -3")
packages = packages_for(arch)
# STAGE: build every package into the local store, collecting out paths.
staged = []
print("Staging packages...", flush=True)
for pkg in packages:
print(f"--- {pkg} ---", flush=True)
out = build(
f"nix build '.#packages.{arch}.{pkg}' "
"--print-build-logs --print-out-paths --no-link"
)
staged.append((pkg, out))
if not push:
print("DRY-RUN: would push the following closures:", flush=True)
for pkg, out in staged:
print(f" attic push {ATTIC_CACHE} {out} ({pkg})", flush=True)
print("Re-run with --push (or PUBLISH=1) to actually publish.", flush=True)
return
# PUBLISH: build attic-client out of nixpkgs, log in, push every closure.
# Token-bearing steps below MUST NOT run under shell tracing — resolve_token
# returns the secret in-process and we never echo it.
token = resolve_token()
attic = (
build(
"nix build --inputs-from . nixpkgs#attic-client "
"--print-build-logs --print-out-paths --no-link"
)
+ "/bin/attic"
)
# Pass the token via argv of a child without printing it: list form so it
# never lands in our `+ ...` trace.
print(f"+ {attic} login ci {ATTIC_SERVER} <token-hidden>", flush=True)
r = subprocess.run([attic, "login", "ci", ATTIC_SERVER, token])
if r.returncode != 0:
sys.exit(r.returncode)
for pkg, out in staged:
print(f"--- push {pkg} ---", flush=True)
run(f"'{attic}' push {ATTIC_CACHE} {out}")
print(f"published {arch} closures to {ATTIC_CACHE}", flush=True)
if __name__ == "__main__":
main()
flake.lock
Generated
+137
View File
@@ -93,6 +93,87 @@
"type": "github"
}
},
"flake-utils_5": {
"inputs": {
"systems": "systems_5"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"repo": "flake-utils",
"type": "github"
}
},
"fleet": {
"inputs": {
"nixpkgs": "nixpkgs_2",
"nixpkgs-armer": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-bim": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-ci": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-emmett": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-howard": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-mermaid": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-mermaid-gpu": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-micron": [
"parity",
"fleet",
"nixpkgs"
],
"nixpkgs-projects": [
"parity",
"fleet",
"nixpkgs"
]
},
"locked": {
"lastModified": 1779533061,
"narHash": "sha256-orWNYXtYURhEj3X4+xGMAhaEcKRvwXqTtJ8x2jV/M+Q=",
"ref": "refs/heads/main",
"rev": "b818e345ec4470e4b3e335bd2f864183c512116d",
"revCount": 13,
"type": "git",
"url": "https://git.oleks.space/oleks/fleet-pins"
},
"original": {
"type": "git",
"url": "https://git.oleks.space/oleks/fleet-pins"
}
},
"fleet-pins": {
"inputs": {
"nixpkgs": "nixpkgs",
@@ -220,6 +301,46 @@
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1777268161,
"narHash": "sha256-bxrdOn8SCOv8tN4JbTF/TXq7kjo9ag4M+C8yzzIRYbE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76",
"type": "github"
},
"original": {
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "1c3fe55ad329cbcb28471bb30f05c9827f724c76",
"type": "github"
}
},
"parity": {
"inputs": {
"flake-utils": "flake-utils_5",
"fleet": "fleet",
"nixpkgs": [
"parity",
"fleet",
"nixpkgs-ci"
]
},
"locked": {
"lastModified": 1780472546,
"narHash": "sha256-cx+821qtyNZNe9t3ab8qImmeg/rxVzPpZIS45SflrI0=",
"ref": "refs/heads/main",
"rev": "d265a79ddb84b297364e6cc3638c9f6b5dc583d7",
"revCount": 10,
"type": "git",
"url": "https://git.oleks.space/oleks/parity-lib"
},
"original": {
"type": "git",
"url": "https://git.oleks.space/oleks/parity-lib"
}
},
"root": {
"inputs": {
"antigravity-nix": "antigravity-nix",
@@ -232,6 +353,7 @@
"fleet-pins",
"nixpkgs-projects"
],
"parity": "parity",
"stalewood": "stalewood",
"woodpecker-peek": "woodpecker-peek"
}
@@ -316,6 +438,21 @@
"type": "github"
}
},
"systems_5": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
},
"woodpecker-peek": {
"inputs": {
"nixpkgs": [
flake.nix
+76 -77
View File
@@ -6,6 +6,12 @@
nixpkgs.follows = "fleet-pins/nixpkgs-projects";
flake-utils.url = "github:numtide/flake-utils";
# Shared per-archetype parity publish-app builders (cluster#104, emmett#44).
# flake-hub is an ATTIC-CLOSURE repo: it builds package closures and warms
# the Attic cache with them — there is NO registry artifact.
# mkAtticClosurePublish makes that archetype explicit to pipeline-doctor.
parity.url = "git+https://git.oleks.space/oleks/parity-lib";
# Hyprspace source; no flake.nix in the repo so we consume it as raw src.
# Pin tracks the last v0.52-compatible commit of Hyprspace.
hyprspace = {
@@ -58,6 +64,7 @@
nixpkgs,
fleet-pins,
flake-utils,
parity,
hyprspace,
antigravity-nix,
nix-deps,
@@ -322,94 +329,86 @@
};
# `nix run .#<app>` — local-parity entrypoints (emmett#44, cluster#192,
# attic-closure archetype). parity-lib has NO attic builder, so this is a
# thin wrap of ci/publish.py (the woodpecker-peek pattern), NOT a parity
# builder conversion. The app IS the shared code: .woodpecker/amd64.yaml
# runs the exact same `ci/publish.py x86_64-linux`, so CI and a local run
# cannot drift.
# ATTIC-CLOSURE archetype, cluster#104). The stage/publish/push-staged
# apps come straight from parity-lib's mkAtticClosurePublish, so CI and a
# local run share one audited implementation and cannot drift. There is NO
# registry artifact — these apps build the package closures and warm the
# Attic cache with them.
#
# TWO HALVES (emmett#44): STAGE `nix build`s every package in the arch's
# list into the local /nix store (cluster-independent, runs on emmett);
# PUBLISH additionally `attic push`es each closure to the cache that lives
# next to the cluster. Local runs DRY-RUN (stage + show the pushes) unless
# `--push`/PUBLISH=1.
# `--publish`/PUBLISH=1.
#
# nix run .#stage-amd64 stage x86_64-linux closures, no publish
# nix run .#publish-amd64 same, then push if `--push` given
# nix run .#publish-amd64 -- --push actually push to attic
# nix run .#publish all locally-buildable arches (amd64)
# nix run .#stage-x86_64-linux stage amd64 closures, no publish
# nix run .#publish stage amd64 then push if PUBLISH=1
# nix run .#publish -- --publish actually push to attic
# nix run .#push-staged replay already-staged amd64 paths
#
# arm64 BLOCKER: aarch64-linux cannot be built on emmett (linux/amd64) and
# there is no cross path for these native packages, so the arm64 leg MUST
# run on an aarch64 node (.woodpecker/arm64.yaml). It is intentionally
# absent from the apps below.
apps = nixpkgs.lib.genAttrs buildSystems (
system:
# arm64 leg: aarch64-linux cannot be built on emmett (linux/amd64) and the
# native packages have no cross path, so it MUST run on an aarch64 node
# (.woodpecker/arm64.yaml runs `PUBLISH=1 nix run .#publish-aarch64-linux`).
#
# The package set per arch mirrors flake-hub's CI warm list (was
# ci/publish.py packages_for): the always-on core plus arch-conditional
# extras. Cross-only/known-broken targets stay out (see comments).
apps =
let
pkgs = import nixpkgs { inherit system; };
mkApp =
{
name,
arch,
defaultPush ? false,
}:
let
prog = pkgs.writeShellApplication {
inherit name;
runtimeInputs = [
pkgs.python3
pkgs.git
pkgs.nix
# The Attic cache token lives at `pass infra/attic/ci_token` and the
# cache is reached via the armer hairpin endpoint — preserve both so
# the push is byte-for-byte the pre-parity behaviour.
atticEndpoint = "https://nix-cache-upload.oleks.space";
atticPass = "infra/attic/ci_token";
# Package names to warm into Attic for a given native arch. Mirrors the
# old ci/publish.py `packages_for`. Native arches only (amd64/arm64);
# mcp-chrome-extension stays OUT (known-broken under nix-daemon at this
# pin, see oleks/mcp-chrome #1); antigravity stays OUT (fontforge
# segfault, pipeline #40).
packageNamesFor =
arch:
[
"hello-world"
"geesefs"
"xonsh"
]
++ nixpkgs.lib.optionals (arch == "x86_64-linux" || arch == "aarch64-linux") [
"woodpecker-peek"
"mcp-chrome-wasm-simd"
"gitea-local-fork"
];
text = ''
${pkgs.lib.optionalString defaultPush "export PUBLISH=\"\${PUBLISH:-1}\""}
exec python3 ci/publish.py ${arch} "$@"
'';
};
in
{
type = "app";
program = "${prog}/bin/${name}";
meta.description =
"flake-hub ${arch} — stage closures then attic push " + "(dry-run unless --push/PUBLISH=1)";
};
in
{
# x86_64-linux: native on emmett.
stage-amd64 = mkApp {
name = "stage-amd64";
arch = "x86_64-linux";
};
publish-amd64 = mkApp {
name = "publish-amd64";
arch = "x86_64-linux";
drvsFor = arch: map (n: self.packages.${arch}.${n}) (packageNamesFor arch);
buildersFor = arch: parity.lib.mkParityBuilders (import nixpkgs { system = arch; });
atticAppsFor =
arch:
(buildersFor arch).mkAtticClosurePublish {
drvs = drvsFor arch;
inherit arch;
endpoint = atticEndpoint;
passEntry = atticPass;
};
# `publish` = every locally-buildable arch + the arm64 blocker note.
# On emmett that is amd64 only; arm64 is node-bound (see comment above).
publish = {
type = "app";
program =
let
p = pkgs.writeShellApplication {
name = "publish";
runtimeInputs = [
pkgs.python3
pkgs.git
pkgs.nix
];
text = ''
echo "publish: amd64 is the only emmett-buildable arch; arm64 is"
echo " node-bound (run on an aarch64 node via"
echo " .woodpecker/arm64.yaml)."
exec python3 ci/publish.py x86_64-linux "$@"
'';
};
amd64Apps = atticAppsFor "x86_64-linux";
arm64Apps = atticAppsFor "aarch64-linux";
in
"${p}/bin/publish";
meta.description = "flake-hub — publish all locally-buildable arches (amd64; arm64 node-bound)";
};
}
);
nixpkgs.lib.genAttrs buildSystems (system:
# amd64 is the emmett-buildable arch: expose its stage/publish/
# push-staged plus the top-level `publish`. Also surface the arm64
# stage/publish under their arch-suffixed names for the node-bound
# CI leg (.woodpecker/arm64.yaml).
{
inherit (amd64Apps)
"stage-x86_64-linux"
"publish-x86_64-linux"
"publish"
"push-staged"
;
"stage-aarch64-linux" = arm64Apps."stage-aarch64-linux";
"publish-aarch64-linux" = arm64Apps."publish-aarch64-linux";
});
};
}