ci: local-pipeline parity via stage/publish flake apps (cluster#192, emmett#44)
attic-closure archetype: no parity-lib builder exists for attic pushes, so
wrap the existing per-arch package build in ci/publish.py (woodpecker-peek
pattern) and expose `nix run .#{stage,publish}-amd64` + `.#publish`.
Two-halves rule: STAGE nix-builds every package in the arch list into the
local store (emmett-buildable); PUBLISH additionally attic-pushes each
closure. Local runs DRY-RUN unless --push/PUBLISH=1; CI sets PUBLISH=1.
The .woodpecker/{amd64,arm64}.yaml now call the same ci/publish.py so CI
and local runs can't drift. arm64 stays node-bound (no emmett cross path),
so it has no local-parity app. ci/build.py becomes a forwarding shim.
This commit is contained in:
Oleks
@@ -36,4 +36,6 @@ steps:
|
||||
pipeline-number: "${CI_PIPELINE_NUMBER}"
|
||||
commands:
|
||||
- sh ci/setup.sh
|
||||
- python3 ci/build.py x86_64-linux
|
||||
# Same entrypoint as a local `nix run .#publish-amd64 -- --push`.
|
||||
# PUBLISH=1 makes the shared script actually push (local runs dry-run).
|
||||
- PUBLISH=1 python3 ci/publish.py x86_64-linux
|
||||
|
||||
@@ -34,4 +34,8 @@ steps:
|
||||
pipeline-number: "${CI_PIPELINE_NUMBER}"
|
||||
commands:
|
||||
- sh ci/setup.sh
|
||||
- python3 ci/build.py aarch64-linux
|
||||
# NODE-BOUND LEG (emmett#44, cluster#192): aarch64-linux can't be built on
|
||||
# emmett (linux/amd64) and these native packages have no cross path, so this
|
||||
# leg has no local-parity flake app — it must run on an aarch64 node (the
|
||||
# arch nodeSelector above). Same shared entrypoint as amd64, only arch differs.
|
||||
- PUBLISH=1 python3 ci/publish.py aarch64-linux
|
||||
|
||||
+13
-81
@@ -1,88 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build all flake-hub packages and push to attic."""
|
||||
"""Deprecated shim — superseded by ci/publish.py (emmett#44, cluster#192).
|
||||
|
||||
Kept so any stale reference keeps working. Forwards to publish.py with the same
|
||||
arch arg and forces a real push (PUBLISH=1), matching the old always-push
|
||||
behaviour. New callers should use ci/publish.py (dry-run by default) or the
|
||||
`nix run .#publish-amd64` flake app.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
def run(cmd):
|
||||
print(f"+ {cmd}", flush=True)
|
||||
r = subprocess.run(cmd, shell=True)
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
|
||||
|
||||
def info(cmd):
|
||||
"""Like run(), but tolerant of failure — for non-load-bearing diagnostics."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
subprocess.run(cmd, shell=True)
|
||||
|
||||
|
||||
def build(cmd):
|
||||
"""Run a `nix build`, streaming stderr live; return stdout (the out path)."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True)
|
||||
out, _ = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
sys.exit(proc.returncode)
|
||||
return out.strip()
|
||||
|
||||
|
||||
ARCH = sys.argv[1]
|
||||
ATTIC_CACHE = "attic-infra-cache-k3s-1"
|
||||
ATTIC_SERVER = "https://nix-cache-upload.oleks.space"
|
||||
ATTIC_TOKEN = os.environ["ATTIC_TOKEN"]
|
||||
|
||||
print(f"=== Building flake-hub packages for {ARCH} ===")
|
||||
|
||||
# Environment context for log readers
|
||||
info("nix --version")
|
||||
info("uname -a")
|
||||
info("df -h /nix 2>/dev/null || df -h /")
|
||||
info("cat /proc/meminfo | head -3")
|
||||
|
||||
# Setup attic
|
||||
attic = (
|
||||
build(
|
||||
"nix build --inputs-from . nixpkgs#attic-client --print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
+ "/bin/attic"
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
env = dict(os.environ, PUBLISH="1")
|
||||
sys.exit(
|
||||
subprocess.run(
|
||||
[sys.executable, os.path.join(here, "publish.py"), *sys.argv[1:]], env=env
|
||||
).returncode
|
||||
)
|
||||
run(f"'{attic}' login ci {ATTIC_SERVER} '{ATTIC_TOKEN}'")
|
||||
|
||||
# Packages per arch
|
||||
packages = ["hello-world", "geesefs", "xonsh"]
|
||||
# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's
|
||||
# default builds for Linux/Darwin; we cache the Linux native arches).
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["woodpecker-peek"]
|
||||
# mcp-chrome: cache the proven-green wasm-simd worker only.
|
||||
# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's
|
||||
# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome
|
||||
# issue #1 close comment); CI would just go red on it.
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["mcp-chrome-wasm-simd"]
|
||||
# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which
|
||||
# transitively builds liberation-fonts; fontforge segfaults while generating
|
||||
# the .ttf files (pipeline #40). Package definitions stay in the flake for
|
||||
# local builds — re-enable here once upstream fontforge is fixed.
|
||||
# if ARCH == "x86_64-linux":
|
||||
# packages += ["google-antigravity", "google-antigravity-no-fhs"]
|
||||
if ARCH == "s390x-linux":
|
||||
packages += ["attic-client"]
|
||||
# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite
|
||||
# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3
|
||||
# compiles from source (~5-8 min cold) on the first push after a rev bump.
|
||||
if ARCH in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["gitea-local-fork"]
|
||||
|
||||
print("Building packages...")
|
||||
for pkg in packages:
|
||||
print(f"--- {pkg} ---")
|
||||
out = build(
|
||||
f"nix build '.#packages.{ARCH}.{pkg}' --print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
run(f"'{attic}' push {ATTIC_CACHE} {out}")
|
||||
|
||||
print(f"Build completed for {ARCH}")
|
||||
|
||||
+168
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build all flake-hub packages for one arch and (optionally) push to attic.
|
||||
|
||||
Single source of truth for both CI and local runs — invoked identically by the
|
||||
`publish-<arch>` flake apps and by .woodpecker/amd64.yaml, so the two can't drift
|
||||
(emmett#44, cluster#192, attic-closure archetype).
|
||||
|
||||
STAGE vs PUBLISH (emmett#44 two-halves rule):
|
||||
- STAGE = `nix build` every package in this arch's list into the local /nix
|
||||
store. No network publish. This half is fully cluster-independent
|
||||
and is what runs on emmett.
|
||||
- PUBLISH = STAGE, then `attic push` each closure to the binary cache. The
|
||||
cache lives next to the cluster, so this half shares fate with it.
|
||||
|
||||
DRY-RUN by default: a local run only STAGES and prints the pushes it *would* do.
|
||||
Pushing requires an explicit opt-in:
|
||||
PUBLISH=1 (env) or --push (flag).
|
||||
CI sets PUBLISH=1 so the pipeline actually publishes.
|
||||
|
||||
Token: the attic cache token is $ATTIC_TOKEN. We never read or print its value.
|
||||
Resolution order (local convenience): $ATTIC_TOKEN, else
|
||||
`pass infra/attic/ci_token` if `pass` is available. Named hard-fail otherwise.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ATTIC_CACHE = "attic-infra-cache-k3s-1"
|
||||
ATTIC_SERVER = "https://nix-cache-upload.oleks.space"
|
||||
|
||||
|
||||
def run(cmd, env=None):
|
||||
print(f"+ {cmd}", flush=True)
|
||||
r = subprocess.run(cmd, shell=True, env=env)
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
|
||||
|
||||
def info(cmd):
|
||||
"""Like run(), but tolerant of failure — non-load-bearing diagnostics."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
subprocess.run(cmd, shell=True)
|
||||
|
||||
|
||||
def build(cmd):
|
||||
"""Run a `nix build`, streaming stderr live; return stdout (the out path)."""
|
||||
print(f"+ {cmd}", flush=True)
|
||||
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, text=True)
|
||||
out, _ = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
sys.exit(proc.returncode)
|
||||
return out.strip()
|
||||
|
||||
|
||||
def resolve_token():
|
||||
"""Return the attic token without ever printing it. Named hard-fail."""
|
||||
tok = os.environ.get("ATTIC_TOKEN")
|
||||
if tok:
|
||||
return tok
|
||||
if shutil.which("pass"):
|
||||
r = subprocess.run(
|
||||
["pass", "infra/attic/ci_token"], capture_output=True, text=True
|
||||
)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout.strip().splitlines()[0]
|
||||
sys.exit(
|
||||
"ERROR: no attic token. Set $ATTIC_TOKEN or store it at "
|
||||
"`pass infra/attic/ci_token`. (Refusing to push without a token.)"
|
||||
)
|
||||
|
||||
|
||||
def packages_for(arch):
|
||||
"""The attic-warmed package set for one arch (verbatim from the old build.py)."""
|
||||
packages = ["hello-world", "geesefs", "xonsh"]
|
||||
# woodpecker-peek: tray app, x86_64 + aarch64 only (the upstream flake's
|
||||
# default builds for Linux/Darwin; we cache the Linux native arches).
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["woodpecker-peek"]
|
||||
# mcp-chrome: cache the proven-green wasm-simd worker only.
|
||||
# mcp-chrome-extension is exposed in the flake but NOT built in CI — it's
|
||||
# KNOWN-BROKEN under nix-daemon at the current pin (see oleks/mcp-chrome
|
||||
# issue #1 close comment); CI would just go red on it.
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["mcp-chrome-wasm-simd"]
|
||||
# google-antigravity{,-no-fhs} skipped in CI: pulls in google-chrome, which
|
||||
# transitively builds liberation-fonts; fontforge segfaults while generating
|
||||
# the .ttf files (pipeline #40). Package definitions stay in the flake for
|
||||
# local builds — re-enable here once upstream fontforge is fixed.
|
||||
# if arch == "x86_64-linux":
|
||||
# packages += ["google-antigravity", "google-antigravity-no-fhs"]
|
||||
if arch == "s390x-linux":
|
||||
packages += ["attic-client"]
|
||||
# gitea-local-fork: only defined for x86_64-linux and aarch64-linux (cgo+sqlite
|
||||
# and pnpm don't cross-compile cleanly — see flake.nix). Slow build: Go 1.26.3
|
||||
# compiles from source (~5-8 min cold) on the first push after a rev bump.
|
||||
if arch in ("x86_64-linux", "aarch64-linux"):
|
||||
packages += ["gitea-local-fork"]
|
||||
return packages
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
push = os.environ.get("PUBLISH") == "1" or "--push" in args
|
||||
args = [a for a in args if a != "--push"]
|
||||
if "--help" in args or "-h" in args or not args:
|
||||
print(__doc__)
|
||||
print("usage: publish.py <arch> [--push] (e.g. publish.py x86_64-linux)")
|
||||
sys.exit(0 if args and args[0] in ("--help", "-h") else (0 if args else 2))
|
||||
arch = args[0]
|
||||
|
||||
mode = "PUBLISH" if push else "STAGE (dry-run)"
|
||||
print(f"=== flake-hub :: {arch} :: {mode} ===", flush=True)
|
||||
|
||||
# Environment context for log readers.
|
||||
info("nix --version")
|
||||
info("uname -a")
|
||||
info("df -h /nix 2>/dev/null || df -h /")
|
||||
info("cat /proc/meminfo | head -3")
|
||||
|
||||
packages = packages_for(arch)
|
||||
|
||||
# STAGE: build every package into the local store, collecting out paths.
|
||||
staged = []
|
||||
print("Staging packages...", flush=True)
|
||||
for pkg in packages:
|
||||
print(f"--- {pkg} ---", flush=True)
|
||||
out = build(
|
||||
f"nix build '.#packages.{arch}.{pkg}' "
|
||||
"--print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
staged.append((pkg, out))
|
||||
|
||||
if not push:
|
||||
print("DRY-RUN: would push the following closures:", flush=True)
|
||||
for pkg, out in staged:
|
||||
print(f" attic push {ATTIC_CACHE} {out} ({pkg})", flush=True)
|
||||
print(
|
||||
"Re-run with --push (or PUBLISH=1) to actually publish.", flush=True
|
||||
)
|
||||
return
|
||||
|
||||
# PUBLISH: build attic-client out of nixpkgs, log in, push every closure.
|
||||
# Token-bearing steps below MUST NOT run under shell tracing — resolve_token
|
||||
# returns the secret in-process and we never echo it.
|
||||
token = resolve_token()
|
||||
attic = (
|
||||
build(
|
||||
"nix build --inputs-from . nixpkgs#attic-client "
|
||||
"--print-build-logs --print-out-paths --no-link"
|
||||
)
|
||||
+ "/bin/attic"
|
||||
)
|
||||
# Pass the token via argv of a child without printing it: list form so it
|
||||
# never lands in our `+ ...` trace.
|
||||
print(f"+ {attic} login ci {ATTIC_SERVER} <token-hidden>", flush=True)
|
||||
r = subprocess.run([attic, "login", "ci", ATTIC_SERVER, token])
|
||||
if r.returncode != 0:
|
||||
sys.exit(r.returncode)
|
||||
for pkg, out in staged:
|
||||
print(f"--- push {pkg} ---", flush=True)
|
||||
run(f"'{attic}' push {ATTIC_CACHE} {out}")
|
||||
print(f"published {arch} closures to {ATTIC_CACHE}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -320,5 +320,98 @@
|
||||
gcc15-fixes = import ./overlays/gcc15-fixes.nix;
|
||||
hyprspace = hyprspaceOverlay;
|
||||
};
|
||||
|
||||
# `nix run .#<app>` — local-parity entrypoints (emmett#44, cluster#192,
|
||||
# attic-closure archetype). parity-lib has NO attic builder, so this is a
|
||||
# thin wrap of ci/publish.py (the woodpecker-peek pattern), NOT a parity
|
||||
# builder conversion. The app IS the shared code: .woodpecker/amd64.yaml
|
||||
# runs the exact same `ci/publish.py x86_64-linux`, so CI and a local run
|
||||
# cannot drift.
|
||||
#
|
||||
# TWO HALVES (emmett#44): STAGE `nix build`s every package in the arch's
|
||||
# list into the local /nix store (cluster-independent, runs on emmett);
|
||||
# PUBLISH additionally `attic push`es each closure to the cache that lives
|
||||
# next to the cluster. Local runs DRY-RUN (stage + show the pushes) unless
|
||||
# `--push`/PUBLISH=1.
|
||||
#
|
||||
# nix run .#stage-amd64 stage x86_64-linux closures, no publish
|
||||
# nix run .#publish-amd64 same, then push if `--push` given
|
||||
# nix run .#publish-amd64 -- --push actually push to attic
|
||||
# nix run .#publish all locally-buildable arches (amd64)
|
||||
#
|
||||
# arm64 BLOCKER: aarch64-linux cannot be built on emmett (linux/amd64) and
|
||||
# there is no cross path for these native packages, so the arm64 leg MUST
|
||||
# run on an aarch64 node (.woodpecker/arm64.yaml). It is intentionally
|
||||
# absent from the apps below.
|
||||
apps = nixpkgs.lib.genAttrs buildSystems (
|
||||
system:
|
||||
let
|
||||
pkgs = import nixpkgs { inherit system; };
|
||||
mkApp =
|
||||
{
|
||||
name,
|
||||
arch,
|
||||
defaultPush ? false,
|
||||
}:
|
||||
let
|
||||
prog = pkgs.writeShellApplication {
|
||||
inherit name;
|
||||
runtimeInputs = [
|
||||
pkgs.python3
|
||||
pkgs.git
|
||||
pkgs.nix
|
||||
];
|
||||
text = ''
|
||||
${pkgs.lib.optionalString defaultPush "export PUBLISH=\"\${PUBLISH:-1}\""}
|
||||
exec python3 ci/publish.py ${arch} "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
type = "app";
|
||||
program = "${prog}/bin/${name}";
|
||||
meta.description =
|
||||
"flake-hub ${arch} — stage closures then attic push "
|
||||
+ "(dry-run unless --push/PUBLISH=1)";
|
||||
};
|
||||
in
|
||||
{
|
||||
# x86_64-linux: native on emmett.
|
||||
stage-amd64 = mkApp {
|
||||
name = "stage-amd64";
|
||||
arch = "x86_64-linux";
|
||||
};
|
||||
publish-amd64 = mkApp {
|
||||
name = "publish-amd64";
|
||||
arch = "x86_64-linux";
|
||||
};
|
||||
|
||||
# `publish` = every locally-buildable arch + the arm64 blocker note.
|
||||
# On emmett that is amd64 only; arm64 is node-bound (see comment above).
|
||||
publish = {
|
||||
type = "app";
|
||||
program =
|
||||
let
|
||||
p = pkgs.writeShellApplication {
|
||||
name = "publish";
|
||||
runtimeInputs = [
|
||||
pkgs.python3
|
||||
pkgs.git
|
||||
pkgs.nix
|
||||
];
|
||||
text = ''
|
||||
echo "publish: amd64 is the only emmett-buildable arch; arm64 is"
|
||||
echo " node-bound (run on an aarch64 node via"
|
||||
echo " .woodpecker/arm64.yaml)."
|
||||
exec python3 ci/publish.py x86_64-linux "$@"
|
||||
'';
|
||||
};
|
||||
in
|
||||
"${p}/bin/publish";
|
||||
meta.description =
|
||||
"flake-hub — publish all locally-buildable arches (amd64; arm64 node-bound)";
|
||||
};
|
||||
}
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user