From db0bf3b9ab9326dde6e9c7fc5432cd2d5d1e96e5 Mon Sep 17 00:00:00 2001 From: Oleks Date: Tue, 2 Jun 2026 23:30:59 +0300 Subject: [PATCH] feat(parity): mkAtticClosurePublish builder + pipeline-doctor non-flake mode (#198, #193) Adds the attic-closure archetype builder (build closure + attic push, no registry artifact) so caddy/overlay-xonsh/flake-hub/woodpecker-peek share one implementation. Adds non-flake mode to pipeline-doctor so ci-script repos (gitea-mcp, helms) pass the gate. Self-check 9/9; gitea-mcp now passes. --- CHANGELOG.md | 15 ++++ ci/pipeline-doctor.sh | 76 ++++++++++++++++---- flake.nix | 1 + lib/builders.nix | 157 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb44c3d..d7cb28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ semantic versioning; the version is a conceptual tag (no git tag is created). ## Unreleased +- **Feature: `mkAtticClosurePublish` — the attic-closure builder (cluster #198).** + Models the archetype parity-lib was missing: build a Nix closure and push it to + the Attic binary cache (NO registry artifact). Yields `stage-` (`nix build` + the closure, no push), `publish-`/`publish` (build + `attic login` + `attic + push`; dry-run by default, `--publish`/`PUBLISH=1` to push; token via `$ATTIC_TOKEN` + or `pass`, never echoed), and `push-staged`. Lets caddy-with-replace (#104) drop its + generic-publish over-reach and overlay-xonsh (#105) convert off N/A; flake-hub / + woodpecker-peek can retire their bespoke attic wraps. +- **`pipeline-doctor` non-flake mode (cluster #191/#193).** A repo with NO root + `flake.nix` is now a VALID parity form if it ships the ci-script entrypoints + (`ci/local.sh`, or `ci/build.sh` + `ci/publish.sh`) called by a thin + `.woodpecker.yaml` — so the non-flake go-binary/helm references (gitea-mcp, helms) + PASS the gate instead of failing for lacking a flake. The token-leak scan and the + #191 no-`set -x`-in-token-scripts scan still run on their `ci/*.sh` in full. + Verified: gitea-mcp now 9/9, parity-lib + numpy-s390x still 9/9. - **Feature: `verify-digest` for nix2container (cluster #195).** `mkNix2ContainerPublish` now also returns a `verify-digest` app that builds each locally-buildable arch image and prints its OCI **manifest digest** with NO registry contact (it diff --git a/ci/pipeline-doctor.sh b/ci/pipeline-doctor.sh index bcfa73c..7d359dc 100755 --- a/ci/pipeline-doctor.sh +++ b/ci/pipeline-doctor.sh @@ -37,6 +37,18 @@ done REPO="$(cd "$REPO" && pwd)" FLAKE="$REPO/flake.nix" +# Non-flake parity form (cluster #193, issue #193). A repo with no root flake.nix +# is still a VALID parity repo if it ships the ci-script form: a ci/local.sh (or a +# ci/build.sh + ci/publish.sh pair) that is the single local==CI entrypoint, with +# a thin .woodpecker.yaml that calls those scripts. In that case the "consumes +# parity / single entrypoint" contract is satisfied via the CI-SCRIPT path instead +# of a parity-lib flake input, and the token-leak + #191 no-set-x scans below still +# apply to its ci/*.sh. +CI_SCRIPT_FORM=0 +if [ -f "$REPO/ci/local.sh" ] || { [ -f "$REPO/ci/build.sh" ] && [ -f "$REPO/ci/publish.sh" ]; }; then + CI_SCRIPT_FORM=1 +fi + fail=0 warn_n=0 pass_n=0 @@ -57,12 +69,17 @@ warn() { echo "pipeline-doctor: $REPO" if [ ! -f "$FLAKE" ]; then - echo "FAIL: no flake.nix at $REPO" >&2 - exit 1 + if [ "$CI_SCRIPT_FORM" -eq 1 ]; then + # Valid non-flake parity repo: assert via the ci-script form below. + flake_txt="" + else + echo "FAIL: no flake.nix and no ci/local.sh (or ci/build.sh + ci/publish.sh) at $REPO" >&2 + exit 1 + fi +else + flake_txt="$(cat "$FLAKE")" fi -flake_txt="$(cat "$FLAKE")" - # Documented escape-hatch (cluster #196): a repo that must keep a hand-written # Dockerfile/OCI pipeline (BuildKit, etc.) instead of a parity-lib builder is # allowed to opt out PROVIDED it ships a `ci/local.sh` that gives a dev the same @@ -74,27 +91,46 @@ if [ -f "$REPO/ci/local.sh" ]; then ESCAPE_HATCH=1 fi +# Text of the ci/*.sh + .woodpecker.yaml — the archetype marker and the single +# local==CI entrypoint of a non-flake (ci-script form) repo live here, not in a +# flake.nix (cluster #193, issue #193). +ci_txt="" +if [ -d "$REPO/ci" ]; then + ci_txt="$(cat "$REPO"/ci/*.sh 2>/dev/null || true)" +fi +for wp in "$REPO/.woodpecker.yaml" "$REPO/.woodpecker.yml"; do + [ -f "$wp" ] && ci_txt="$ci_txt +$(cat "$wp")" +done + # 1. archetype declared (a publish/stage app whose name encodes the arch, or an -# explicit archetype marker comment). +# explicit archetype marker comment) — in the flake OR, for a non-flake +# ci-script repo, in its ci/*.sh / .woodpecker.yaml. if printf '%s' "$flake_txt" | grep -Eq 'archetype|stage-[a-z0-9_]+|publish-[a-z0-9_]+'; then ok "archetype declared (stage-*/publish-* app or archetype marker present)" +elif [ "$CI_SCRIPT_FORM" -eq 1 ] && printf '%s' "$ci_txt" | grep -Eq 'archetype|ci/build\.sh|ci/publish\.sh|ci/local\.sh'; then + ok "archetype declared (non-flake ci-script form: ci/*.sh entrypoints, cluster #193)" elif [ "$ESCAPE_HATCH" -eq 1 ]; then warn "no parity-lib archetype, but ci/local.sh escape-hatch present (cluster #196)" else - bad "no archetype: expected a stage-/publish- app or 'archetype' marker (or a ci/local.sh escape-hatch)" + bad "no archetype: expected a stage-/publish- app or 'archetype' marker (or a ci-script form)" fi -# 2. consumes parity-lib (the shared module) OR a documented ci/local.sh hatch. +# 2. consumes parity-lib (the shared module) OR is a valid non-flake ci-script +# repo whose ci/local.sh (or ci/build.sh + ci/publish.sh) is the single +# local==CI entrypoint, OR a documented ci/local.sh escape-hatch. CONSUMES_PARITY=0 -if printf '%s' "$flake_txt" | grep -Eq 'parity-lib|parity\.lib|mk(PyPiWheel|S390xNpm|GenericBinary|Nix2Container|GoBinary|Helm)Publish'; then +if printf '%s' "$flake_txt" | grep -Eq 'parity-lib|parity\.lib|mk(PyPiWheel|S390xNpm|GenericBinary|Nix2Container|GoBinary|Helm|AtticClosure)Publish'; then CONSUMES_PARITY=1 fi if [ "$CONSUMES_PARITY" -eq 1 ]; then ok "consumes parity-lib (input or one of the mk*Publish builders)" +elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then + ok "single local==CI entrypoint via ci-script form (ci/local.sh or ci/build.sh + ci/publish.sh, cluster #193)" elif [ "$ESCAPE_HATCH" -eq 1 ]; then warn "does not consume parity-lib, but ci/local.sh escape-hatch present (cluster #196)" else - bad "does not consume parity-lib: inline the input and use a mk*Publish builder (or ship a ci/local.sh escape-hatch)" + bad "does not consume parity-lib: inline the input and use a mk*Publish builder (or ship a ci-script form)" fi # Checks 3–6 below are GUARANTEED by parity-lib for a consumer: the token @@ -167,11 +203,17 @@ else bad "set -x in token-bearing ci/*.sh:$xtrace_hits (xtrace would echo the token)" fi -# 4. dev-tag guard present. +# 4. dev-tag guard present. For a non-flake ci-script repo the equivalent guard +# is the dry-run-DEFAULT publish + the .woodpecker.yaml `refs/tags/v*` gate: +# nothing mutates the registry unless a v* tag run flips PUBLISH/DRY_RUN. if [ "$CONSUMES_PARITY" -eq 1 ]; then ok "dev-tag guard present (delegated to parity-lib parity_devtag_guard)" elif printf '%s' "$token_src" | grep -Eq 'parity_devtag_guard|refusing to publish without an explicit'; then ok "dev-tag guard present" +elif [ "$CI_SCRIPT_FORM" -eq 1 ] && + printf '%s' "$ci_txt" | grep -Eq 'DRY_RUN|dry-run|PUBLISH' && + printf '%s' "$ci_txt" | grep -Eq 'refs/tags/v\*|event:[[:space:]]*tag|tag:'; then + ok "dev-tag guard equivalent (ci-script dry-run default + .woodpecker refs/tags/v* gate, cluster #193)" else bad "no dev-tag guard: a default-version --publish could clobber the registry" fi @@ -185,20 +227,30 @@ else bad "no dry-run default: publish must NOT mutate the registry by default" fi -# 6. apps carry meta.description. +# 6. apps carry meta.description. N/A for a non-flake ci-script repo (it exposes +# shell-script entrypoints, not flake apps with meta); its self-documenting +# ci/*.sh header comments are the equivalent. if [ "$CONSUMES_PARITY" -eq 1 ]; then ok "apps carry meta.description (delegated to parity-lib mkApp)" elif printf '%s' "$flake_txt" | grep -Eq 'meta\.description|meta = \{'; then ok "apps carry meta.description" +elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then + ok "meta.description N/A (non-flake ci-script form; ci/*.sh are the documented entrypoints)" else bad "apps missing meta.description" fi -# 7. enumerable publish-* naming. +# 7. enumerable publish-* naming. For a non-flake ci-script repo the equivalent +# is the ci/build.sh + ci/publish.sh (or ci/local.sh) entrypoint set. pubs="$(printf '%s' "$flake_txt" | grep -oE '(stage|publish)(-[a-z0-9_]+)?' | sort -u || true)" if [ -n "$pubs" ]; then ok "enumerable publish-* / stage-* app naming:" printf '%s\n' "$pubs" | sed 's/^/ /' +elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then + ok "enumerable ci-script entrypoints (non-flake parity form, cluster #193):" + for s in ci/local.sh ci/build.sh ci/publish.sh; do + [ -f "$REPO/$s" ] && printf ' %s\n' "$s" + done else bad "no enumerable publish-*/stage-* apps found" fi diff --git a/flake.nix b/flake.nix index 591c446..2b8c122 100644 --- a/flake.nix +++ b/flake.nix @@ -39,6 +39,7 @@ mkNix2ContainerPublish = wrap "mkNix2ContainerPublish"; mkGoBinaryPublish = wrap "mkGoBinaryPublish"; mkHelmPublish = wrap "mkHelmPublish"; + mkAtticClosurePublish = wrap "mkAtticClosurePublish"; }; } // flake-utils.lib.eachDefaultSystem ( diff --git a/lib/builders.nix b/lib/builders.nix index ecc1446..8e649d6 100644 --- a/lib/builders.nix +++ b/lib/builders.nix @@ -971,6 +971,162 @@ let in mkApp app "Replay the staged ${pname} ${arch} artifact (${kind}) from .parity-stage (dry-run by default; --publish to push)."; + # ========================================================================= + # Attic closure (NO registry artifact). The ATTIC-CLOSURE archetype: the + # "artifact" is a Nix closure pushed to an Attic binary cache, not a registry + # blob/image. Modelled on the bespoke attic CI the caddy-with-replace / + # woodpecker-peek / flake-hub repos run (`attic login ci $ATTIC_TOKEN` + # then push/watch-store the built store paths). + # + # Apps: + # stage- BUILD-parity: `nix build` the drvs into the local store and + # record their out-paths in .parity-stage (NO push, no token). + # publish- / publish stage + `attic login` + `attic push` the built + # paths to the cache. DRY-RUN by default; --publish / PUBLISH=1 + # actually pushes. Token from $ATTIC_TOKEN or `pass`. + # push-staged replay the staged out-paths to the cache (no rebuild). + # + # TOKEN HYGIENE: $ATTIC_TOKEN is never echoed and no `set -x` is enabled; the + # token is piped straight into `attic login` and any output is sed-redacted. + # + # Args: { drvs (list of derivations/attrs to build), + # cache ? "attic-infra-cache-k3s-1", + # endpoint ? "http://attic.infra-cache.k3s", + # arch ? "x86_64-linux", + # passEntry ? "infra/attic/token", ... } + # ========================================================================= + mkAtticClosurePublish = + { + drvs, + cache ? "attic-infra-cache-k3s-1", + endpoint ? "http://attic.infra-cache.k3s", + arch ? "x86_64-linux", + passEntry ? "infra/attic/token", + }: + let + # Closure parity does not contact the Gitea registry; only the Attic cache + # endpoint matters, but reuse the shared preamble for the stage-dir + arg + # conventions. passEntry here points at the ATTIC token, not the registry. + head = preamble { inherit passEntry; }; + drvList = if builtins.isList drvs then drvs else [ drvs ]; + drvRefs = lib.concatMapStringsSep " " (d: lib.escapeShellArg "${d}") drvList; + atticInputs = baseInputs ++ [ + pkgs.nix + pkgs.attic-client + ]; + + # Resolve the Attic token: $ATTIC_TOKEN, else `pass `. Never + # echoed; prints on stdout for a caller to capture (the one blessed emit). + atticTokenFn = '' + parity_attic_token() { + if [ -n "''${ATTIC_TOKEN:-}" ]; then printf '%s' "$ATTIC_TOKEN"; return 0; fi + if command -v pass >/dev/null 2>&1; then + t="$(pass show ${lib.escapeShellArg passEntry} 2>/dev/null || true)" + if [ -n "$t" ]; then printf '%s' "$t"; return 0; fi + fi + echo "BLOCKER(empty-token): set \$ATTIC_TOKEN or store ${passEntry} in pass; refusing to push without credentials." >&2 + return 1 + } + ''; + + # Log into the cache + push the given out-paths. Token piped, never echoed; + # any attic output is sed-redacted so a token can never reach the log. + atticPushFn = '' + parity_attic_push() { + tok="$(parity_attic_token)" || return 1 + set +e + out="$(attic login ci ${lib.escapeShellArg endpoint} "$tok" 2>&1)" + rc=$? + set -e + printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g" + if [ "$rc" -ne 0 ]; then return "$rc"; fi + attic push ${lib.escapeShellArg cache} "$@" + } + ''; + + stage = pkgs.writeShellApplication { + name = "stage-${arch}"; + runtimeInputs = atticInputs; + text = '' + ${head} + echo "→ staging attic closure (${arch}, ${toString (builtins.length drvList)} drv(s), no push)" + d="$(parity_stage_reset ${lib.escapeShellArg arch})" + : >"$d/out-paths" + # The drv refs are a fixed Nix-eval list (may be a single literal path). + # shellcheck disable=SC2043 + for drv in ${drvRefs}; do + # Realise the path into the local store (build-parity, no push). + p="$(nix build --no-link --print-out-paths "$drv")" + printf '%s\n' "$p" >>"$d/out-paths" + echo " built: $p" + done + printf 'attic %s %s\n' ${lib.escapeShellArg cache} ${lib.escapeShellArg endpoint} >"$d/.parity-meta" + echo " staged out-paths under $d" + ''; + }; + + publish = pkgs.writeShellApplication { + name = "publish-${arch}"; + runtimeInputs = atticInputs; + text = '' + ${head} + ${atticTokenFn} + ${atticPushFn} + parity_parse_args "Build + push the ${arch} Nix closure to the Attic cache ${cache}" "$@" + ${lib.getExe stage} + d="$(parity_stage_path ${lib.escapeShellArg arch})" + mapfile -t paths <"$d/out-paths" + if [ "''${#paths[@]}" -eq 0 ]; then + echo "BLOCKER(no-stage): no built out-paths under $d — run 'nix run .#stage-${arch}' first." >&2 + exit 1 + fi + if [ "$PARITY_PUBLISH" != "1" ]; then + echo "[dry-run] would 'attic push ${cache}' these paths to ${endpoint}:" + for p in "''${paths[@]}"; do echo " - $p"; done + echo "[dry-run] re-run with --publish / PUBLISH=1 to push." + exit 0 + fi + parity_attic_push "''${paths[@]}" + echo "Pushed ''${#paths[@]} path(s) to ${cache}" + ''; + }; + + pushStaged = pkgs.writeShellApplication { + name = "push-staged"; + runtimeInputs = atticInputs; + text = '' + ${head} + ${atticTokenFn} + ${atticPushFn} + parity_parse_args "Replay the staged ${arch} closure out-paths to the Attic cache ${cache}" "$@" + d="$(parity_stage_path ${lib.escapeShellArg arch})" + if [ ! -f "$d/out-paths" ]; then + echo "BLOCKER(no-stage): no staged out-paths at $d — run 'nix run .#stage-${arch}' first." >&2 + exit 1 + fi + mapfile -t paths <"$d/out-paths" + if [ "$PARITY_PUBLISH" != "1" ]; then + echo "[dry-run] would 'attic push ${cache}' these staged paths to ${endpoint}:" + for p in "''${paths[@]}"; do echo " - $p"; done + echo "[dry-run] re-run with --publish / PUBLISH=1 to push." + exit 0 + fi + parity_attic_push "''${paths[@]}" + echo "Replayed ''${#paths[@]} path(s) to ${cache}" + ''; + }; + in + { + "stage-${arch}" = + mkApp stage "Build the ${arch} Nix closure into the local store + stage its out-paths (no push)."; + "publish-${arch}" = + mkApp publish "Build + push the ${arch} closure to the Attic cache ${cache} (dry-run by default; --publish to push)."; + "publish" = + mkApp publish "Push all locally-buildable arches to the Attic cache (${arch} only for this closure repo)."; + "push-staged" = + mkApp pushStaged "Replay the staged ${arch} closure out-paths to the Attic cache (dry-run by default; --publish to push)."; + }; + mkPushStagedHelm = { pname, @@ -1019,6 +1175,7 @@ in mkNix2ContainerPublish mkGoBinaryPublish mkHelmPublish + mkAtticClosurePublish ; # Shared building blocks, exposed for advanced/bespoke consumers. inherit mkApp shellLib;