Adds the attic-closure archetype builder (build closure + attic push, no registry artifact) so caddy/overlay-xonsh/flake-hub/woodpecker-peek share one implementation. Adds non-flake mode to pipeline-doctor so ci-script repos (gitea-mcp, helms) pass the gate. Self-check 9/9; gitea-mcp now passes.
This commit is contained in:
Oleks
@@ -7,6 +7,21 @@ semantic versioning; the version is a conceptual tag (no git tag is created).
|
||||
|
||||
## Unreleased
|
||||
|
||||
- **Feature: `mkAtticClosurePublish` — the attic-closure builder (cluster #198).**
|
||||
Models the archetype parity-lib was missing: build a Nix closure and push it to
|
||||
the Attic binary cache (NO registry artifact). Yields `stage-<arch>` (`nix build`
|
||||
the closure, no push), `publish-<arch>`/`publish` (build + `attic login` + `attic
|
||||
push`; dry-run by default, `--publish`/`PUBLISH=1` to push; token via `$ATTIC_TOKEN`
|
||||
or `pass`, never echoed), and `push-staged`. Lets caddy-with-replace (#104) drop its
|
||||
generic-publish over-reach and overlay-xonsh (#105) convert off N/A; flake-hub /
|
||||
woodpecker-peek can retire their bespoke attic wraps.
|
||||
- **`pipeline-doctor` non-flake mode (cluster #191/#193).** A repo with NO root
|
||||
`flake.nix` is now a VALID parity form if it ships the ci-script entrypoints
|
||||
(`ci/local.sh`, or `ci/build.sh` + `ci/publish.sh`) called by a thin
|
||||
`.woodpecker.yaml` — so the non-flake go-binary/helm references (gitea-mcp, helms)
|
||||
PASS the gate instead of failing for lacking a flake. The token-leak scan and the
|
||||
#191 no-`set -x`-in-token-scripts scan still run on their `ci/*.sh` in full.
|
||||
Verified: gitea-mcp now 9/9, parity-lib + numpy-s390x still 9/9.
|
||||
- **Feature: `verify-digest` for nix2container (cluster #195).** `mkNix2ContainerPublish`
|
||||
now also returns a `verify-digest` app that builds each locally-buildable arch
|
||||
image and prints its OCI **manifest digest** with NO registry contact (it
|
||||
|
||||
+64
-12
@@ -37,6 +37,18 @@ done
|
||||
REPO="$(cd "$REPO" && pwd)"
|
||||
FLAKE="$REPO/flake.nix"
|
||||
|
||||
# Non-flake parity form (cluster #193, issue #193). A repo with no root flake.nix
|
||||
# is still a VALID parity repo if it ships the ci-script form: a ci/local.sh (or a
|
||||
# ci/build.sh + ci/publish.sh pair) that is the single local==CI entrypoint, with
|
||||
# a thin .woodpecker.yaml that calls those scripts. In that case the "consumes
|
||||
# parity / single entrypoint" contract is satisfied via the CI-SCRIPT path instead
|
||||
# of a parity-lib flake input, and the token-leak + #191 no-set-x scans below still
|
||||
# apply to its ci/*.sh.
|
||||
CI_SCRIPT_FORM=0
|
||||
if [ -f "$REPO/ci/local.sh" ] || { [ -f "$REPO/ci/build.sh" ] && [ -f "$REPO/ci/publish.sh" ]; }; then
|
||||
CI_SCRIPT_FORM=1
|
||||
fi
|
||||
|
||||
fail=0
|
||||
warn_n=0
|
||||
pass_n=0
|
||||
@@ -57,12 +69,17 @@ warn() {
|
||||
echo "pipeline-doctor: $REPO"
|
||||
|
||||
if [ ! -f "$FLAKE" ]; then
|
||||
echo "FAIL: no flake.nix at $REPO" >&2
|
||||
exit 1
|
||||
if [ "$CI_SCRIPT_FORM" -eq 1 ]; then
|
||||
# Valid non-flake parity repo: assert via the ci-script form below.
|
||||
flake_txt=""
|
||||
else
|
||||
echo "FAIL: no flake.nix and no ci/local.sh (or ci/build.sh + ci/publish.sh) at $REPO" >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
flake_txt="$(cat "$FLAKE")"
|
||||
fi
|
||||
|
||||
flake_txt="$(cat "$FLAKE")"
|
||||
|
||||
# Documented escape-hatch (cluster #196): a repo that must keep a hand-written
|
||||
# Dockerfile/OCI pipeline (BuildKit, etc.) instead of a parity-lib builder is
|
||||
# allowed to opt out PROVIDED it ships a `ci/local.sh` that gives a dev the same
|
||||
@@ -74,27 +91,46 @@ if [ -f "$REPO/ci/local.sh" ]; then
|
||||
ESCAPE_HATCH=1
|
||||
fi
|
||||
|
||||
# Text of the ci/*.sh + .woodpecker.yaml — the archetype marker and the single
|
||||
# local==CI entrypoint of a non-flake (ci-script form) repo live here, not in a
|
||||
# flake.nix (cluster #193, issue #193).
|
||||
ci_txt=""
|
||||
if [ -d "$REPO/ci" ]; then
|
||||
ci_txt="$(cat "$REPO"/ci/*.sh 2>/dev/null || true)"
|
||||
fi
|
||||
for wp in "$REPO/.woodpecker.yaml" "$REPO/.woodpecker.yml"; do
|
||||
[ -f "$wp" ] && ci_txt="$ci_txt
|
||||
$(cat "$wp")"
|
||||
done
|
||||
|
||||
# 1. archetype declared (a publish/stage app whose name encodes the arch, or an
|
||||
# explicit archetype marker comment).
|
||||
# explicit archetype marker comment) — in the flake OR, for a non-flake
|
||||
# ci-script repo, in its ci/*.sh / .woodpecker.yaml.
|
||||
if printf '%s' "$flake_txt" | grep -Eq 'archetype|stage-[a-z0-9_]+|publish-[a-z0-9_]+'; then
|
||||
ok "archetype declared (stage-*/publish-* app or archetype marker present)"
|
||||
elif [ "$CI_SCRIPT_FORM" -eq 1 ] && printf '%s' "$ci_txt" | grep -Eq 'archetype|ci/build\.sh|ci/publish\.sh|ci/local\.sh'; then
|
||||
ok "archetype declared (non-flake ci-script form: ci/*.sh entrypoints, cluster #193)"
|
||||
elif [ "$ESCAPE_HATCH" -eq 1 ]; then
|
||||
warn "no parity-lib archetype, but ci/local.sh escape-hatch present (cluster #196)"
|
||||
else
|
||||
bad "no archetype: expected a stage-<arch>/publish-<arch> app or 'archetype' marker (or a ci/local.sh escape-hatch)"
|
||||
bad "no archetype: expected a stage-<arch>/publish-<arch> app or 'archetype' marker (or a ci-script form)"
|
||||
fi
|
||||
|
||||
# 2. consumes parity-lib (the shared module) OR a documented ci/local.sh hatch.
|
||||
# 2. consumes parity-lib (the shared module) OR is a valid non-flake ci-script
|
||||
# repo whose ci/local.sh (or ci/build.sh + ci/publish.sh) is the single
|
||||
# local==CI entrypoint, OR a documented ci/local.sh escape-hatch.
|
||||
CONSUMES_PARITY=0
|
||||
if printf '%s' "$flake_txt" | grep -Eq 'parity-lib|parity\.lib|mk(PyPiWheel|S390xNpm|GenericBinary|Nix2Container|GoBinary|Helm)Publish'; then
|
||||
if printf '%s' "$flake_txt" | grep -Eq 'parity-lib|parity\.lib|mk(PyPiWheel|S390xNpm|GenericBinary|Nix2Container|GoBinary|Helm|AtticClosure)Publish'; then
|
||||
CONSUMES_PARITY=1
|
||||
fi
|
||||
if [ "$CONSUMES_PARITY" -eq 1 ]; then
|
||||
ok "consumes parity-lib (input or one of the mk*Publish builders)"
|
||||
elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then
|
||||
ok "single local==CI entrypoint via ci-script form (ci/local.sh or ci/build.sh + ci/publish.sh, cluster #193)"
|
||||
elif [ "$ESCAPE_HATCH" -eq 1 ]; then
|
||||
warn "does not consume parity-lib, but ci/local.sh escape-hatch present (cluster #196)"
|
||||
else
|
||||
bad "does not consume parity-lib: inline the input and use a mk*Publish builder (or ship a ci/local.sh escape-hatch)"
|
||||
bad "does not consume parity-lib: inline the input and use a mk*Publish builder (or ship a ci-script form)"
|
||||
fi
|
||||
|
||||
# Checks 3–6 below are GUARANTEED by parity-lib for a consumer: the token
|
||||
@@ -167,11 +203,17 @@ else
|
||||
bad "set -x in token-bearing ci/*.sh:$xtrace_hits (xtrace would echo the token)"
|
||||
fi
|
||||
|
||||
# 4. dev-tag guard present.
|
||||
# 4. dev-tag guard present. For a non-flake ci-script repo the equivalent guard
|
||||
# is the dry-run-DEFAULT publish + the .woodpecker.yaml `refs/tags/v*` gate:
|
||||
# nothing mutates the registry unless a v* tag run flips PUBLISH/DRY_RUN.
|
||||
if [ "$CONSUMES_PARITY" -eq 1 ]; then
|
||||
ok "dev-tag guard present (delegated to parity-lib parity_devtag_guard)"
|
||||
elif printf '%s' "$token_src" | grep -Eq 'parity_devtag_guard|refusing to publish without an explicit'; then
|
||||
ok "dev-tag guard present"
|
||||
elif [ "$CI_SCRIPT_FORM" -eq 1 ] &&
|
||||
printf '%s' "$ci_txt" | grep -Eq 'DRY_RUN|dry-run|PUBLISH' &&
|
||||
printf '%s' "$ci_txt" | grep -Eq 'refs/tags/v\*|event:[[:space:]]*tag|tag:'; then
|
||||
ok "dev-tag guard equivalent (ci-script dry-run default + .woodpecker refs/tags/v* gate, cluster #193)"
|
||||
else
|
||||
bad "no dev-tag guard: a default-version --publish could clobber the registry"
|
||||
fi
|
||||
@@ -185,20 +227,30 @@ else
|
||||
bad "no dry-run default: publish must NOT mutate the registry by default"
|
||||
fi
|
||||
|
||||
# 6. apps carry meta.description.
|
||||
# 6. apps carry meta.description. N/A for a non-flake ci-script repo (it exposes
|
||||
# shell-script entrypoints, not flake apps with meta); its self-documenting
|
||||
# ci/*.sh header comments are the equivalent.
|
||||
if [ "$CONSUMES_PARITY" -eq 1 ]; then
|
||||
ok "apps carry meta.description (delegated to parity-lib mkApp)"
|
||||
elif printf '%s' "$flake_txt" | grep -Eq 'meta\.description|meta = \{'; then
|
||||
ok "apps carry meta.description"
|
||||
elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then
|
||||
ok "meta.description N/A (non-flake ci-script form; ci/*.sh are the documented entrypoints)"
|
||||
else
|
||||
bad "apps missing meta.description"
|
||||
fi
|
||||
|
||||
# 7. enumerable publish-* naming.
|
||||
# 7. enumerable publish-* naming. For a non-flake ci-script repo the equivalent
|
||||
# is the ci/build.sh + ci/publish.sh (or ci/local.sh) entrypoint set.
|
||||
pubs="$(printf '%s' "$flake_txt" | grep -oE '(stage|publish)(-[a-z0-9_]+)?' | sort -u || true)"
|
||||
if [ -n "$pubs" ]; then
|
||||
ok "enumerable publish-* / stage-* app naming:"
|
||||
printf '%s\n' "$pubs" | sed 's/^/ /'
|
||||
elif [ "$CI_SCRIPT_FORM" -eq 1 ]; then
|
||||
ok "enumerable ci-script entrypoints (non-flake parity form, cluster #193):"
|
||||
for s in ci/local.sh ci/build.sh ci/publish.sh; do
|
||||
[ -f "$REPO/$s" ] && printf ' %s\n' "$s"
|
||||
done
|
||||
else
|
||||
bad "no enumerable publish-*/stage-* apps found"
|
||||
fi
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
mkNix2ContainerPublish = wrap "mkNix2ContainerPublish";
|
||||
mkGoBinaryPublish = wrap "mkGoBinaryPublish";
|
||||
mkHelmPublish = wrap "mkHelmPublish";
|
||||
mkAtticClosurePublish = wrap "mkAtticClosurePublish";
|
||||
};
|
||||
}
|
||||
// flake-utils.lib.eachDefaultSystem (
|
||||
|
||||
@@ -971,6 +971,162 @@ let
|
||||
in
|
||||
mkApp app "Replay the staged ${pname} ${arch} artifact (${kind}) from .parity-stage (dry-run by default; --publish to push).";
|
||||
|
||||
# =========================================================================
|
||||
# Attic closure (NO registry artifact). The ATTIC-CLOSURE archetype: the
|
||||
# "artifact" is a Nix closure pushed to an Attic binary cache, not a registry
|
||||
# blob/image. Modelled on the bespoke attic CI the caddy-with-replace /
|
||||
# woodpecker-peek / flake-hub repos run (`attic login ci <endpoint> $ATTIC_TOKEN`
|
||||
# then push/watch-store the built store paths).
|
||||
#
|
||||
# Apps:
|
||||
# stage-<arch> BUILD-parity: `nix build` the drvs into the local store and
|
||||
# record their out-paths in .parity-stage (NO push, no token).
|
||||
# publish-<arch> / publish stage + `attic login` + `attic push` the built
|
||||
# paths to the cache. DRY-RUN by default; --publish / PUBLISH=1
|
||||
# actually pushes. Token from $ATTIC_TOKEN or `pass`.
|
||||
# push-staged replay the staged out-paths to the cache (no rebuild).
|
||||
#
|
||||
# TOKEN HYGIENE: $ATTIC_TOKEN is never echoed and no `set -x` is enabled; the
|
||||
# token is piped straight into `attic login` and any output is sed-redacted.
|
||||
#
|
||||
# Args: { drvs (list of derivations/attrs to build),
|
||||
# cache ? "attic-infra-cache-k3s-1",
|
||||
# endpoint ? "http://attic.infra-cache.k3s",
|
||||
# arch ? "x86_64-linux",
|
||||
# passEntry ? "infra/attic/token", ... }
|
||||
# =========================================================================
|
||||
mkAtticClosurePublish =
|
||||
{
|
||||
drvs,
|
||||
cache ? "attic-infra-cache-k3s-1",
|
||||
endpoint ? "http://attic.infra-cache.k3s",
|
||||
arch ? "x86_64-linux",
|
||||
passEntry ? "infra/attic/token",
|
||||
}:
|
||||
let
|
||||
# Closure parity does not contact the Gitea registry; only the Attic cache
|
||||
# endpoint matters, but reuse the shared preamble for the stage-dir + arg
|
||||
# conventions. passEntry here points at the ATTIC token, not the registry.
|
||||
head = preamble { inherit passEntry; };
|
||||
drvList = if builtins.isList drvs then drvs else [ drvs ];
|
||||
drvRefs = lib.concatMapStringsSep " " (d: lib.escapeShellArg "${d}") drvList;
|
||||
atticInputs = baseInputs ++ [
|
||||
pkgs.nix
|
||||
pkgs.attic-client
|
||||
];
|
||||
|
||||
# Resolve the Attic token: $ATTIC_TOKEN, else `pass <passEntry>`. Never
|
||||
# echoed; prints on stdout for a caller to capture (the one blessed emit).
|
||||
atticTokenFn = ''
|
||||
parity_attic_token() {
|
||||
if [ -n "''${ATTIC_TOKEN:-}" ]; then printf '%s' "$ATTIC_TOKEN"; return 0; fi
|
||||
if command -v pass >/dev/null 2>&1; then
|
||||
t="$(pass show ${lib.escapeShellArg passEntry} 2>/dev/null || true)"
|
||||
if [ -n "$t" ]; then printf '%s' "$t"; return 0; fi
|
||||
fi
|
||||
echo "BLOCKER(empty-token): set \$ATTIC_TOKEN or store ${passEntry} in pass; refusing to push without credentials." >&2
|
||||
return 1
|
||||
}
|
||||
'';
|
||||
|
||||
# Log into the cache + push the given out-paths. Token piped, never echoed;
|
||||
# any attic output is sed-redacted so a token can never reach the log.
|
||||
atticPushFn = ''
|
||||
parity_attic_push() {
|
||||
tok="$(parity_attic_token)" || return 1
|
||||
set +e
|
||||
out="$(attic login ci ${lib.escapeShellArg endpoint} "$tok" 2>&1)"
|
||||
rc=$?
|
||||
set -e
|
||||
printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g"
|
||||
if [ "$rc" -ne 0 ]; then return "$rc"; fi
|
||||
attic push ${lib.escapeShellArg cache} "$@"
|
||||
}
|
||||
'';
|
||||
|
||||
stage = pkgs.writeShellApplication {
|
||||
name = "stage-${arch}";
|
||||
runtimeInputs = atticInputs;
|
||||
text = ''
|
||||
${head}
|
||||
echo "→ staging attic closure (${arch}, ${toString (builtins.length drvList)} drv(s), no push)"
|
||||
d="$(parity_stage_reset ${lib.escapeShellArg arch})"
|
||||
: >"$d/out-paths"
|
||||
# The drv refs are a fixed Nix-eval list (may be a single literal path).
|
||||
# shellcheck disable=SC2043
|
||||
for drv in ${drvRefs}; do
|
||||
# Realise the path into the local store (build-parity, no push).
|
||||
p="$(nix build --no-link --print-out-paths "$drv")"
|
||||
printf '%s\n' "$p" >>"$d/out-paths"
|
||||
echo " built: $p"
|
||||
done
|
||||
printf 'attic %s %s\n' ${lib.escapeShellArg cache} ${lib.escapeShellArg endpoint} >"$d/.parity-meta"
|
||||
echo " staged out-paths under $d"
|
||||
'';
|
||||
};
|
||||
|
||||
publish = pkgs.writeShellApplication {
|
||||
name = "publish-${arch}";
|
||||
runtimeInputs = atticInputs;
|
||||
text = ''
|
||||
${head}
|
||||
${atticTokenFn}
|
||||
${atticPushFn}
|
||||
parity_parse_args "Build + push the ${arch} Nix closure to the Attic cache ${cache}" "$@"
|
||||
${lib.getExe stage}
|
||||
d="$(parity_stage_path ${lib.escapeShellArg arch})"
|
||||
mapfile -t paths <"$d/out-paths"
|
||||
if [ "''${#paths[@]}" -eq 0 ]; then
|
||||
echo "BLOCKER(no-stage): no built out-paths under $d — run 'nix run .#stage-${arch}' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PARITY_PUBLISH" != "1" ]; then
|
||||
echo "[dry-run] would 'attic push ${cache}' these paths to ${endpoint}:"
|
||||
for p in "''${paths[@]}"; do echo " - $p"; done
|
||||
echo "[dry-run] re-run with --publish / PUBLISH=1 to push."
|
||||
exit 0
|
||||
fi
|
||||
parity_attic_push "''${paths[@]}"
|
||||
echo "Pushed ''${#paths[@]} path(s) to ${cache}"
|
||||
'';
|
||||
};
|
||||
|
||||
pushStaged = pkgs.writeShellApplication {
|
||||
name = "push-staged";
|
||||
runtimeInputs = atticInputs;
|
||||
text = ''
|
||||
${head}
|
||||
${atticTokenFn}
|
||||
${atticPushFn}
|
||||
parity_parse_args "Replay the staged ${arch} closure out-paths to the Attic cache ${cache}" "$@"
|
||||
d="$(parity_stage_path ${lib.escapeShellArg arch})"
|
||||
if [ ! -f "$d/out-paths" ]; then
|
||||
echo "BLOCKER(no-stage): no staged out-paths at $d — run 'nix run .#stage-${arch}' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
mapfile -t paths <"$d/out-paths"
|
||||
if [ "$PARITY_PUBLISH" != "1" ]; then
|
||||
echo "[dry-run] would 'attic push ${cache}' these staged paths to ${endpoint}:"
|
||||
for p in "''${paths[@]}"; do echo " - $p"; done
|
||||
echo "[dry-run] re-run with --publish / PUBLISH=1 to push."
|
||||
exit 0
|
||||
fi
|
||||
parity_attic_push "''${paths[@]}"
|
||||
echo "Replayed ''${#paths[@]} path(s) to ${cache}"
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
"stage-${arch}" =
|
||||
mkApp stage "Build the ${arch} Nix closure into the local store + stage its out-paths (no push).";
|
||||
"publish-${arch}" =
|
||||
mkApp publish "Build + push the ${arch} closure to the Attic cache ${cache} (dry-run by default; --publish to push).";
|
||||
"publish" =
|
||||
mkApp publish "Push all locally-buildable arches to the Attic cache (${arch} only for this closure repo).";
|
||||
"push-staged" =
|
||||
mkApp pushStaged "Replay the staged ${arch} closure out-paths to the Attic cache (dry-run by default; --publish to push).";
|
||||
};
|
||||
|
||||
mkPushStagedHelm =
|
||||
{
|
||||
pname,
|
||||
@@ -1019,6 +1175,7 @@ in
|
||||
mkNix2ContainerPublish
|
||||
mkGoBinaryPublish
|
||||
mkHelmPublish
|
||||
mkAtticClosurePublish
|
||||
;
|
||||
# Shared building blocks, exposed for advanced/bespoke consumers.
|
||||
inherit mkApp shellLib;
|
||||
|
||||
Reference in New Issue
Block a user