diff --git a/CHANGELOG.md b/CHANGELOG.md index 55de6f0..eb44c3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,40 @@ semantic versioning; the version is a conceptual tag (no git tag is created). ## Unreleased +- **Feature: `verify-digest` for nix2container (cluster #195).** `mkNix2ContainerPublish` + now also returns a `verify-digest` app that builds each locally-buildable arch + image and prints its OCI **manifest digest** with NO registry contact (it + `copyTo`s a throwaway local `oci:` dir and reads the digest skopeo derives). + This formalizes the manifest digest as the parity contract: the OCI layers are + built `reproducible = false` ON PURPOSE (the fix for the "Digest did not match" + caused by non-reproducible layer deps + nix2container's lazy tar regeneration), + so byte-identical-tar parity is NOT promised — but the content-addressed + manifest digest the registry stores the image under IS stable. Identical local + vs CI digest ⇒ identical registry artifact. We do NOT flip `reproducible` to + `true` (the inputs are not reproducible); the LOW-RISK digest-as-contract path + was chosen instead. Generalized from the claude-plugin-registry prototype + (`55f2d0b`) so every nix2container consumer gets it for free. +- **`pipeline-doctor` is now GATE-READY (cluster #191/#193).** It already exited + non-zero on any failing required check; added a `--strict` mode that ALSO fails + on any `WARN`, so a `.woodpecker.yaml` step or a server pre-receive hook can call + `pipeline-doctor --strict ` and rely on the exit code. Added a documented + **`ci/local.sh` escape-hatch** (cluster #196): a repo that must keep a + hand-written Dockerfile/BuildKit pipeline may opt out of the archetype / + parity-lib asserts (downgraded to warnings) if it ships a `ci/local.sh` + local==CI entrypoint. Fixed a false-negative: the token / dev-tag / dry-run / + `meta.description` contracts are GUARANTEED by parity-lib for a consumer (they + live in the generated apps, not the consumer's `flake.nix` text), so a repo that + consumes parity-lib now PASSES those by delegation instead of being penalized + for not re-implementing them inline. Self-check stays green and a known-good + consumer (`numpy-s390x`) now passes 9/9. +- **Audit: stage + push-staged uniform across all 8 builders (cluster #194).** + Verified every archetype builder exposes a `stage-` (build-parity, no + registry contact, writes `./.parity-stage`) AND a `push-staged` (replay the + staged artifact): `mkPyPiWheelPublish`, `mkPyPiWheelPublishMulti`, + `mkS390xNpmPublish`, `mkS390xNpmPublishMulti`, `mkGenericBinaryPublish`, + `mkGoBinaryPublish` (alias), `mkNix2ContainerPublish` and `mkHelmPublish` + (`stage-chart`). All were already complete — no gaps to fill; the build-parity / + publish-parity split is uniform. - **Feature: `mkS390xNpmPublishMulti` (cluster #192).** A multi-version npm builder mirroring the PyPI multi one: publishes a fixed list of `{ version; file; distTag? }` per tag, each staged into its own dir and diff --git a/ci/pipeline-doctor.sh b/ci/pipeline-doctor.sh index 022b679..bcfa73c 100755 --- a/ci/pipeline-doctor.sh +++ b/ci/pipeline-doctor.sh @@ -5,15 +5,40 @@ # emmett#44 parity standard, and prints the local-equivalent commands a dev can # run. It is read-only: it never touches the registry and never needs a token. # -# Usage: pipeline-doctor [] (default: .) -# Exit: 0 = all required checks pass; 1 = one or more required checks failed. +# Usage: pipeline-doctor [--strict] [] (default repo: .) +# --strict also exit non-zero if any WARN was emitted (gate the soft checks). +# Exit: 0 = all required checks pass (and, in --strict, no warnings); +# 1 = one or more required checks failed (or a warning under --strict). +# +# This is a GATE-READY check (cluster #191/#193): a .woodpecker.yaml step or a +# server pre-receive hook can call `pipeline-doctor --strict ` and rely on +# the exit code. It is read-only: it never touches the registry, never needs a +# token, and never prints one. set -euo pipefail -REPO="${1:-.}" +STRICT=0 +REPO="." +for a in "$@"; do + case "$a" in + --strict) STRICT=1 ;; + -h | --help) + echo "Usage: pipeline-doctor [--strict] []" + echo " Assert the parity contract for (default: .)." + echo " --strict exit non-zero on any WARN as well as any FAIL." + exit 0 + ;; + -*) + echo "error: unknown option '$a' (try --help)" >&2 + exit 2 + ;; + *) REPO="$a" ;; + esac +done REPO="$(cd "$REPO" && pwd)" FLAKE="$REPO/flake.nix" fail=0 +warn_n=0 pass_n=0 note() { printf ' %s %s\n' "$1" "$2"; } ok() { @@ -24,7 +49,10 @@ bad() { fail=$((fail + 1)) note "FAIL" "$1" } -warn() { note "WARN" "$1"; } +warn() { + warn_n=$((warn_n + 1)) + note "WARN" "$1" +} echo "pipeline-doctor: $REPO" @@ -35,20 +63,47 @@ fi flake_txt="$(cat "$FLAKE")" +# Documented escape-hatch (cluster #196): a repo that must keep a hand-written +# Dockerfile/OCI pipeline (BuildKit, etc.) instead of a parity-lib builder is +# allowed to opt out PROVIDED it ships a `ci/local.sh` that gives a dev the same +# local==CI entrypoint. When present, the archetype + parity-lib asserts are +# DOWNGRADED to warnings (the escape-hatch is the contract); the token / dev-tag +# / dry-run / set-x asserts below still apply in full to ci/local.sh. +ESCAPE_HATCH=0 +if [ -f "$REPO/ci/local.sh" ]; then + ESCAPE_HATCH=1 +fi + # 1. archetype declared (a publish/stage app whose name encodes the arch, or an # explicit archetype marker comment). if printf '%s' "$flake_txt" | grep -Eq 'archetype|stage-[a-z0-9_]+|publish-[a-z0-9_]+'; then ok "archetype declared (stage-*/publish-* app or archetype marker present)" +elif [ "$ESCAPE_HATCH" -eq 1 ]; then + warn "no parity-lib archetype, but ci/local.sh escape-hatch present (cluster #196)" else - bad "no archetype: expected a stage-/publish- app or 'archetype' marker" + bad "no archetype: expected a stage-/publish- app or 'archetype' marker (or a ci/local.sh escape-hatch)" fi -# 2. consumes parity-lib (the shared module). +# 2. consumes parity-lib (the shared module) OR a documented ci/local.sh hatch. +CONSUMES_PARITY=0 if printf '%s' "$flake_txt" | grep -Eq 'parity-lib|parity\.lib|mk(PyPiWheel|S390xNpm|GenericBinary|Nix2Container|GoBinary|Helm)Publish'; then - ok "consumes parity-lib (input or one of the mk*Publish builders)" -else - bad "does not consume parity-lib: inline the input and use a mk*Publish builder" + CONSUMES_PARITY=1 fi +if [ "$CONSUMES_PARITY" -eq 1 ]; then + ok "consumes parity-lib (input or one of the mk*Publish builders)" +elif [ "$ESCAPE_HATCH" -eq 1 ]; then + warn "does not consume parity-lib, but ci/local.sh escape-hatch present (cluster #196)" +else + bad "does not consume parity-lib: inline the input and use a mk*Publish builder (or ship a ci/local.sh escape-hatch)" +fi + +# Checks 3–6 below are GUARANTEED by parity-lib for a consumer: the token +# resolution, dev-tag guard, dry-run gate and meta.description all live inside +# the mk*Publish-generated apps (in the parity-lib store path), not in the +# consumer's flake.nix text. So when the repo consumes parity-lib, those four +# contracts are satisfied BY DELEGATION and we don't re-grep the consumer's +# source for them. They are only grepped for a non-consumer (escape-hatch or +# bespoke) repo, where the consumer's own ci/*.sh must carry them. # 3. token = $REGISTRY_TOKEN with pass fallback, and never printed. token_src="$flake_txt" @@ -56,7 +111,9 @@ if [ -d "$REPO/ci" ]; then token_src="$token_src $(cat "$REPO"/ci/*.sh 2>/dev/null || true)" fi -if printf '%s' "$token_src" | grep -Eq 'REGISTRY_TOKEN' && +if [ "$CONSUMES_PARITY" -eq 1 ]; then + ok "token = \$REGISTRY_TOKEN with pass fallback (delegated to parity-lib)" +elif printf '%s' "$token_src" | grep -Eq 'REGISTRY_TOKEN' && printf '%s' "$token_src" | grep -Eq 'parity_resolve_token|pass (show )?infra/gitea/personal_access_token_packages_rw'; then ok "token = \$REGISTRY_TOKEN with pass fallback" else @@ -111,21 +168,27 @@ else fi # 4. dev-tag guard present. -if printf '%s' "$token_src" | grep -Eq 'parity_devtag_guard|refusing to publish without an explicit'; then +if [ "$CONSUMES_PARITY" -eq 1 ]; then + ok "dev-tag guard present (delegated to parity-lib parity_devtag_guard)" +elif printf '%s' "$token_src" | grep -Eq 'parity_devtag_guard|refusing to publish without an explicit'; then ok "dev-tag guard present" else bad "no dev-tag guard: a default-version --publish could clobber the registry" fi # 5. a --dry-run default exists. -if printf '%s' "$token_src" | grep -Eq 'parity_parse_args|DRY-RUN|dry-run|PUBLISH:-0|PUBLISH=\$\{PUBLISH:-0\}'; then +if [ "$CONSUMES_PARITY" -eq 1 ]; then + ok "dry-run default exists (delegated to parity-lib parity_parse_args)" +elif printf '%s' "$token_src" | grep -Eq 'parity_parse_args|DRY-RUN|dry-run|PUBLISH:-0|PUBLISH=\$\{PUBLISH:-0\}'; then ok "dry-run default exists (publish mutates only on --publish/PUBLISH=1)" else bad "no dry-run default: publish must NOT mutate the registry by default" fi # 6. apps carry meta.description. -if printf '%s' "$flake_txt" | grep -Eq 'meta\.description|meta = \{'; then +if [ "$CONSUMES_PARITY" -eq 1 ]; then + ok "apps carry meta.description (delegated to parity-lib mkApp)" +elif printf '%s' "$flake_txt" | grep -Eq 'meta\.description|meta = \{'; then ok "apps carry meta.description" else bad "apps missing meta.description" @@ -147,6 +210,15 @@ echo " nix run .#publish- # stage + push that arch (DRY-RUN; add echo " nix run .#publish-index # build-free multi-arch assembly from pushed digests" echo " nix run .#publish # all local arches + index; :latest = digest copy of :TAG" echo " nix run .#push-staged # replay .parity-stage to the registry (cluster-was-down)" +echo " nix run .#verify-digest # OCI only: print local manifest digests (parity assertion)" echo "" -echo "Summary: $pass_n passed, $fail failed." -[ "$fail" -eq 0 ] +echo "Summary: $pass_n passed, $fail failed, $warn_n warned." +if [ "$fail" -ne 0 ]; then + echo "RESULT: parity contract VIOLATED ($fail failing check(s))." >&2 + exit 1 +fi +if [ "$STRICT" -eq 1 ] && [ "$warn_n" -ne 0 ]; then + echo "RESULT: --strict and $warn_n warning(s) present — failing the gate." >&2 + exit 1 +fi +echo "RESULT: parity contract satisfied." diff --git a/lib/builders.nix b/lib/builders.nix index 6269e1f..ecc1446 100644 --- a/lib/builders.nix +++ b/lib/builders.nix @@ -12,6 +12,10 @@ # publish all locally-buildable arches + publish-index; ':latest' is a # digest copy of ':TAG' done LAST as the idempotent mutation. # push-staged replay artifacts from ./.parity-stage to the registry. +# verify-digest (OCI only, cluster #195) build each local arch image and print +# its OCI manifest digest — the content-addressed parity contract +# (layers are reproducible = false, so parity is at the digest, +# not byte-identical tars). No registry contact. # # Not every archetype yields every app: a single-arch wheel/binary/npm addon # has no multi-arch index, so it exposes stage/publish/push-staged only. The OCI @@ -699,6 +703,42 @@ let ''; }; + # Compute one arch image's OCI manifest digest with NO registry contact + # (cluster #195). copyTo a throwaway local OCI dir and read back the + # content-addressed manifest digest skopeo derives — this is the digest the + # registry stores the image under. Because the layer is built with + # reproducible = false (a DELIBERATE fix for the "Digest did not match" + # from non-reproducible layer deps — nix2container's lazy tar regeneration + # rehashes differently across hosts), byte-identical-tar parity is NOT + # promised; the manifest digest IS the parity contract. Identical local vs + # CI digest ⇒ identical image in the registry. + digestArch = arch: '' + echo "→ ${imageName}:$VERSION-${arch} (local build, no registry contact)" + ocidir="$(mktemp -d)" + ${lib.escapeShellArg "${images.${arch}.copyTo}"}/bin/copy-to "oci:$ocidir:${arch}" >/dev/null + digest="$(skopeo manifest-digest "$ocidir/blobs/sha256/$( + jq -r '.manifests[0].digest | sub("sha256:";"")' "$ocidir/index.json" + )")" + rm -rf "$ocidir" + echo " ${arch}: $digest" + ''; + + verifyDigest = pkgs.writeShellApplication { + name = "verify-digest"; + runtimeInputs = baseInputs ++ [ + pkgs.nix + pkgs.skopeo + pkgs.jq + ]; + text = '' + ${head} + VERSION="$(parity_derive_version ${lib.escapeShellArg version})" + echo "OCI manifest digests for ${imageName}:$VERSION (content-addressed parity, cluster #195)" + echo " reproducible = false ⇒ parity is asserted at this digest, not byte-identical tars." + ${lib.concatMapStringsSep "" digestArch localArches} + ''; + }; + mkArchPublish = arch: pkgs.writeShellApplication { @@ -838,6 +878,8 @@ let mkApp publishAll "Publish all local arches + index; :latest = digest copy of :TAG (last mutation)."; "push-staged" = mkApp pushStaged "Replay staged ${imageName} arch closures from .parity-stage to the registry."; + "verify-digest" = + mkApp verifyDigest "Build each local arch image and print its OCI manifest digest (content-addressed parity check, cluster #195; no registry contact)."; }; # =========================================================================