# Per-archetype publish-app BUILDERS (cluster #192/#194, emmett#44). # # `mkParityBuilders { pkgs }` returns the six mk*Publish builders plus the # shared building blocks. Each mk*Publish takes a small attrset and returns an # attrset of flake apps following the corrected parity standard: # # stage- BUILD-parity, cluster-independent: writes the artifact to the # on-disk stage (./.parity-stage/), NO registry contact. # publish- stage + push that arch. DRY-RUN by default (--publish to push). # publish-index build-free multi-arch assembly from digest-pinned refs # (regctl), fail-closed if a required arch was not pushed. # publish all locally-buildable arches + publish-index; ':latest' is a # digest copy of ':TAG' done LAST as the idempotent mutation. # push-staged replay artifacts from ./.parity-stage to the registry. # verify-digest (OCI only, cluster #195) build each local arch image and print # its OCI manifest digest — the content-addressed parity contract # (layers are reproducible = false, so parity is at the digest, # not byte-identical tars). No registry contact. # # Not every archetype yields every app: a single-arch wheel/binary/npm addon # has no multi-arch index, so it exposes stage/publish/push-staged only. The OCI # (nix2container) builder is the one that yields the full index/latest set. { pkgs }: let inherit (pkgs) lib; shellLib = ../ci/parity-lib.sh; # Common preamble: source the shared shell helpers from the store and apply # any per-app coordinate overrides BEFORE the helpers are used. preamble = { registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: '' set -euo pipefail # shellcheck source=/dev/null . ${shellLib} # Coordinate overrides are read by the sourced lib; export so shellcheck # treats them as used (SC2034) and child processes inherit them. export PARITY_REGISTRY_HOST=${lib.escapeShellArg registryHost} export PARITY_REGISTRY_OWNER=${lib.escapeShellArg registryOwner} export PARITY_PASS_ENTRY=${lib.escapeShellArg passEntry} export PARITY_STAGE_DIR="''${PARITY_STAGE_DIR:-''${PWD}/.parity-stage}" ''; mkApp = drv: desc: { type = "app"; program = lib.getExe drv; meta.description = desc; }; baseInputs = with pkgs; [ coreutils curl findutils gnugrep gnused pass ]; # ========================================================================= # PyPI wheel (single-arch). Stage a built wheel into .parity-stage, then # POST it to the Gitea PyPI registry (idempotent: 409 = already present). # Args: { pname, version|versionFn, wheel (drv|path producing *.whl), # arch ? "s390x", registryHost?, registryOwner?, passEntry? } # ========================================================================= mkPyPiWheelPublish = { pname, version, wheel, arch ? "s390x", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", extractCmd ? null, }: let head = preamble { inherit registryHost registryOwner passEntry; }; # The wheel input may be a directory (Nix store output) we glob for *.whl, # or an explicit single .whl path. wheelRef = "${wheel}"; stageText = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "→ staging ${pname} $VERSION wheel (${arch}, no registry contact)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" src=${lib.escapeShellArg wheelRef} if [ -d "$src" ]; then # shellcheck disable=SC2044 for w in $(find "$src" -name '*.whl'); do cp "$w" "$d/"; done else cp "$src" "$d/" fi whl="$(find "$d" -name '*.whl' | head -1)" if [ -z "$whl" ]; then echo "BLOCKER(no-wheel): no *.whl under $src" >&2; exit 1; fi printf 'pypi %s %s\n' "${pname}" "$VERSION" >"$d/.parity-meta" sha256sum "$whl" echo " staged: $whl" ''; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = stageText; }; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} parity_parse_args "Build + publish the ${pname} ${arch} wheel to the Gitea PyPI registry" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" whl="$(find "$d" -name '*.whl' | head -1)" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would POST $(basename "$whl") -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi (name=${pname} version=$VERSION)" echo "[dry-run] re-run with --publish / PUBLISH=1 to upload." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight sha="$(sha256sum "$whl" | cut -d' ' -f1)" http="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ -H "Authorization: token $tok" \ -F ':action=file_upload' -F 'protocol_version=1' \ -F "sha256_digest=$sha" -F "name=${pname}" -F "version=$VERSION" \ -F 'filetype=bdist_wheel' -F "content=@$whl" \ "https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi")" case "$http" in 200 | 201 | 409) echo "Published ${pname} $VERSION (HTTP $http)" ;; *) echo "BLOCKER(upload-failed): POST returned HTTP $http" >&2; exit 1 ;; esac ''; }; pushStaged = mkPushStagedGeneric { inherit pname arch registryHost registryOwner passEntry ; kind = "pypi"; }; in { "stage-${arch}" = mkApp stage "Stage the ${pname} ${arch} wheel into .parity-stage (no registry contact)."; "publish-${arch}" = mkApp publish "Stage + publish the ${pname} ${arch} wheel (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish all locally-buildable arches (${arch} only for this PyPI repo)."; "push-staged" = pushStaged; }; # ========================================================================= # PyPI wheels, MULTI-version (cluster #197). Same archetype as the single # builder, but publishes a fixed list of {version, wheel} per tag (the old # per-repo CI behaviour several *-s390x repos rely on) instead of just the # default version. The real version of each wheel is read from its filename # (PEP 427), so stage/publish/push-staged need no side-channel map and a # re-run is idempotent (409-skip per version). No dev-tag guard: the versions # are an explicit fixed list of real releases, gated by the dry-run default # (--publish / PUBLISH=1 is the deliberate intent), matching the old "publish # all on a v* tag" pipeline. # Args: { pname, versions = [ { version; wheel; } ... ], arch ? "s390x", registry* } # ========================================================================= mkPyPiWheelPublishMulti = { pname, versions, arch ? "s390x", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; nVersions = builtins.length versions; # One copy block per version's wheel (drv dir we glob, or explicit .whl). stageCopies = lib.concatMapStringsSep "\n" (e: '' src=${lib.escapeShellArg "${e.wheel}"} if [ -d "$src" ]; then # shellcheck disable=SC2044 for w in $(find "$src" -name '*.whl'); do cp "$w" "$d/"; done else cp "$src" "$d/" fi '') versions; stageText = '' ${head} echo "→ staging ${pname} (${toString nVersions} versions, ${arch}, no registry contact)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" ${stageCopies} n="$(find "$d" -name '*.whl' | wc -l)" if [ "$n" -eq 0 ]; then echo "BLOCKER(no-wheel): no *.whl staged under $d" >&2; exit 1; fi printf 'pypi-multi %s %s\n' ${lib.escapeShellArg pname} "$n" >"$d/.parity-meta" echo " staged $n wheel(s) under $d" ''; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = stageText; }; # Shared publish loop body, reused by publish and push-staged: POST every # staged wheel, version parsed from its filename; aggregate failures. uploadLoop = '' tok="$(parity_resolve_token)" parity_registry_preflight rc=0 for whl in "$d"/*.whl; do ver="$(parity_wheel_version "$whl")" parity_pypi_post ${lib.escapeShellArg pname} "$ver" "$whl" "$tok" || rc=1 done exit "$rc" ''; dryList = '' echo "[dry-run] would POST these wheels -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi (name=${pname}):" for whl in "$d"/*.whl; do echo " - $(basename "$whl") (version $(parity_wheel_version "$whl"))"; done echo "[dry-run] re-run with --publish / PUBLISH=1 to upload." ''; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} parity_parse_args "Build + publish all ${toString nVersions} ${pname} ${arch} wheels to the Gitea PyPI registry" "$@" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ "$PARITY_PUBLISH" != "1" ]; then ${dryList} exit 0 fi ${uploadLoop} ''; }; pushStaged = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} parity_parse_args "Replay the staged ${pname} ${arch} wheels from .parity-stage" "$@" d="$(parity_stage_path ${lib.escapeShellArg arch})" if ! find "$d" -name '*.whl' -print -quit | grep -q .; then echo "BLOCKER(no-stage): no staged wheels under $d — run 'nix run .#stage-${arch}' first." >&2 exit 1 fi if [ "$PARITY_PUBLISH" != "1" ]; then ${dryList} exit 0 fi ${uploadLoop} ''; }; in { "stage-${arch}" = mkApp stage "Stage all ${nVersions} ${pname} ${arch} wheels into .parity-stage (no registry contact)."; "publish-${arch}" = mkApp publish "Stage + publish all ${pname} ${arch} wheels (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish all ${pname} ${arch} wheels (${arch} only for this PyPI repo)."; "push-staged" = mkApp pushStaged "Replay the staged ${pname} ${arch} wheels from .parity-stage (dry-run by default; --publish to push)."; }; # ========================================================================= # s390x npm native addon (single-arch). Stage a ready-to-publish package dir # (the .node + a generated package.json) then `npm publish`. # Args: { pname (npm full name, e.g. @rollup/rollup-linux-s390x-gnu), # version, nodeFile (drv|path to the *.node), nodeFileName, # packageJson (string, with $version interpolated by the app), # arch ? "s390x", registry* } # ========================================================================= mkS390xNpmPublish = { pname, version, nodeFile, nodeFileName, packageJson, arch ? "s390x", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nodejs ]; text = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "→ staging ${pname}@$VERSION (${arch}, no registry contact)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" install -m 0644 ${lib.escapeShellArg "${nodeFile}"} "$d/${nodeFileName}" cat >"$d/package.json" <"$d/.parity-meta" echo " staged into $d:" ls -lh "$d" ''; }; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nodejs ]; text = '' ${head} parity_parse_args "Build + publish the ${pname} ${arch} npm addon to the Gitea npm registry" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'npm publish' $d -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/" echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight reg="https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/" ( cd "$d" set +e out="$(npm publish --registry="$reg" \ "--//$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/:_authToken=$tok" 2>&1)" rc=$? set -e printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g" if [ "$rc" -ne 0 ]; then if printf '%s' "$out" | grep -qiE 'already exists|409|conflict'; then echo "Version already published — idempotent success."; exit 0 fi exit "$rc" fi ) echo "Published ${pname}@$VERSION" ''; }; in { "stage-${arch}" = mkApp stage "Stage the ${pname} ${arch} npm addon into .parity-stage (no registry contact)."; "publish-${arch}" = mkApp publish "Stage + publish the ${pname} ${arch} npm addon (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish all locally-buildable arches (${arch} only for this npm repo)."; "push-staged" = mkPushStagedGeneric { inherit pname arch registryHost registryOwner passEntry ; kind = "npm"; }; }; # ========================================================================= # s390x npm native addon / binary, MULTI-version (cluster #192). Publishes a # fixed list of { version; file; distTag? } per tag — the behaviour repos like # nextjs-swc (16.1.6 @latest + 15.2.0 @next15) and sentry-cli (3.2.2 + 2.38.2) # rely on. `file` may be a .node addon OR a plain binary; `fileName` is its name # inside the package and `packageJson` (with a $VERSION the stage heredoc # expands) declares the shape (a `main` for an addon, a `bin` for an exe). Each # version is staged into its own dir and published with its dist-tag; idempotent # (npm "already exists" == success). No dev-tag guard: explicit fixed version # list, gated by the dry-run default. # Args: { pname, versions = [ { version; file; distTag ? "latest"; } ], # fileName, packageJson, arch ? "s390x", registry* } # ========================================================================= mkS390xNpmPublishMulti = { pname, versions, fileName, packageJson, arch ? "s390x", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; nVersions = builtins.length versions; # One stage block per version: its own dir, the file, a version-stamped # package.json (the <"$vd/.disttag" VERSION=${lib.escapeShellArg e.version} cat >"$vd/package.json" <"$d/.parity-meta" echo " staged ${toString nVersions} version dir(s) under $d" ''; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nodejs ]; text = stageText; }; dryList = '' echo "[dry-run] would 'npm publish' these ${pname} versions -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/:" for vd in "$d"/*/; do [ -f "$vd/package.json" ] || continue echo " - $(basename "$vd") (dist-tag $(cat "$vd/.disttag"))" done echo "[dry-run] re-run with --publish / PUBLISH=1 to push." ''; uploadLoop = '' tok="$(parity_resolve_token)" parity_registry_preflight rc=0 for vd in "$d"/*/; do [ -f "$vd/package.json" ] || continue parity_npm_publish_dir "$vd" "$tok" "$(cat "$vd/.disttag")" || rc=1 done exit "$rc" ''; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nodejs ]; text = '' ${head} parity_parse_args "Build + publish all ${toString nVersions} ${pname} ${arch} npm versions to the Gitea npm registry" "$@" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ "$PARITY_PUBLISH" != "1" ]; then ${dryList} exit 0 fi ${uploadLoop} ''; }; pushStaged = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = baseInputs ++ [ pkgs.nodejs ]; text = '' ${head} parity_parse_args "Replay the staged ${pname} ${arch} npm versions from .parity-stage" "$@" d="$(parity_stage_path ${lib.escapeShellArg arch})" if ! find "$d" -mindepth 2 -name package.json -print -quit | grep -q .; then echo "BLOCKER(no-stage): no staged versions under $d — run 'nix run .#stage-${arch}' first." >&2 exit 1 fi if [ "$PARITY_PUBLISH" != "1" ]; then ${dryList} exit 0 fi ${uploadLoop} ''; }; in { "stage-${arch}" = mkApp stage "Stage all ${nVersions} ${pname} ${arch} npm versions into .parity-stage (no registry contact)."; "publish-${arch}" = mkApp publish "Stage + publish all ${pname} ${arch} npm versions (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish all ${pname} ${arch} npm versions (${arch} only for this repo)."; "push-staged" = mkApp pushStaged "Replay the staged ${pname} ${arch} npm versions from .parity-stage (dry-run by default; --publish to push)."; }; # ========================================================================= # Generic binary (single-arch). Stage a built binary, PUT it to the Gitea # generic registry (idempotent: 201 created / 409 exists). # Args: { pname, version, binary (drv|path), assetName, arch ? "s390x", # registry* } # ========================================================================= mkGenericBinaryPublish = { pname, version, binary, assetName, arch ? "s390x", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix pkgs.file ]; text = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "→ staging ${pname} $VERSION (${arch}, no registry contact)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" cp ${lib.escapeShellArg "${binary}"} "$d/${assetName}" printf 'generic %s %s %s\n' "${pname}" "$VERSION" "${assetName}" >"$d/.parity-meta" file "$d/${assetName}" 2>/dev/null || true echo " staged: $d/${assetName}" ''; }; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix pkgs.file ]; text = '' ${head} parity_parse_args "Build + publish the ${pname} ${arch} binary to the Gitea generic registry" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would PUT $d/${assetName} -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/generic/${pname}/$VERSION/${assetName}" echo "[dry-run] re-run with --publish / PUBLISH=1 to upload." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight http="$(curl -s -o /dev/null -w '%{http_code}' -X PUT \ -H "Authorization: token $tok" --upload-file "$d/${assetName}" \ "https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/generic/${pname}/$VERSION/${assetName}")" case "$http" in 201 | 409) echo "Published ${assetName} $VERSION (HTTP $http)" ;; *) echo "BLOCKER(upload-failed): PUT returned HTTP $http" >&2; exit 1 ;; esac ''; }; in { "stage-${arch}" = mkApp stage "Stage the ${pname} ${arch} binary into .parity-stage (no registry contact)."; "publish-${arch}" = mkApp publish "Stage + publish the ${pname} ${arch} binary (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish all locally-buildable arches (${arch} only for this generic-binary repo)."; "push-staged" = mkPushStagedGeneric { inherit pname arch assetName registryHost registryOwner passEntry ; kind = "generic"; inherit version; }; }; # ========================================================================= # Go binary (single-arch generic). Thin alias over the generic-binary builder # — a cross-compiled Go binary is published to the generic registry exactly # like any other binary. Kept as a distinct named builder so the archetype is # explicit in consumer flakes and pipeline-doctor. # ========================================================================= mkGoBinaryPublish = mkGenericBinaryPublish; # ========================================================================= # Helm chart (single artifact). Stage `helm package` output (.tgz) then # `helm push` to the OCI registry (idempotent: a duplicate version errors, # which we treat as success). # Args: { pname (chart name), version, chartSrc (drv|path to chart dir), # ociRepo ? "oci://git.oleks.space/oleks", registry* } # ========================================================================= mkHelmPublish = { pname, version, chartSrc, ociRepo ? "oci://git.oleks.space/oleks", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; stage = pkgs.writeShellApplication { name = "stage-chart"; runtimeInputs = baseInputs ++ [ pkgs.kubernetes-helm ]; text = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "→ staging helm chart ${pname} $VERSION (no registry contact)" d="$(parity_stage_reset chart)" helm package ${lib.escapeShellArg "${chartSrc}"} --version "$VERSION" --app-version "$VERSION" --destination "$d" printf 'helm %s %s\n' "${pname}" "$VERSION" >"$d/.parity-meta" echo " staged:"; ls -lh "$d"/*.tgz ''; }; publish = pkgs.writeShellApplication { name = "publish-chart"; runtimeInputs = baseInputs ++ [ pkgs.kubernetes-helm ]; text = '' ${head} parity_parse_args "Package + publish the ${pname} helm chart to ${ociRepo}" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" ${lib.getExe stage} d="$(parity_stage_path chart)" tgz="$(find "$d" -name '*.tgz' | head -1)" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'helm push $(basename "$tgz")' -> ${ociRepo}" echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight echo "$tok" | helm registry login "$PARITY_REGISTRY_HOST" -u "$PARITY_REGISTRY_OWNER" --password-stdin >/dev/null 2>&1 set +e out="$(helm push "$tgz" ${lib.escapeShellArg ociRepo} 2>&1)" rc=$? set -e printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g" if [ "$rc" -ne 0 ]; then if printf '%s' "$out" | grep -qiE 'already exists|409|conflict'; then echo "Chart version already published — idempotent success."; exit 0 fi exit "$rc" fi echo "Published chart ${pname} $VERSION" ''; }; in { "stage-chart" = mkApp stage "Package the ${pname} helm chart into .parity-stage (no registry contact)."; "publish-chart" = mkApp publish "Package + publish the ${pname} helm chart (dry-run by default; --publish to push)."; "publish" = mkApp publish "Publish the ${pname} helm chart."; "push-staged" = mkPushStagedHelm { inherit pname ociRepo registryHost registryOwner passEntry ; }; }; # ========================================================================= # nix2container OCI image (MULTI-ARCH). This is the archetype that yields the # full stage/publish-/publish-index/publish/push-staged set. # # Args: { # imageName, # e.g. git.oleks.space/oleks/nix-ci # version|tagFn, # the immutable :TAG # images, # { = { copyTo = ; }; } # # only arches buildable on THIS host should appear # arches ? attrNames images, # the full set the index must cover (fail-closed) # registry* } # # Per-arch publish copies the arch image to :-. # publish-index assembles : from the per-arch digest-pinned # tags via `regctl index create`, failing closed if any required arch tag is # missing. publish runs all local arches, then the index, then copies # : -> :latest as the LAST idempotent mutation. # ========================================================================= mkNix2ContainerPublish = { imageName, version, images, arches ? lib.attrNames images, registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", # IMPURE-BUILD mode (cluster #205). Default: each `images.` carries a # `copyTo` derivation referenced as an EVAL-TIME store path (a build dep of # the app) — correct for a pure image. But some images are built IMPURELY # (need a registry token / network / sandbox off at build time, e.g. # ii-agent fetching private wheels), so the closure CANNOT be a build dep # of the app. When `impureBuild = true`, each `images.` instead # carries `attr` (a flake attribute on the CONSUMER's own flake, e.g. # "frontend-image"), and stage/publish/verify-digest build it at RUN time # via `nix build --impure --option sandbox false ".#.copyTo"`. impureBuild ? false, # When true, `publish-index` ALSO moves `:latest` (a dev-guarded digest # copy of `:VERSION`) after assembling the index. Default off: `:latest` # is owned by the `publish` app. Repos that publish each arch on a SEPARATE # CI agent and converge in a final index-only step (ii-agent) need the # latest-move to live in `publish-index`. cluster #205. indexMovesLatest ? false, }: let head = preamble { inherit registryHost registryOwner passEntry; }; localArches = lib.attrNames images; # Resolve one arch's nix2container copy-to wrapper into shell var $1. # Pure: an eval-time store path (build dep of the app). Impure (cluster # #205): build the consumer's own flake attr at RUN time and read the path # back — the impure layers (token/network/sandbox-off) never become a build # dep of the app, which would otherwise fail under the pure sandbox. resolveCopyTo = arch: var: if impureBuild then '' IMAGE_TAG="$VERSION-${arch}" nix build --impure --no-link \ --option sandbox false ".#${images.${arch}.attr}.copyTo" >/dev/null ${var}="$(IMAGE_TAG="$VERSION-${arch}" nix eval --impure --raw \ ".#${images.${arch}.attr}.copyTo")" '' else "${var}=${lib.escapeShellArg "${images.${arch}.copyTo}"}"; # per-arch stage: realise the image's copy-to closure locally (no push). mkArchStage = arch: pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "→ staging ${imageName}:$VERSION-${arch} (build closure only, no push)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" # Realise the nix2container copyTo wrapper (its closure is the image). ${resolveCopyTo arch "out"} printf 'oci %s %s %s\n' ${lib.escapeShellArg imageName} "$VERSION" "${arch}" >"$d/.parity-meta" printf '%s\n' "$out" >"$d/copy-to-path" echo " staged copyTo: $out" ''; }; # Compute one arch image's OCI manifest digest with NO registry contact # (cluster #195). copyTo a throwaway local OCI dir and read back the # content-addressed manifest digest skopeo derives — this is the digest the # registry stores the image under. Because the layer is built with # reproducible = false (a DELIBERATE fix for the "Digest did not match" # from non-reproducible layer deps — nix2container's lazy tar regeneration # rehashes differently across hosts), byte-identical-tar parity is NOT # promised; the manifest digest IS the parity contract. Identical local vs # CI digest ⇒ identical image in the registry. digestArch = arch: '' echo "→ ${imageName}:$VERSION-${arch} (local build, no registry contact)" ocidir="$(mktemp -d)" ${resolveCopyTo arch "copyto"} "$copyto/bin/copy-to" "oci:$ocidir:${arch}" >/dev/null digest="$(skopeo manifest-digest "$ocidir/blobs/sha256/$( jq -r '.manifests[0].digest | sub("sha256:";"")' "$ocidir/index.json" )")" rm -rf "$ocidir" echo " ${arch}: $digest" ''; verifyDigest = pkgs.writeShellApplication { name = "verify-digest"; runtimeInputs = baseInputs ++ [ pkgs.nix pkgs.skopeo pkgs.jq ]; text = '' ${head} VERSION="$(parity_derive_version ${lib.escapeShellArg version})" echo "OCI manifest digests for ${imageName}:$VERSION (content-addressed parity, cluster #195)" echo " reproducible = false ⇒ parity is asserted at this digest, not byte-identical tars." ${lib.concatMapStringsSep "" digestArch localArches} ''; }; mkArchPublish = arch: pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} parity_parse_args "Build + push ${imageName}:-${arch}" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" ${lib.getExe (mkArchStage arch)} if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would copy ${imageName}:$VERSION-${arch} (digest-pinned per-arch tag)" echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight creds="$PARITY_REGISTRY_OWNER:$tok" # Read the copy-to path the stage step just wrote (works for both # pure and impure modes — stage already realised/built the closure). copyto="$(cat "$PARITY_STAGE_DIR/${arch}/copy-to-path")" "$copyto/bin/copy-to" \ --dest-creds "$creds" \ "docker://${imageName}:$VERSION-${arch}" echo "Pushed ${imageName}:$VERSION-${arch}" ''; }; archStageApps = lib.listToAttrs ( map (a: { name = "stage-${a}"; value = mkApp (mkArchStage a) "Stage the ${imageName} ${a} image closure into .parity-stage (no push)."; }) localArches ); archPublishApps = lib.listToAttrs ( map (a: { name = "publish-${a}"; value = mkApp (mkArchPublish a) "Build + push ${imageName}:-${a} (dry-run by default; --publish to push)."; }) localArches ); # build-free multi-arch index from the per-arch digest-pinned tags. indexApp = pkgs.writeShellApplication { name = "publish-index"; runtimeInputs = baseInputs ++ [ pkgs.regclient ]; text = '' ${head} parity_parse_args "Assemble the multi-arch index ${imageName}: from pushed per-arch tags" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" req=(${lib.concatStringsSep " " (map lib.escapeShellArg arches)}) if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'regctl index create ${imageName}:$VERSION' from:" for a in "''${req[@]}"; do echo " ${imageName}:$VERSION-$a"; done echo "[dry-run] re-run with --publish / PUBLISH=1 to assemble." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight regctl registry login "$PARITY_REGISTRY_HOST" -u "$PARITY_REGISTRY_OWNER" --pass-stdin <<<"$tok" >/dev/null 2>&1 # Fail-closed: every required arch tag must exist before assembly. refs=() for a in "''${req[@]}"; do ref="${imageName}:$VERSION-$a" if ! regctl manifest head "$ref" >/dev/null 2>&1; then echo "BLOCKER(missing-arch): required arch tag $ref was not pushed this run; refusing to assemble a partial index." >&2 exit 1 fi refs+=("$ref") done regctl index create "${imageName}:$VERSION" "''${refs[@]/#/--ref=}" echo "Assembled index ${imageName}:$VERSION" ${lib.optionalString indexMovesLatest '' # Repos that converge per-arch legs in this index-only step own the # ':latest' move here (rather than in `publish`). Dev-guarded; a pure # digest copy of the just-assembled ':VERSION', done LAST. case "$VERSION" in *dev*) echo " dev version $VERSION — refusing to move :latest" ;; *) regctl image copy "${imageName}:$VERSION" "${imageName}:latest" echo "→ ${imageName}:$VERSION -> :latest (digest copy)" ;; esac ''} ''; }; # publish: all locally-buildable arches -> index -> :latest (last). publishAll = pkgs.writeShellApplication { name = "publish"; runtimeInputs = baseInputs ++ [ pkgs.nix pkgs.regclient ]; text = '' ${head} parity_parse_args "Publish all local arches of ${imageName} + the multi-arch index; :latest = digest copy of :TAG" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" fwd=() [ "$PARITY_PUBLISH" = "1" ] && fwd=(--publish) ${lib.concatMapStringsSep "\n" ( a: " ${lib.getExe (mkArchPublish a)} \"\${fwd[@]}\"" ) localArches} ${lib.getExe indexApp} "''${fwd[@]}" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would copy ${imageName}:$VERSION -> ${imageName}:latest (digest copy, LAST mutation)" exit 0 fi tok="$(parity_resolve_token)" regctl registry login "$PARITY_REGISTRY_HOST" -u "$PARITY_REGISTRY_OWNER" --pass-stdin <<<"$tok" >/dev/null 2>&1 # ':latest' is a pure digest copy of the just-assembled ':TAG', done # LAST so it is the single idempotent mutation that flips the channel. regctl image copy "${imageName}:$VERSION" "${imageName}:latest" echo "Published ${imageName}:$VERSION and flipped :latest" ''; }; pushStaged = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = baseInputs ++ [ pkgs.nix ]; text = '' ${head} parity_parse_args "Replay staged ${imageName} arch closures from .parity-stage to the registry" "$@" VERSION="$(parity_derive_version ${lib.escapeShellArg version})" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would replay staged arch images under $PARITY_STAGE_DIR to ${imageName}:$VERSION-" echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight creds="$PARITY_REGISTRY_OWNER:$tok" shopt -s nullglob for d in "$PARITY_STAGE_DIR"/*/; do [ -f "$d/copy-to-path" ] || continue arch="$(basename "$d")" copyto="$(cat "$d/copy-to-path")" "$copyto/bin/copy-to" --dest-creds "$creds" "docker://${imageName}:$VERSION-$arch" echo "Replayed ${imageName}:$VERSION-$arch" done ''; }; in archStageApps // archPublishApps // { "publish-index" = mkApp indexApp "Assemble the multi-arch index ${imageName}: from pushed per-arch tags (fail-closed)."; "publish" = mkApp publishAll "Publish all local arches + index; :latest = digest copy of :TAG (last mutation)."; "push-staged" = mkApp pushStaged "Replay staged ${imageName} arch closures from .parity-stage to the registry."; "verify-digest" = mkApp verifyDigest "Build each local arch image and print its OCI manifest digest (content-addressed parity check, cluster #195; no registry contact)."; }; # ========================================================================= # Shared push-staged for single-artifact archetypes (pypi/npm/generic): replay # the staged file from .parity-stage to the registry without rebuilding. # ========================================================================= mkPushStagedGeneric = { pname, arch, kind, version ? "", assetName ? "", registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; # The kind is known at Nix-eval time, so the kind-specific replay body is # selected here (not via a runtime `case` on a constant, which trips # shellcheck SC2194). kindBody = { pypi = '' whl="$(find "$d" -name '*.whl' | head -1)" sha="$(sha256sum "$whl" | cut -d' ' -f1)" http="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ -H "Authorization: token $tok" -F ':action=file_upload' \ -F 'protocol_version=1' -F "sha256_digest=$sha" \ -F "name=${pname}" -F "version=$VERSION" -F 'filetype=bdist_wheel' \ -F "content=@$whl" \ "https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi")" case "$http" in 200 | 201 | 409) echo "Replayed ${pname} $VERSION (HTTP $http)" ;; *) echo "BLOCKER(upload-failed): HTTP $http" >&2; exit 1 ;; esac ''; generic = '' http="$(curl -s -o /dev/null -w '%{http_code}' -X PUT \ -H "Authorization: token $tok" --upload-file "$d/${assetName}" \ "https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/generic/${pname}/$VERSION/${assetName}")" case "$http" in 201 | 409) echo "Replayed ${assetName} $VERSION (HTTP $http)" ;; *) echo "BLOCKER(upload-failed): HTTP $http" >&2; exit 1 ;; esac ''; npm = '' reg="https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/" ( cd "$d" set +e out="$(npm publish --registry="$reg" "--//$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/npm/:_authToken=$tok" 2>&1)" rc=$? set -e printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g" if [ "$rc" -ne 0 ]; then if printf '%s' "$out" | grep -qiE 'already exists|409|conflict'; then echo "Already published — idempotent." exit 0 fi exit "$rc" fi ) ''; }; app = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = baseInputs ++ lib.optional (kind == "npm") pkgs.nodejs; text = '' ${head} parity_parse_args "Replay the staged ${pname} ${arch} artifact (${kind}) from .parity-stage to the registry" "$@" d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ ! -d "$d" ]; then echo "BLOCKER(no-stage): nothing staged at $d — run 'nix run .#stage-${arch}' first." >&2 exit 1 fi VERSION="$(parity_derive_version ${lib.escapeShellArg version})" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would replay $d (${kind}) -> https://$PARITY_REGISTRY_HOST (${pname} $VERSION)" echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight ${kindBody.${kind}} ''; }; in mkApp app "Replay the staged ${pname} ${arch} artifact (${kind}) from .parity-stage (dry-run by default; --publish to push)."; # ========================================================================= # Attic closure (NO registry artifact). The ATTIC-CLOSURE archetype: the # "artifact" is a Nix closure pushed to an Attic binary cache, not a registry # blob/image. Modelled on the bespoke attic CI the caddy-with-replace / # woodpecker-peek / flake-hub repos run (`attic login ci $ATTIC_TOKEN` # then push/watch-store the built store paths). # # Apps: # stage- BUILD-parity: `nix build` the drvs into the local store and # record their out-paths in .parity-stage (NO push, no token). # publish- / publish stage + `attic login` + `attic push` the built # paths to the cache. DRY-RUN by default; --publish / PUBLISH=1 # actually pushes. Token from $ATTIC_TOKEN or `pass`. # push-staged replay the staged out-paths to the cache (no rebuild). # # TOKEN HYGIENE: $ATTIC_TOKEN is never echoed and no `set -x` is enabled; the # token is piped straight into `attic login` and any output is sed-redacted. # # Args: { drvs (list of derivations/attrs to build), # cache ? "attic-infra-cache-k3s-1", # endpoint ? "http://attic.infra-cache.k3s", # arch ? "x86_64-linux", # passEntry ? "infra/attic/token", ... } # ========================================================================= mkAtticClosurePublish = { drvs, cache ? "attic-infra-cache-k3s-1", endpoint ? "http://attic.infra-cache.k3s", arch ? "x86_64-linux", passEntry ? "infra/attic/token", }: let # Closure parity does not contact the Gitea registry; only the Attic cache # endpoint matters, but reuse the shared preamble for the stage-dir + arg # conventions. passEntry here points at the ATTIC token, not the registry. head = preamble { inherit passEntry; }; drvList = if builtins.isList drvs then drvs else [ drvs ]; drvRefs = lib.concatMapStringsSep " " (d: lib.escapeShellArg "${d}") drvList; atticInputs = baseInputs ++ [ pkgs.nix pkgs.attic-client ]; # Resolve the Attic token: $ATTIC_TOKEN, else `pass `. Never # echoed; prints on stdout for a caller to capture (the one blessed emit). atticTokenFn = '' parity_attic_token() { if [ -n "''${ATTIC_TOKEN:-}" ]; then printf '%s' "$ATTIC_TOKEN"; return 0; fi if command -v pass >/dev/null 2>&1; then t="$(pass show ${lib.escapeShellArg passEntry} 2>/dev/null || true)" if [ -n "$t" ]; then printf '%s' "$t"; return 0; fi fi echo "BLOCKER(empty-token): set \$ATTIC_TOKEN or store ${passEntry} in pass; refusing to push without credentials." >&2 return 1 } ''; # Log into the cache + push the given out-paths. Token piped, never echoed; # any attic output is sed-redacted so a token can never reach the log. atticPushFn = '' parity_attic_push() { tok="$(parity_attic_token)" || return 1 set +e out="$(attic login ci ${lib.escapeShellArg endpoint} "$tok" 2>&1)" rc=$? set -e printf '%s\n' "$out" | sed "s/$tok/***REDACTED***/g" if [ "$rc" -ne 0 ]; then return "$rc"; fi attic push ${lib.escapeShellArg cache} "$@" } ''; stage = pkgs.writeShellApplication { name = "stage-${arch}"; runtimeInputs = atticInputs; text = '' ${head} echo "→ staging attic closure (${arch}, ${toString (builtins.length drvList)} drv(s), no push)" d="$(parity_stage_reset ${lib.escapeShellArg arch})" : >"$d/out-paths" # The drv refs are a fixed Nix-eval list (may be a single literal path). # shellcheck disable=SC2043 for drv in ${drvRefs}; do # Realise the path into the local store (build-parity, no push). p="$(nix build --no-link --print-out-paths "$drv")" printf '%s\n' "$p" >>"$d/out-paths" echo " built: $p" done printf 'attic %s %s\n' ${lib.escapeShellArg cache} ${lib.escapeShellArg endpoint} >"$d/.parity-meta" echo " staged out-paths under $d" ''; }; publish = pkgs.writeShellApplication { name = "publish-${arch}"; runtimeInputs = atticInputs; text = '' ${head} ${atticTokenFn} ${atticPushFn} parity_parse_args "Build + push the ${arch} Nix closure to the Attic cache ${cache}" "$@" ${lib.getExe stage} d="$(parity_stage_path ${lib.escapeShellArg arch})" mapfile -t paths <"$d/out-paths" if [ "''${#paths[@]}" -eq 0 ]; then echo "BLOCKER(no-stage): no built out-paths under $d — run 'nix run .#stage-${arch}' first." >&2 exit 1 fi if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'attic push ${cache}' these paths to ${endpoint}:" for p in "''${paths[@]}"; do echo " - $p"; done echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_attic_push "''${paths[@]}" echo "Pushed ''${#paths[@]} path(s) to ${cache}" ''; }; pushStaged = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = atticInputs; text = '' ${head} ${atticTokenFn} ${atticPushFn} parity_parse_args "Replay the staged ${arch} closure out-paths to the Attic cache ${cache}" "$@" d="$(parity_stage_path ${lib.escapeShellArg arch})" if [ ! -f "$d/out-paths" ]; then echo "BLOCKER(no-stage): no staged out-paths at $d — run 'nix run .#stage-${arch}' first." >&2 exit 1 fi mapfile -t paths <"$d/out-paths" if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'attic push ${cache}' these staged paths to ${endpoint}:" for p in "''${paths[@]}"; do echo " - $p"; done echo "[dry-run] re-run with --publish / PUBLISH=1 to push." exit 0 fi parity_attic_push "''${paths[@]}" echo "Replayed ''${#paths[@]} path(s) to ${cache}" ''; }; in { "stage-${arch}" = mkApp stage "Build the ${arch} Nix closure into the local store + stage its out-paths (no push)."; "publish-${arch}" = mkApp publish "Build + push the ${arch} closure to the Attic cache ${cache} (dry-run by default; --publish to push)."; "publish" = mkApp publish "Push all locally-buildable arches to the Attic cache (${arch} only for this closure repo)."; "push-staged" = mkApp pushStaged "Replay the staged ${arch} closure out-paths to the Attic cache (dry-run by default; --publish to push)."; }; mkPushStagedHelm = { pname, ociRepo, registryHost ? "git.oleks.space", registryOwner ? "oleks", passEntry ? "infra/gitea/personal_access_token_packages_rw", }: let head = preamble { inherit registryHost registryOwner passEntry; }; app = pkgs.writeShellApplication { name = "push-staged"; runtimeInputs = baseInputs ++ [ pkgs.kubernetes-helm ]; text = '' ${head} parity_parse_args "Replay the staged ${pname} helm chart from .parity-stage to ${ociRepo}" "$@" d="$(parity_stage_path chart)" tgz="$(find "$d" -name '*.tgz' 2>/dev/null | head -1 || true)" if [ -z "$tgz" ]; then echo "BLOCKER(no-stage): no staged chart under $d — run 'nix run .#stage-chart' first." >&2 exit 1 fi if [ "$PARITY_PUBLISH" != "1" ]; then echo "[dry-run] would 'helm push $(basename "$tgz")' -> ${ociRepo}" exit 0 fi parity_devtag_guard tok="$(parity_resolve_token)" parity_registry_preflight echo "$tok" | helm registry login "$PARITY_REGISTRY_HOST" -u "$PARITY_REGISTRY_OWNER" --password-stdin >/dev/null 2>&1 helm push "$tgz" ${lib.escapeShellArg ociRepo} echo "Replayed chart $(basename "$tgz")" ''; }; in mkApp app "Replay the staged ${pname} helm chart from .parity-stage (dry-run by default; --publish to push)."; in { inherit mkPyPiWheelPublish mkPyPiWheelPublishMulti mkS390xNpmPublish mkS390xNpmPublishMulti mkGenericBinaryPublish mkNix2ContainerPublish mkGoBinaryPublish mkHelmPublish mkAtticClosurePublish ; # Shared building blocks, exposed for advanced/bespoke consumers. inherit mkApp shellLib; }