From cda7a190c040c60abf80c71f2095bf6e4d2f29aa Mon Sep 17 00:00:00 2001 From: Oleks Date: Tue, 2 Jun 2026 05:23:42 +0300 Subject: [PATCH] =?UTF-8?q?feat(pypi):=20mkPyPiWheelPublishMulti=20?= =?UTF-8?q?=E2=80=94=20publish=20all=20versions=20per=20tag=20(#197)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Single-version mkPyPiWheelPublish made consumers ship only the default version per tag. Add a multi-version builder that loops over a fixed {version,wheel} list (version parsed from the wheel filename, idempotent 409-skip), plus shared parity_pypi_post/parity_wheel_version helpers. --- CHANGELOG.md | 7 +++ ci/parity-lib.sh | 33 ++++++++++++++ flake.nix | 1 + lib/builders.nix | 111 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 152 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed23cc9..05e9563 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ semantic versioning; the version is a conceptual tag (no git tag is created). ## Unreleased +- **Feature: `mkPyPiWheelPublishMulti` (cluster #197).** A multi-version PyPI + builder that publishes a fixed list of `{ version; wheel; }` per tag instead of + just the default — the pre-parity behaviour several `*-s390x` repos rely on. + Each wheel's real version is read from its filename (PEP 427), so + stage/publish/push-staged need no side-channel map and a re-run is idempotent + (409-skip per version). Shared `parity_pypi_post` / `parity_wheel_version` + helpers added to `ci/parity-lib.sh`. First consumer: `numpy-s390x` (5 versions). - **Fix (safety): dev-tag guard was ineffective.** Every publish app body runs `VERSION="$(parity_derive_version )"` before `parity_devtag_guard`, so by the time the guard checked `$VERSION` it was always non-empty (the derived diff --git a/ci/parity-lib.sh b/ci/parity-lib.sh index 4b4fe37..39223fc 100644 --- a/ci/parity-lib.sh +++ b/ci/parity-lib.sh @@ -140,6 +140,39 @@ parity_registry_preflight() { fi } +# --------------------------------------------------------------------------- +# POST one wheel to the Gitea PyPI registry. Idempotent: 409 == already present. +# Args: . Never echoes the token. Used by +# both the single- and multi-version PyPI publish apps so the upload contract +# lives in one place. Returns non-zero (without aborting a caller loop) on a +# real upload failure so a multi-version run can attempt every version. +# --------------------------------------------------------------------------- +parity_pypi_post() { + local pname="$1" ver="$2" whl="$3" tok="$4" sha http + sha="$(sha256sum "$whl" | cut -d' ' -f1)" + http="$(curl -s -o /dev/null -w '%{http_code}' -X POST \ + -H "Authorization: token $tok" \ + -F ':action=file_upload' -F 'protocol_version=1' \ + -F "sha256_digest=$sha" -F "name=$pname" -F "version=$ver" \ + -F 'filetype=bdist_wheel' -F "content=@$whl" \ + "https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi")" + case "$http" in + 200 | 201 | 409) echo "Published $pname $ver (HTTP $http)" ;; + *) + echo "BLOCKER(upload-failed): $pname $ver POST returned HTTP $http" >&2 + return 1 + ;; + esac +} + +# --------------------------------------------------------------------------- +# Derive a wheel's version from its filename. PEP 427: the second dash-separated +# field of "----.whl" is the version. +# --------------------------------------------------------------------------- +parity_wheel_version() { + basename "$1" | cut -d- -f2 +} + # --------------------------------------------------------------------------- # Stage directory helpers. The convention: each staged artifact is written to # ${PARITY_STAGE_DIR}// alongside a one-line meta file recording the diff --git a/flake.nix b/flake.nix index de0d6c9..474414e 100644 --- a/flake.nix +++ b/flake.nix @@ -32,6 +32,7 @@ { mkParityBuilders = builders; mkPyPiWheelPublish = wrap "mkPyPiWheelPublish"; + mkPyPiWheelPublishMulti = wrap "mkPyPiWheelPublishMulti"; mkS390xNpmPublish = wrap "mkS390xNpmPublish"; mkGenericBinaryPublish = wrap "mkGenericBinaryPublish"; mkNix2ContainerPublish = wrap "mkNix2ContainerPublish"; diff --git a/lib/builders.nix b/lib/builders.nix index 87ecaf6..13d7aa9 100644 --- a/lib/builders.nix +++ b/lib/builders.nix @@ -153,6 +153,116 @@ let "push-staged" = pushStaged; }; + # ========================================================================= + # PyPI wheels, MULTI-version (cluster #197). Same archetype as the single + # builder, but publishes a fixed list of {version, wheel} per tag (the old + # per-repo CI behaviour several *-s390x repos rely on) instead of just the + # default version. The real version of each wheel is read from its filename + # (PEP 427), so stage/publish/push-staged need no side-channel map and a + # re-run is idempotent (409-skip per version). No dev-tag guard: the versions + # are an explicit fixed list of real releases, gated by the dry-run default + # (--publish / PUBLISH=1 is the deliberate intent), matching the old "publish + # all on a v* tag" pipeline. + # Args: { pname, versions = [ { version; wheel; } ... ], arch ? "s390x", registry* } + # ========================================================================= + mkPyPiWheelPublishMulti = + { + pname, + versions, + arch ? "s390x", + registryHost ? "git.oleks.space", + registryOwner ? "oleks", + passEntry ? "infra/gitea/personal_access_token_packages_rw", + }: + let + head = preamble { inherit registryHost registryOwner passEntry; }; + nVersions = builtins.length versions; + # One copy block per version's wheel (drv dir we glob, or explicit .whl). + stageCopies = lib.concatMapStringsSep "\n" (e: '' + src=${lib.escapeShellArg "${e.wheel}"} + if [ -d "$src" ]; then + # shellcheck disable=SC2044 + for w in $(find "$src" -name '*.whl'); do cp "$w" "$d/"; done + else + cp "$src" "$d/" + fi + '') versions; + stageText = '' + ${head} + echo "→ staging ${pname} (${toString nVersions} versions, ${arch}, no registry contact)" + d="$(parity_stage_reset ${lib.escapeShellArg arch})" + ${stageCopies} + n="$(find "$d" -name '*.whl' | wc -l)" + if [ "$n" -eq 0 ]; then echo "BLOCKER(no-wheel): no *.whl staged under $d" >&2; exit 1; fi + printf 'pypi-multi %s %s\n' ${lib.escapeShellArg pname} "$n" >"$d/.parity-meta" + echo " staged $n wheel(s) under $d" + ''; + stage = pkgs.writeShellApplication { + name = "stage-${arch}"; + runtimeInputs = baseInputs ++ [ pkgs.nix ]; + text = stageText; + }; + # Shared publish loop body, reused by publish and push-staged: POST every + # staged wheel, version parsed from its filename; aggregate failures. + uploadLoop = '' + tok="$(parity_resolve_token)" + parity_registry_preflight + rc=0 + for whl in "$d"/*.whl; do + ver="$(parity_wheel_version "$whl")" + parity_pypi_post ${lib.escapeShellArg pname} "$ver" "$whl" "$tok" || rc=1 + done + exit "$rc" + ''; + dryList = '' + echo "[dry-run] would POST these wheels -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi (name=${pname}):" + for whl in "$d"/*.whl; do echo " - $(basename "$whl") (version $(parity_wheel_version "$whl"))"; done + echo "[dry-run] re-run with --publish / PUBLISH=1 to upload." + ''; + publish = pkgs.writeShellApplication { + name = "publish-${arch}"; + runtimeInputs = baseInputs ++ [ pkgs.nix ]; + text = '' + ${head} + parity_parse_args "Build + publish all ${toString nVersions} ${pname} ${arch} wheels to the Gitea PyPI registry" "$@" + ${lib.getExe stage} + d="$(parity_stage_path ${lib.escapeShellArg arch})" + if [ "$PARITY_PUBLISH" != "1" ]; then + ${dryList} + exit 0 + fi + ${uploadLoop} + ''; + }; + pushStaged = pkgs.writeShellApplication { + name = "push-staged"; + runtimeInputs = baseInputs ++ [ pkgs.nix ]; + text = '' + ${head} + parity_parse_args "Replay the staged ${pname} ${arch} wheels from .parity-stage" "$@" + d="$(parity_stage_path ${lib.escapeShellArg arch})" + if ! find "$d" -name '*.whl' -print -quit | grep -q .; then + echo "BLOCKER(no-stage): no staged wheels under $d — run 'nix run .#stage-${arch}' first." >&2 + exit 1 + fi + if [ "$PARITY_PUBLISH" != "1" ]; then + ${dryList} + exit 0 + fi + ${uploadLoop} + ''; + }; + in + { + "stage-${arch}" = + mkApp stage "Stage all ${nVersions} ${pname} ${arch} wheels into .parity-stage (no registry contact)."; + "publish-${arch}" = + mkApp publish "Stage + publish all ${pname} ${arch} wheels (dry-run by default; --publish to push)."; + "publish" = mkApp publish "Publish all ${pname} ${arch} wheels (${arch} only for this PyPI repo)."; + "push-staged" = + mkApp pushStaged "Replay the staged ${pname} ${arch} wheels from .parity-stage (dry-run by default; --publish to push)."; + }; + # ========================================================================= # s390x npm native addon (single-arch). Stage a ready-to-publish package dir # (the .node + a generated package.json) then `npm publish`. @@ -745,6 +855,7 @@ in { inherit mkPyPiWheelPublish + mkPyPiWheelPublishMulti mkS390xNpmPublish mkGenericBinaryPublish mkNix2ContainerPublish