feat(pypi): mkPyPiWheelPublishMulti — publish all versions per tag (#197)
Single-version mkPyPiWheelPublish made consumers ship only the default
version per tag. Add a multi-version builder that loops over a fixed
{version,wheel} list (version parsed from the wheel filename, idempotent
409-skip), plus shared parity_pypi_post/parity_wheel_version helpers.
This commit is contained in:
Oleks
@@ -7,6 +7,13 @@ semantic versioning; the version is a conceptual tag (no git tag is created).
|
||||
|
||||
## Unreleased
|
||||
|
||||
- **Feature: `mkPyPiWheelPublishMulti` (cluster #197).** A multi-version PyPI
|
||||
builder that publishes a fixed list of `{ version; wheel; }` per tag instead of
|
||||
just the default — the pre-parity behaviour several `*-s390x` repos rely on.
|
||||
Each wheel's real version is read from its filename (PEP 427), so
|
||||
stage/publish/push-staged need no side-channel map and a re-run is idempotent
|
||||
(409-skip per version). Shared `parity_pypi_post` / `parity_wheel_version`
|
||||
helpers added to `ci/parity-lib.sh`. First consumer: `numpy-s390x` (5 versions).
|
||||
- **Fix (safety): dev-tag guard was ineffective.** Every publish app body runs
|
||||
`VERSION="$(parity_derive_version <default>)"` before `parity_devtag_guard`, so
|
||||
by the time the guard checked `$VERSION` it was always non-empty (the derived
|
||||
|
||||
@@ -140,6 +140,39 @@ parity_registry_preflight() {
|
||||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# POST one wheel to the Gitea PyPI registry. Idempotent: 409 == already present.
|
||||
# Args: <pname> <version> <wheel-path> <token>. Never echoes the token. Used by
|
||||
# both the single- and multi-version PyPI publish apps so the upload contract
|
||||
# lives in one place. Returns non-zero (without aborting a caller loop) on a
|
||||
# real upload failure so a multi-version run can attempt every version.
|
||||
# ---------------------------------------------------------------------------
|
||||
parity_pypi_post() {
|
||||
local pname="$1" ver="$2" whl="$3" tok="$4" sha http
|
||||
sha="$(sha256sum "$whl" | cut -d' ' -f1)"
|
||||
http="$(curl -s -o /dev/null -w '%{http_code}' -X POST \
|
||||
-H "Authorization: token $tok" \
|
||||
-F ':action=file_upload' -F 'protocol_version=1' \
|
||||
-F "sha256_digest=$sha" -F "name=$pname" -F "version=$ver" \
|
||||
-F 'filetype=bdist_wheel' -F "content=@$whl" \
|
||||
"https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi")"
|
||||
case "$http" in
|
||||
200 | 201 | 409) echo "Published $pname $ver (HTTP $http)" ;;
|
||||
*)
|
||||
echo "BLOCKER(upload-failed): $pname $ver POST returned HTTP $http" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Derive a wheel's version from its filename. PEP 427: the second dash-separated
|
||||
# field of "<dist>-<version>-<pytag>-<abitag>-<plat>.whl" is the version.
|
||||
# ---------------------------------------------------------------------------
|
||||
parity_wheel_version() {
|
||||
basename "$1" | cut -d- -f2
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage directory helpers. The convention: each staged artifact is written to
|
||||
# ${PARITY_STAGE_DIR}/<arch>/ alongside a one-line meta file recording the
|
||||
|
||||
@@ -32,6 +32,7 @@
|
||||
{
|
||||
mkParityBuilders = builders;
|
||||
mkPyPiWheelPublish = wrap "mkPyPiWheelPublish";
|
||||
mkPyPiWheelPublishMulti = wrap "mkPyPiWheelPublishMulti";
|
||||
mkS390xNpmPublish = wrap "mkS390xNpmPublish";
|
||||
mkGenericBinaryPublish = wrap "mkGenericBinaryPublish";
|
||||
mkNix2ContainerPublish = wrap "mkNix2ContainerPublish";
|
||||
|
||||
@@ -153,6 +153,116 @@ let
|
||||
"push-staged" = pushStaged;
|
||||
};
|
||||
|
||||
# =========================================================================
|
||||
# PyPI wheels, MULTI-version (cluster #197). Same archetype as the single
|
||||
# builder, but publishes a fixed list of {version, wheel} per tag (the old
|
||||
# per-repo CI behaviour several *-s390x repos rely on) instead of just the
|
||||
# default version. The real version of each wheel is read from its filename
|
||||
# (PEP 427), so stage/publish/push-staged need no side-channel map and a
|
||||
# re-run is idempotent (409-skip per version). No dev-tag guard: the versions
|
||||
# are an explicit fixed list of real releases, gated by the dry-run default
|
||||
# (--publish / PUBLISH=1 is the deliberate intent), matching the old "publish
|
||||
# all on a v* tag" pipeline.
|
||||
# Args: { pname, versions = [ { version; wheel; } ... ], arch ? "s390x", registry* }
|
||||
# =========================================================================
|
||||
mkPyPiWheelPublishMulti =
|
||||
{
|
||||
pname,
|
||||
versions,
|
||||
arch ? "s390x",
|
||||
registryHost ? "git.oleks.space",
|
||||
registryOwner ? "oleks",
|
||||
passEntry ? "infra/gitea/personal_access_token_packages_rw",
|
||||
}:
|
||||
let
|
||||
head = preamble { inherit registryHost registryOwner passEntry; };
|
||||
nVersions = builtins.length versions;
|
||||
# One copy block per version's wheel (drv dir we glob, or explicit .whl).
|
||||
stageCopies = lib.concatMapStringsSep "\n" (e: ''
|
||||
src=${lib.escapeShellArg "${e.wheel}"}
|
||||
if [ -d "$src" ]; then
|
||||
# shellcheck disable=SC2044
|
||||
for w in $(find "$src" -name '*.whl'); do cp "$w" "$d/"; done
|
||||
else
|
||||
cp "$src" "$d/"
|
||||
fi
|
||||
'') versions;
|
||||
stageText = ''
|
||||
${head}
|
||||
echo "→ staging ${pname} (${toString nVersions} versions, ${arch}, no registry contact)"
|
||||
d="$(parity_stage_reset ${lib.escapeShellArg arch})"
|
||||
${stageCopies}
|
||||
n="$(find "$d" -name '*.whl' | wc -l)"
|
||||
if [ "$n" -eq 0 ]; then echo "BLOCKER(no-wheel): no *.whl staged under $d" >&2; exit 1; fi
|
||||
printf 'pypi-multi %s %s\n' ${lib.escapeShellArg pname} "$n" >"$d/.parity-meta"
|
||||
echo " staged $n wheel(s) under $d"
|
||||
'';
|
||||
stage = pkgs.writeShellApplication {
|
||||
name = "stage-${arch}";
|
||||
runtimeInputs = baseInputs ++ [ pkgs.nix ];
|
||||
text = stageText;
|
||||
};
|
||||
# Shared publish loop body, reused by publish and push-staged: POST every
|
||||
# staged wheel, version parsed from its filename; aggregate failures.
|
||||
uploadLoop = ''
|
||||
tok="$(parity_resolve_token)"
|
||||
parity_registry_preflight
|
||||
rc=0
|
||||
for whl in "$d"/*.whl; do
|
||||
ver="$(parity_wheel_version "$whl")"
|
||||
parity_pypi_post ${lib.escapeShellArg pname} "$ver" "$whl" "$tok" || rc=1
|
||||
done
|
||||
exit "$rc"
|
||||
'';
|
||||
dryList = ''
|
||||
echo "[dry-run] would POST these wheels -> https://$PARITY_REGISTRY_HOST/api/packages/$PARITY_REGISTRY_OWNER/pypi (name=${pname}):"
|
||||
for whl in "$d"/*.whl; do echo " - $(basename "$whl") (version $(parity_wheel_version "$whl"))"; done
|
||||
echo "[dry-run] re-run with --publish / PUBLISH=1 to upload."
|
||||
'';
|
||||
publish = pkgs.writeShellApplication {
|
||||
name = "publish-${arch}";
|
||||
runtimeInputs = baseInputs ++ [ pkgs.nix ];
|
||||
text = ''
|
||||
${head}
|
||||
parity_parse_args "Build + publish all ${toString nVersions} ${pname} ${arch} wheels to the Gitea PyPI registry" "$@"
|
||||
${lib.getExe stage}
|
||||
d="$(parity_stage_path ${lib.escapeShellArg arch})"
|
||||
if [ "$PARITY_PUBLISH" != "1" ]; then
|
||||
${dryList}
|
||||
exit 0
|
||||
fi
|
||||
${uploadLoop}
|
||||
'';
|
||||
};
|
||||
pushStaged = pkgs.writeShellApplication {
|
||||
name = "push-staged";
|
||||
runtimeInputs = baseInputs ++ [ pkgs.nix ];
|
||||
text = ''
|
||||
${head}
|
||||
parity_parse_args "Replay the staged ${pname} ${arch} wheels from .parity-stage" "$@"
|
||||
d="$(parity_stage_path ${lib.escapeShellArg arch})"
|
||||
if ! find "$d" -name '*.whl' -print -quit | grep -q .; then
|
||||
echo "BLOCKER(no-stage): no staged wheels under $d — run 'nix run .#stage-${arch}' first." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PARITY_PUBLISH" != "1" ]; then
|
||||
${dryList}
|
||||
exit 0
|
||||
fi
|
||||
${uploadLoop}
|
||||
'';
|
||||
};
|
||||
in
|
||||
{
|
||||
"stage-${arch}" =
|
||||
mkApp stage "Stage all ${nVersions} ${pname} ${arch} wheels into .parity-stage (no registry contact).";
|
||||
"publish-${arch}" =
|
||||
mkApp publish "Stage + publish all ${pname} ${arch} wheels (dry-run by default; --publish to push).";
|
||||
"publish" = mkApp publish "Publish all ${pname} ${arch} wheels (${arch} only for this PyPI repo).";
|
||||
"push-staged" =
|
||||
mkApp pushStaged "Replay the staged ${pname} ${arch} wheels from .parity-stage (dry-run by default; --publish to push).";
|
||||
};
|
||||
|
||||
# =========================================================================
|
||||
# s390x npm native addon (single-arch). Stage a ready-to-publish package dir
|
||||
# (the .node + a generated package.json) then `npm publish`.
|
||||
@@ -745,6 +855,7 @@ in
|
||||
{
|
||||
inherit
|
||||
mkPyPiWheelPublish
|
||||
mkPyPiWheelPublishMulti
|
||||
mkS390xNpmPublish
|
||||
mkGenericBinaryPublish
|
||||
mkNix2ContainerPublish
|
||||
|
||||
Reference in New Issue
Block a user