8000 Make release verification resilient to PyPI challenges by DaoyuanLi2816 · Pull Request #19 · DaoyuanLi2816/mini-verl · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,8 @@ jobs:
--artifact-root release-artifacts \
--repository https://github.com/DaoyuanLi2816/mini-verl \
--attempts 12 \
--delay 10
--delay 10 \
--allow-rendered-page-challenge

- name: Install and exercise the exact public version
env:
Expand Down
130 changes: 130 additions & 0 deletions .github/workflows/verify-existing-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
name: verify existing release

on:
workflow_dispatch:
inputs:
version:
description: Published version without the v prefix
required: true
type: string
source_run_id:
description: Tag release run that built and published the distributions
required: true
type: string

permissions:
actions: read
contents: read

jobs:
verify:
name: verify existing PyPI release and immutable artifacts
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0

- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"

- name: Bind the source run to the immutable release tag
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ inputs.version }}
SOURCE_RUN_ID: ${{ inputs.source_run_id }}
run: |
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
[[ "$SOURCE_RUN_ID" =~ ^[0-9]+$ ]]
TAG="v$VERSION"
TAG_COMMIT="$(git rev-list -n 1 "$TAG")"
RUN_COMMIT="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID" --jq .head_sha)"
RUN_EVENT="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID" --jq .event)"
RUN_REF="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID" --jq .head_branch)"
RUN_PATH="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$SOURCE_RUN_ID" --jq .path)"
test "$RUN_COMMIT" = "$TAG_COMMIT"
test "$RUN_EVENT" = "push"
test "$RUN_REF" = "$TAG"
test "$RUN_PATH" = ".github/workflows/release.yml"
echo "source run $SOURCE_RUN_ID is bound to $TAG at $TAG_COMMIT"

- name: Download the original release distributions
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-distributions
path: release-artifacts
repository: ${{ github.repository }}
run-id: ${{ inputs.source_run_id }}
github-token: ${{ github.token }}

- name: Verify public files, hashes, links and attestations
env:
VERSION: ${{ inputs.version }}
run: |
(cd release-artifacts && sha256sum --check SHA256SUMS)
python -m pip install --upgrade pip pypi-attestations
python scripts/verify_pypi_release.py \
--project miniverl \
--version "$VERSION" \
--sums release-artifacts/SHA256SUMS \
--artifact-root release-artifacts \
--repository https://github.com/DaoyuanLi2816/mini-verl \
--attempts 12 \
--delay 10 \
--allow-rendered-page-challenge

- name: Install and exercise the exact public version
env:
VERSION: ${{ inputs.version }}
run: |
python -m venv release-verify
release-verify/bin/python -m pip install --upgrade pip
release-verify/bin/python -m pip install \
--no-cache-dir --index-url https://pypi.org/simple \
"miniverl==$VERSION"
test "$(release-verify/bin/miniverl --version)" = "miniverl $VERSION"
release-verify/bin/miniverl doctor --json > doctor.json
release-verify/bin/python - <<'PY'
import json
import pathlib

payload = json.loads(pathlib.Path("doctor.json").read_text(encoding="utf-8"))
assert payload["verdict"]["core_commands"] is True
PY

create-github-release:
name: create GitHub Release from the verified original distributions
needs: verify
runs-on: ubuntu-latest
permissions:
actions: read
contents: write
steps:
- name: Download the verified original distributions
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-distributions
path: release-artifacts
repository: ${{ github.repository }}
run-id: ${{ inputs.source_run_id }}
github-token: ${{ github.token }}

- name: Recheck hashes and create the release
env:
GH_TOKEN: ${{ github.token }}
VERSION: ${{ inputs.version }}
run: |
TAG="v$VERSION"
(cd release-artifacts && sha256sum --check SHA256SUMS)
if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then
echo "GitHub Release $TAG already exists"
exit 0
fi
gh release create "$TAG" \
release-artifacts/dist/* \
release-artifacts/SHA256SUMS \
--repo "$GITHUB_REPOSITORY" \
--verify-tag \
--title "miniVERL $TAG" \
--generate-notes
48 changes: 36 additions & 12 deletions scripts/verify_pypi_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,13 @@ def _verify_integrity_metadata(*, project: str, version: str, filenames: list[st
raise RuntimeError(f"PyPI exposes no attestation for {filename}")


def _verify_long_description_links(*, project: str, version: str, repository: str) -> None:
def _verify_long_description_links(
*,
project: str,
version: str,
repository: str,
allow_rendered_page_challenge: bool = False,
) -> None:
metadata = _request_json(f"https://pypi.org/pypi/{project}/{version}/json")
description = str((metadata.get("info") or {}).get("description") or "")
tag = f"v{version}"
Expand All @@ -129,12 +135,6 @@ def _verify_long_description_links(*, project: str, version: str, repository: st
if f"{github}/blob/main/" in description or f"{raw}/main/" in description:
raise RuntimeError("stable PyPI long description still links repository files through main")

page = _request_text(f"https://pypi.org/project/{project}/{version}/")
if "<title>Client Challenge</title>" in page:
raise RuntimeError("PyPI rendered project page returned a client challenge")
absent_from_page = [url for url in required_file_urls if url not in page]
if absent_from_page:
raise RuntimeError(f"rendered PyPI project page is missing links: {absent_from_page}")
image_matches = re.findall(
rf"!\[([^\]]+)\]\({re.escape(required_image_url)}\)",
description,
Expand All @@ -143,11 +143,6 @@ def _verify_long_description_links(*, project: str, version: str, repository: st
raise RuntimeError(
"PyPI long description does not use the release-pinned banner as an image"
)
if "pypi-camo." not in page or not all(
html.escape(alt, quote=True) in page for alt in image_matches
):
raise RuntimeError("rendered PyPI project page is missing the proxied release banner")

project_links = sorted(
set(
re.findall(
Expand All @@ -163,6 +158,24 @@ def _verify_long_description_links(*, project: str, version: str, repository: st
for url in project_links:
_request_text(url)

page = _request_text(f"https://pypi.org/project/{project}/{version}/")
if "<title>Client Challenge</title>" in page:
if allow_rendered_page_challenge:
print(
"warning: PyPI rendered project page returned a client challenge; "
"release-pinned description links and their targets were verified through "
"the public JSON API, but rendered-page inspection requires a browser"
)
return
raise RuntimeError("PyPI rendered project page returned a client challenge")
absent_from_page = [url for url in required_file_urls if url not in page]
if absent_from_page:
raise RuntimeError(f"rendered PyPI project page is missing links: {absent_from_page}")
if "pypi-camo." not in page or not all(
html.escape(alt, quote=True) in page for alt in image_matches
):
raise RuntimeError("rendered PyPI project page is missing the proxied release banner")


def verify_release(
*,
Expand All @@ -173,6 +186,7 @@ def verify_release(
repository: str,
attempts: int,
delay: float,
allow_rendered_page_challenge: bool,
) -> None:
expected = _expected_hashes(sums, artifact_root)
last_error: Exception | None = None
Expand All @@ -189,6 +203,7 @@ def verify_release(
project=project,
version=version,
repository=repository,
allow_rendered_page_challenge=allow_rendered_page_challenge,
)
break
except (RuntimeError, urllib.error.HTTPError, urllib.error.URLError) as exc:
Expand Down Expand Up @@ -222,6 +237,14 @@ def main() -> None:
parser.add_argument("--repository", required=True)
parser.add_argument("--attempts", type=int, default=12)
parser.add_argument("--delay", type=float, default=10.0)
parser.add_argument(
"--allow-rendered-page-challenge",
action="store_true",
help=(
"accept PyPI's browser challenge only after public metadata and every "
"release-pinned repository target have been verified"
),
)
args = parser.parse_args()
if args.attempts < 1:
parser.error("--attempts must be at least 1")
Expand All @@ -233,6 +256,7 @@ def main() -> None:
repository=args.repository,
attempts=args.attempts,
delay=args.delay,
allow_rendered_page_challenge=args.allow_rendered_page_challenge,
)


Expand Down
45 changes: 45 additions & 0 deletions tests/unit/test_pypi_release_verifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,51 @@ def test_long_description_retries_a_pypi_client_challenge(monkeypatch) -> None:
)


def test_long_description_can_defer_a_client_challenge_to_browser_inspection(
monkeypatch,
capsys,
) -> None:
verifier = _module()
repository = "https://github.com/DaoyuanLi2816/mini-verl"
tag = "v0.2.4"
paths = (
"docs/single-gpu-guide.md",
"recipes/qwen_consumer_gpu_calc.yaml",
"benchmarks/results/gpu-calc-hard-equal-update-v2.json",
"CHANGELOG.md",
"CITATION.cff",
"LICENSE",
"CONTRIBUTING.md",
"SECURITY.md",
)
banner = f"https://raw.githubusercontent.com/DaoyuanLi2816/mini-verl/{tag}/docs/banner.svg"
links = [f"{repository}/blob/{tag}/{path}" for path in paths]
description = "\n".join([*links, f"![release banner]({banner})"])
monkeypatch.setattr(
verifier,
"_request_json",
lambda *_args, **_kwargs: {"info": {"description": description}},
)
requested: list[str] = []

def request_text(url: str) -> str:
requested.append(url)
if url == "https://pypi.org/project/miniverl/0.2.4/":
return "<title>Client Challenge</title>"
return "ok"

monkeypatch.setattr(verifier, "_request_text", request_text)
verifier._verify_long_description_links(
project="miniverl",
version="0.2.4",
repository=repository,
allow_rendered_page_challenge=True,
)

assert {*links, banner} <= set(requested)
assert "requires a browser" in capsys.readouterr().out


def test_long_description_rejects_main_drift_for_a_stable_release(monkeypatch) -> None:
verifier = _module()
monkeypatch.setattr(
Expand Down
0