From 0a7d3b4830aadf8ffa2d4cb8d922b432ea2c677e Mon Sep 17 00:00:00 2001 From: schapper Date: Wed, 28 Jan 2026 15:21:42 -0800 Subject: [PATCH 1/3] Drop redundant filename precixes in GH action scripts --- .../{github-actions-check-pydantic.yaml => check-pydantic.yaml} | 0 ...test-docs-to-staging.yaml => copy-latest-docs-to-staging.yaml} | 0 ...-copy-pr-docs-to-staging.yaml => copy-pr-docs-to-staging.yaml} | 0 ...orce-change-type-label.yaml => enforce-change-type-label.yaml} | 0 ...ions-publish-docs-gh-pages.yaml => publish-docs-gh-pages.yaml} | 0 .../{github-actions-test-schema.yaml => test-schema.yaml} | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{github-actions-check-pydantic.yaml => check-pydantic.yaml} (100%) rename .github/workflows/{github-actions-copy-latest-docs-to-staging.yaml => copy-latest-docs-to-staging.yaml} (100%) rename .github/workflows/{github-actions-copy-pr-docs-to-staging.yaml => copy-pr-docs-to-staging.yaml} (100%) rename .github/workflows/{github-actions-enforce-change-type-label.yaml => enforce-change-type-label.yaml} (100%) rename .github/workflows/{github-actions-publish-docs-gh-pages.yaml => publish-docs-gh-pages.yaml} (100%) rename .github/workflows/{github-actions-test-schema.yaml => test-schema.yaml} (100%) diff --git a/.github/workflows/github-actions-check-pydantic.yaml b/.github/workflows/check-pydantic.yaml similarity index 100% rename from .github/workflows/github-actions-check-pydantic.yaml rename to .github/workflows/check-pydantic.yaml diff --git a/.github/workflows/github-actions-copy-latest-docs-to-staging.yaml b/.github/workflows/copy-latest-docs-to-staging.yaml similarity index 100% rename from .github/workflows/github-actions-copy-latest-docs-to-staging.yaml rename to .github/workflows/copy-latest-docs-to-staging.yaml diff --git a/.github/workflows/github-actions-copy-pr-docs-to-staging.yaml b/.github/workflows/copy-pr-docs-to-staging.yaml similarity index 100% rename from .github/workflows/github-actions-copy-pr-docs-to-staging.yaml rename to .github/workflows/copy-pr-docs-to-staging.yaml diff --git a/.github/workflows/github-actions-enforce-change-type-label.yaml b/.github/workflows/enforce-change-type-label.yaml similarity index 100% rename from .github/workflows/github-actions-enforce-change-type-label.yaml rename to .github/workflows/enforce-change-type-label.yaml diff --git a/.github/workflows/github-actions-publish-docs-gh-pages.yaml b/.github/workflows/publish-docs-gh-pages.yaml similarity index 100% rename from .github/workflows/github-actions-publish-docs-gh-pages.yaml rename to .github/workflows/publish-docs-gh-pages.yaml diff --git a/.github/workflows/github-actions-test-schema.yaml b/.github/workflows/test-schema.yaml similarity index 100% rename from .github/workflows/github-actions-test-schema.yaml rename to .github/workflows/test-schema.yaml From deced3ce8593232fe60f1d7401a729f6b48cac0c Mon Sep 17 00:00:00 2001 From: schapper Date: Thu, 29 Jan 2026 15:42:23 -0800 Subject: [PATCH 2/3] Verify Python package version changes via GitHub action https://github.com/OvertureMaps/schema-wg/issues/406 This commit adds a GitHub action that runs whenever an `__about__.py` or `pyproject.toml` file gets changed. WHAT THE ACTION DOES -------------------- - Identify any packages with changed version numbers. - Verify that that version number doesn't already exist in PyPI.* WHAT THE ACTION DOES NOT DO --------------------------- - It does not verify that the version number is a valid semantic version (semver) because this validation is now done by `$ make check` and `$ make check-versions` in `Makefile`. WHY WE WANT THIS BEHAVIOR ------------------------- - The purpose of this GitHub Actions workflow is to give us confidence that we are catching invalid version numbers, including attempts to re-publish to an existing version number, at the stage when the PR is drafted, rather than finding out later when it has been merged and our (soon to be written) publish automation kicks in. * For now, the PyPI repository is the Overture-run one. Later, when we're ready to release, we'll point ourselves at public PyPI. --- ...k-pydantic.yaml => check-python-code.yaml} | 2 +- .../check-python-package-versions.yaml | 15 ++ .../check-python-package-versions.yaml | 84 +++++++++++ .../reusable/get-code-artifact-index-url.yaml | 57 ++++++++ .github/workflows/scripts/package-versions.py | 134 ++++++++++++++++++ 5 files changed, 291 insertions(+), 1 deletion(-) rename .github/workflows/{check-pydantic.yaml => check-python-code.yaml} (95%) create mode 100644 .github/workflows/check-python-package-versions.yaml create mode 100644 .github/workflows/reusable/check-python-package-versions.yaml create mode 100644 .github/workflows/reusable/get-code-artifact-index-url.yaml create mode 100755 .github/workflows/scripts/package-versions.py diff --git a/.github/workflows/check-pydantic.yaml b/.github/workflows/check-python-code.yaml similarity index 95% rename from .github/workflows/check-pydantic.yaml rename to .github/workflows/check-python-code.yaml index 406d77f76..599e9ea09 100644 --- a/.github/workflows/check-pydantic.yaml +++ b/.github/workflows/check-python-code.yaml @@ -1,4 +1,4 @@ -name: Check Pydantic models +name: Check Python package code on: pull_request_target: diff --git a/.github/workflows/check-python-package-versions.yaml b/.github/workflows/check-python-package-versions.yaml new file mode 100644 index 000000000..4b0fdf41e --- /dev/null +++ b/.github/workflows/check-python-package-versions.yaml @@ -0,0 +1,15 @@ +name: Check Python package version numbers + +on: + pull_request_target: + paths: + - '**/pyproject.toml' + - 'packages/**/__about__.py' + +jobs: + check: + if: github.event.pull_request.head.repo.full_name == github.repository + uses: ./.github/workflows/reusable/check-python-package-versions.yaml' + with: + before_commit: ${{ github.event.pull_request.base.sha }} + after_commit: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/reusable/check-python-package-versions.yaml b/.github/workflows/reusable/check-python-package-versions.yaml new file mode 100644 index 000000000..7c88177b2 --- /dev/null +++ b/.github/workflows/reusable/check-python-package-versions.yaml @@ -0,0 +1,84 @@ +name: Check Python package versions + +on: + workflow_call: + inputs: + before_commit: + description: >- + The base Git commit to compare against, i.e., the base of the PR or the previous commit + in a push. + type: string + required: true + after_commit: + description: >- + The Git commit representing the head of the change to be checked, i.e. the head of the + PR or the latest commit in a push. + type: string + required: true + +jobs: + get-index-url: + uses: ./.github/workflows/reusable/get-code-artifact-index-url.yaml + + check-python-package-versions: + needs: get-index-url + runs-on: ubuntu-latest + steps: + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Check out code before change + uses: actions/checkout@v4 + with: + ref: ${{ inputs.before_commit }} + + - name: Capture package versions before change + run: uv run python scripts/package-versions.py collect > /tmp/package-versions-before.json + + - name: Check out code after change + uses: actions/checkout@v4 + with: + ref: ${{ inputs.after_commit }} + + - name: Capture package versions after change + run: uv run python scripts/package-versions.py collect > /tmp/package-versions-after.json + + - name: Compare package versions before and after change + run: | + uv run python scripts/package-versions.py compare \ + /tmp/package-versions-before.json \ + /tmp/package-versions-after.json \ + >/tmp/package-version-diff.json + + - name: Print changed versions + run: cat /tmp/package-version-diff.json + + - name: Fail if any of the new versions already exist in the repo + run: | + jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do + package=$(echo "$entry" | jq -r '.package') + after=$(echo "$entry" | jq -r '.after') + exit_code=0 + output=$(uv run pip download "${package}==${after}" --index-url "${{ needs.get-index-url.outputs.index_url }}simple/" --no-deps -d /tmp --quiet 2>&1) | exit_code=$? + if [[ $exit_code -eq 0 || ( + "${output,,}" != *"could not find a version"* && + "${output,,}" != *"no matching distributions"* + ) ]]; then + echo "Package ${package} version ${after} already exists in the repository. Failing the workflow." + echo " pip exit code: ${exit_code}." + echo " pip stderr: ${output}." + exit 1 + else + echo "Package ${package} version ${after} is new, as expected. Continuing." + fi + done diff --git a/.github/workflows/reusable/get-code-artifact-index-url.yaml b/.github/workflows/reusable/get-code-artifact-index-url.yaml new file mode 100644 index 000000000..e8dec13f0 --- /dev/null +++ b/.github/workflows/reusable/get-code-artifact-index-url.yaml @@ -0,0 +1,57 @@ +name: Get CodeArtifact Python package index URL + +on: + workflow_call: + inputs: + account_id: + description: The AWS account ID that owns the CodeArtifact domain + type: string + required: false + default: 505071440022 + aws_region: + description: The AWS region where the CodeArtifact repository is hosted + type: string + required: false + default: us-west-2 + role_name: + description: The name of the IAM role to assume for accessing CodeArtifact + type: string + required: false + default: GithubActions_Schema_CodeArtifact_ReadOnly + domain: + description: The CodeArtifact domain name + type: string + required: false + default: overture-pypi + repository: + description: The CodeArtifact repository name + type: string + required: false + default: overture + outputs: + index_url: + description: The CodeArtifact Python index URL + value: ${{ jobs.get-code-artifact-index-url.outputs.index_url }} + +jobs: + get-code-artifact-index-url: + runs-on: ubuntu-latest + outputs: + index_url: ${{ steps.get-code-artifact-index-url.outputs.index_url }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ inputs.aws_region }} + role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role_name }} + role-session-name: GitHubActions_${{github.workflow}}_${{github.job}}_${{github.run_id}} + + - name: Get CodeArtifact authorization token + id: get-code-artifact-auth-token + run: | + AUTH_TOKEN=$(aws codeartifact get-authorization-token \ + --domain ${{ inputs.domain }} \ + --domain-owner ${{ inputs.account_id }} \ + --query authorizationToken \ + --output text) + echo "https://aws:${AUTH_TOKEN}@$${{ inputs.domain }}-${{ inputs.account_id }}.d.codeartifact.${{ inputs.aws_region }}.amazonaws.com/simple/" >> $GITHUB_OUTPUT diff --git a/.github/workflows/scripts/package-versions.py b/.github/workflows/scripts/package-versions.py new file mode 100755 index 000000000..4507d2672 --- /dev/null +++ b/.github/workflows/scripts/package-versions.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +from importlib import metadata +from pathlib import Path +import json +import sys + + +def collect(): + """ + Collect Python package versions and print them as a JSON array. + + Form of the JSON array: + + [ {"package": "p1", "version": "v1"}, {"package": "p2", "version": "v2"}, ... ] + """ + packages_dir = Path("packages") + + packages = sorted( + d.name + for d in packages_dir.iterdir() + if d.is_dir() and d.name.startswith("overture-schema") + ) + + package_versions = [ + {"package": p, "version": metadata.version(p.replace("-", "."))} + for p in packages + ] + + print(json.dumps(package_versions, indent=2)) + + +def compare(before_file: str, after_file: str): + """ + Compare two JSON files containing package versions and print the packages that have a version + number change as a JSON array. + + Form of the JSON array: + + [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] + + Note that `before` will be `null` if the package did not exist in the "before" file, and `after` + will be `null` if the package did not exist in the "after" file. + """ + before_array = load(before_file) + after_array = load(after_file) + + before_dict = {item["package"]: item["version"] for item in before_array} + after_dict = {item["package"]: item["version"] for item in after_array} + + combined_keys = sorted(list(set(before_dict.keys()) | set(after_dict.keys()))) + + changed_packages = [] + for package in combined_keys: + before_version = before_dict.get(package) + after_version = after_dict.get(package) + if before_version != after_version: + changed_packages.append( + { + "package": package, + "before": before_version, + "after": after_version, + } + ) + + print(json.dumps(changed_packages, indent=2)) + + +def load(file_path: str) -> list[dict[str, str]]: + path = Path(file_path) + if not path.exists(): + print(f"File not found: {file_path}") + sys.exit(1) + + with path.open() as f: + value = json.load(f) + + if not isinstance(value, list): + print( + f"File {file_path} contains unexpected root value: expected a `list` but got value {repr(value)} of type `{type(value).__name__}`" + ) + sys.exit(1) + + for i, item in enumerate(value): + if not isinstance(item, dict): + print( + f"File {file_path} contains unexpected item at index {i}: expected `dict` but got value {repr(item)} of type `{type(item).__name__}`" + ) + sys.exit(1) + elif sorted(item.keys()) != ["package", "version"]: + print( + f"File {file_path} contains unexpected item at index {i}: expected keys `['package', 'version']` but got keys {sorted(item.keys())}" + ) + sys.exit(1) + elif not isinstance(item["package"], str): + print( + f"File {file_path} contains unexpected item at index {i}: expected `package` to be of type `str` but got value {repr(item['package'])} of type `{type(item['package']).__name__}`" + ) + sys.exit(1) + elif not isinstance(item["version"], str): + print( + f"File {file_path} contains unexpected item at index {i}: expected `version` to be of type `str` but got value {repr(item['version'])} of type `{type(item['version']).__name__}`" + ) + sys.exit(1) + + return value + + +def usage(): + print("Usage:") + print(f" ./{sys.argv[0]} collect") + print(f" ./{sys.argv[0]} compare BEFORE_FILE AFTER_FILE") + sys.exit(1) + + +def main(): + if len(sys.argv) < 2: + usage() + + cmd = sys.argv[1] + + if cmd == "collect": + collect() + elif cmd == "compare": + if len(sys.argv) != 4: + usage() + compare(sys.argv[2], sys.argv[3]) + else: + print(f"Unknown command: {cmd}") + usage() + + +if __name__ == "__main__": + main() From 1ba206f8db1ffcd5f4e7ac579a29b3d6f19dd2b8 Mon Sep 17 00:00:00 2001 From: schapper Date: Thu, 29 Jan 2026 15:49:58 -0800 Subject: [PATCH 3/3] Run `$ make check` validation on commit push, not just PR --- .github/workflows/check-python-code.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/check-python-code.yaml b/.github/workflows/check-python-code.yaml index 599e9ea09..97d37eb4b 100644 --- a/.github/workflows/check-python-code.yaml +++ b/.github/workflows/check-python-code.yaml @@ -6,6 +6,12 @@ on: - 'packages/**' - 'pyproject.toml' - 'uv.lock' + push: + branches: [main, dev] + paths: + - 'packages/**' + - 'pyproject.toml' + - 'uv.lock' jobs: check: