build: increase upload concurrency to 8 #207
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| on: | |
| schedule: | |
| # update the pointers once a week | |
| # https://crontab.guru/once-a-week | |
| - cron: "0 0 * * 0" | |
| push: | |
| branches: | |
| - 'main' | |
| env: | |
| TARGET_PLATFORMS: linux/amd64,linux/arm64 | |
| jobs: | |
| build_images: | |
| name: "Create runtime and buildroot OCI Images" | |
| services: | |
| registry: | |
| image: registry:3 | |
| ports: | |
| - 5000:5000 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python: | |
| - '3.14' | |
| - '3.13' | |
| - '3.12' | |
| - '3.11' | |
| - '3.10' | |
| - '3.9' | |
| - '3.8' | |
| alpine: | |
| - '3.20' | |
| - '3.21' | |
| - '3.22' | |
| os: | |
| - 'ubuntu-latest' | |
| exclude: | |
| # No tag | |
| - python: '3.8' | |
| alpine: '3.21' | |
| - python: '3.8' | |
| alpine: '3.22' | |
| - python: '3.14' | |
| alpine: '3.20' | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v4 | |
| - | |
| name: Setup | |
| run: | | |
| sudo apt-get -y install skopeo | |
| - | |
| name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| - | |
| name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| with: | |
| driver-opts: network=host | |
| - | |
| id: image_env | |
| run: | | |
| . ./env.sh \ | |
| '${{ matrix.alpine }}' \ | |
| '${{ matrix.python }}' \ | |
| '${{ github.repository_owner }}' \ | |
| 'localhost:5000' | |
| docker pull "${SOURCE_IMAGE}" || true | |
| echo "IMAGE_HOME=$(mktemp -d)" >> "$GITHUB_OUTPUT" | |
| echo ALPINE_VERSION="${ALPINE_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo PYTHON_VERSION="${PYTHON_VERSION}" >> "$GITHUB_OUTPUT" | |
| echo SOURCE_IMAGE="${SOURCE_IMAGE}" >> "$GITHUB_OUTPUT" | |
| echo IMAGE_TAG="${IMAGE_TAG}" >> "$GITHUB_OUTPUT" | |
| echo IMAGE_BUILDROOT_TAG="${IMAGE_TAG}-buildroot" >> "$GITHUB_OUTPUT" | |
| echo IMAGE_TAG_SAFE="$(echo "$IMAGE_TAG" | base64 -w 0 )" >> "$GITHUB_OUTPUT" | |
| echo IMAGE_BUILDROOT_TAG_SAFE="$(echo "${IMAGE_TAG}-buildroot" | base64 -w 0 )" >> "$GITHUB_OUTPUT" | |
| echo REPOSITORY="${REPOSITORY}" >> "$GITHUB_OUTPUT" | |
| echo REPOSITORY_SAFE="$(echo "${REPOSITORY}" | base64 -w 0 )" >> "$GITHUB_OUTPUT" | |
| echo BASE_IMAGE_DIGEST="$(digest_of "$SOURCE_IMAGE")" >> "$GITHUB_OUTPUT" | |
| echo 'IMAGE_DESCRIPTION=${{ github.event.repository.description }}. See ${{ github.server_url }}/${{ github.repository }} for more info.' >> "$GITHUB_OUTPUT" | |
| - | |
| name: Create Buildroot | |
| uses: docker/build-push-action@v6 | |
| id: buildroot | |
| with: | |
| push: true | |
| platforms: ${{ env.TARGET_PLATFORMS }} | |
| context: "." | |
| file: Dockerfile.alpine | |
| sbom: true | |
| provenance: mode=max | |
| target: buildroot | |
| cache-to: | | |
| type=gha,mode=max | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=${{ steps.image_env.outputs.IMAGE_TAG }}-buildroot | |
| type=registry,ref=${{ steps.image_env.outputs.SOURCE_IMAGE }}@${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| build-args: | | |
| ALPINE_VERSION=${{ steps.image_env.outputs.ALPINE_VERSION }} | |
| BASE_IMAGE_DIGEST=${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| PYTHON_VERSION=${{ steps.image_env.outputs.PYTHON_VERSION }} | |
| SOURCE_IMAGE=${{ steps.image_env.outputs.SOURCE_IMAGE }} | |
| BUILD_ROOT=/d | |
| tags: "${{ steps.image_env.outputs.IMAGE_TAG }}-buildroot" | |
| outputs: type=oci,dest=${{ steps.image_env.outputs.IMAGE_HOME }}/image-${{ steps.image_env.outputs.IMAGE_BUILDROOT_TAG_SAFE }}.tar | |
| - | |
| name: Create Image | |
| id: runtime | |
| uses: docker/build-push-action@v6 | |
| env: | |
| SOURCE_DATE_EPOCH: 0 | |
| with: | |
| push: true | |
| context: "." | |
| platforms: ${{ env.TARGET_PLATFORMS }} | |
| file: Dockerfile.alpine | |
| cache-to: | | |
| type=gha,mode=max | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=${{ steps.image_env.outputs.IMAGE_TAG }} | |
| type=registry,ref=${{ steps.image_env.outputs.IMAGE_TAG }}-buildroot | |
| type=registry,ref=${{ steps.image_env.outputs.SOURCE_IMAGE }}@${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| build-args: | | |
| ALPINE_VERSION=${{ steps.image_env.outputs.ALPINE_VERSION }} | |
| BASE_IMAGE_DIGEST=${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| PYTHON_VERSION=${{ steps.image_env.outputs.PYTHON_VERSION }} | |
| SOURCE_IMAGE=${{ steps.image_env.outputs.SOURCE_IMAGE }} | |
| BUILD_ROOT=/d | |
| tags: "${{ steps.image_env.outputs.IMAGE_TAG }}" | |
| outputs: type=oci,dest=${{ steps.image_env.outputs.IMAGE_HOME }}/image-${{ steps.image_env.outputs.IMAGE_TAG_SAFE }}.tar | |
| labels: ${{steps.image_env.outputs.IMAGE_LABELS}} | |
| sbom: true | |
| provenance: mode=max | |
| annotations: | | |
| index,manifest:org.opencontainers.image.authors=distroless-python image developers <autumn.jolitz+distroless-python@gmail.com> | |
| index,manifest:org.opencontainers.image.source=https://github.com/autumnjolitz/distroless-python | |
| index,manifest:org.opencontainers.image.title=distroless-python${{ steps.image_env.outputs.PYTHON_VERSION }}-alpine${{ steps.image_env.outputs.ALPINE_VERSION }} | |
| index,manifest:org.opencontainers.image.description=${{ steps.image_env.outputs.IMAGE_DESCRIPTION }} | |
| index,manifest:org.opencontainers.image.base.digest=${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| index,manifest:org.opencontainers.image.base.name=${{ steps.image_env.outputs.SOURCE_IMAGE }} | |
| index,manifest:distroless.python-version=${{ steps.image_env.outputs.PYTHON_VERSION }} | |
| index,manifest:distroless.alpine-version=${{ steps.image_env.outputs.ALPINE_VERSION }} | |
| index,manifest:distroless.base-image=alpine${{ steps.image_env.outputs.ALPINE_VERSION }} | |
| - | |
| name: examples/simple-flask | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: "examples/simple-flask" | |
| platforms: ${{ env.TARGET_PLATFORMS }} | |
| cache-from: | | |
| type=gha | |
| type=registry,ref=${{ steps.image_env.outputs.IMAGE_TAG }} | |
| type=registry,ref=${{ steps.image_env.outputs.IMAGE_TAG }}-buildroot | |
| type=registry,ref=${{ steps.image_env.outputs.SOURCE_IMAGE }}@${{ steps.image_env.outputs.BASE_IMAGE_DIGEST }} | |
| build-args: | | |
| SOURCE_IMAGE=${{ steps.image_env.outputs.IMAGE_TAG }} | |
| tags: "${{ steps.image_env.outputs.IMAGE_TAG }}-example1-amd64" | |
| outputs: type=oci,dest=${{ steps.image_env.outputs.IMAGE_HOME }}/example1-${{ steps.image_env.outputs.IMAGE_TAG_SAFE }}.tar | |
| - | |
| id: post_build | |
| run: | | |
| IMAGE_DIGEST='${{steps.runtime.outputs.digest}}' | |
| IMAGE_BUILDROOT_DIGEST='${{steps.buildroot.outputs.digest}}' | |
| echo 'IMAGE_DIGEST='$IMAGE_DIGEST >> "GITHUB_OUTPUT" | |
| echo 'IMAGE_BUILDROOT_DIGEST='$IMAGE_BUILDROOT_DIGEST >> "GITHUB_OUTPUT" | |
| echo '${{ steps.image_env.outputs.IMAGE_TAG }}@'"$IMAGE_DIGEST" | tee -a ${{ steps.image_env.outputs.IMAGE_HOME }}/manifest.txt | |
| echo '${{ steps.image_env.outputs.IMAGE_BUILDROOT_TAG }}@'"$IMAGE_BUILDROOT_DIGEST" | tee -a ${{ steps.image_env.outputs.IMAGE_HOME }}/manifest.txt | |
| ls ${{ steps.image_env.outputs.IMAGE_HOME }} | |
| - name: upload build | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| if-no-files-found: error | |
| retention-days: 1 | |
| name: images-alpine${{steps.image_env.outputs.ALPINE_VERSION}}-python${{steps.image_env.outputs.PYTHON_VERSION}} | |
| path: | | |
| ${{ steps.image_env.outputs.IMAGE_HOME }}/image-${{ steps.image_env.outputs.IMAGE_TAG_SAFE }}.tar | |
| ${{ steps.image_env.outputs.IMAGE_HOME }}/image-${{ steps.image_env.outputs.IMAGE_BUILDROOT_TAG_SAFE }}.tar | |
| - name: upload build info | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| if-no-files-found: error | |
| retention-days: 1 | |
| name: metadata-alpine${{steps.image_env.outputs.ALPINE_VERSION}}-python${{steps.image_env.outputs.PYTHON_VERSION}} | |
| path: | | |
| ${{ steps.image_env.outputs.IMAGE_HOME }}/*.txt | |
| update-dockerhub-desc: | |
| name: "Upload and update metadata" | |
| needs: [build_images] | |
| runs-on: "ubuntu-latest" | |
| permissions: | |
| packages: write | |
| steps: | |
| - | |
| name: Checkout | |
| uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.12' | |
| - | |
| name: install dependencies | |
| run: | | |
| sudo apt-get -y install skopeo | |
| python -m pip install jinplate | |
| - | |
| name: fetch images | |
| uses: actions/download-artifact@v5 | |
| with: | |
| pattern: images-* | |
| path: dist-images | |
| merge-multiple: true | |
| - | |
| name: prep files | |
| run: | | |
| printf "Images:\n\n" >> "$GITHUB_STEP_SUMMARY" | |
| for filename in $(find dist-images -type f -name "*.tar" -print) | |
| do | |
| image_uri="$(basename -s .tar $filename | sed 's/image-//g' | base64 -d)" | |
| for repository in docker.io ghcr.io | |
| do | |
| new_image_uri=$(echo $image_uri | sed 's|localhost:5000|'"$repository"'|g') | |
| new_filename="dist-images/image-$(echo $new_image_uri | base64 -w 0).tar" | |
| cp ${filename} ${new_filename} | |
| echo '* `'${new_filename}'` (`'${new_image_uri}'`)' >> "$GITHUB_STEP_SUMMARY" | |
| done | |
| rm $filename | |
| done | |
| - name: login to registry | |
| env: | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| printf "${GITHUB_TOKEN}\n" | skopeo login --password-stdin --username ${{ github.repository_owner }} ghcr.io | |
| printf "${DOCKERHUB_TOKEN}\n" | skopeo login --password-stdin --username ${{ github.repository_owner }} docker.io | |
| - | |
| name: upload to registry | |
| run: | | |
| upload_image () { | |
| local manifest_file= | |
| local packages_file= | |
| local image_digest= | |
| local filename="$1" | |
| shift | |
| local image_uri="$1" | |
| shift | |
| if [ x$1 != x ]; then | |
| manifest_file="$1" | |
| shift | |
| fi | |
| if [ x$1 != x ]; then | |
| packages_file="$1" | |
| shift | |
| fi | |
| if ! case "$filename" in /*) true ;; *) false ;; esac ; then | |
| filename="$(realpath $filename)" | |
| fi | |
| local START_TIME=$(date +%s) | |
| local T="$(mktemp)" | |
| local rc=0 | |
| >&2 echo "uploading ${image_uri}" | |
| skopeo copy \ | |
| --quiet \ | |
| --retry-times 32 \ | |
| --multi-arch all \ | |
| --dest-precompute-digests \ | |
| --digestfile "$T" \ | |
| oci-archive://$filename "docker://$image_uri" || rc=$? | |
| if [ $rc -ne 0 ]; then | |
| echo 'failed to upload '$image_uri' (return code: '$rc')!' >>"$GITHUB_STEP_SUMMARY" | |
| rm -f $T | |
| return $rc | |
| fi | |
| image_digest="$(cat $T)" | |
| rm -f $T | |
| if [ x$manifest_file != x ]; then | |
| >>"$manifest_file" echo "${image_uri}@${image_digest}" | |
| fi | |
| printf '* '"${image_uri}@${image_digest}"'\n' >>"$GITHUB_STEP_SUMMARY" | |
| >&2 echo "uploaded ${image_uri} in $(expr $(date +%s) - $START_TIME) seconds" | |
| echo "${image_uri}@${image_digest}" | |
| if case $image_uri in 'ghcr.io'*) true ;; *) false ;; esac ; then | |
| ghcr_artifact_url "${image_uri}" "${packages_file}" | |
| fi | |
| } | |
| ghcr_artifact_url () { | |
| local output=- | |
| local image_tag="$1" | |
| shift | |
| if ! case $image_tag in 'ghcr.io'*) true ;; *) false ;; esac ; then | |
| >&2 echo 'image '"$image_tag"' is not a ghcr.io one!' | |
| return 1 | |
| fi | |
| if [ x"$1" != x ]; then | |
| output="$1" | |
| shift | |
| fi | |
| local tag=$(echo "${image_tag}" | rev | cut -d: -f1 | rev) | |
| local package_name=$(echo "${image_tag}" | rev | cut -d: -f2- | cut -d/ -f1 | rev) | |
| local repo_name='${{ github.repository }}' | |
| local owner='${{ github.repository_owner }}' | |
| local version_id=$(curl -s -H "Authorization: Bearer "'${{ secrets.GITHUB_TOKEN }}' \ | |
| "https://api.github.com/orgs/${owner}/packages/container/${package_name}/versions" | \ | |
| jq -r --arg tag "$tag" '.[] | select(.metadata.container.tags[] == $tag) | .id') | |
| local package_url="https://github.com/${repo_name}/pkgs/container/${package_name}/${version_id}?tag=${tag}" | |
| if ! [ x$output = x -o x$output = x- ]; then | |
| >>"$output" echo "${image_tag}@${package_url}" | |
| else | |
| echo "${image_tag}@${package_url}" | |
| fi | |
| } | |
| printf '#Upload to registry\n\n' >> "$GITHUB_STEP_SUMMARY" | |
| num_outstanding=0 | |
| concurrency_limit=8 | |
| for filename in $(find dist-images -type f -name "*.tar" -print) | |
| do | |
| IMAGE_URI="$(basename -s .tar $filename | sed 's/image-//g' | base64 -d)" | |
| upload_image "$filename" "$IMAGE_URI" ./manifest.txt ./packages.txt & | |
| num_outstanding=$(expr $num_outstanding \+ 1) | |
| if [ $num_outstanding -ge $concurrency_limit ]; then | |
| wait | |
| num_outstanding=0 | |
| fi | |
| done | |
| wait | |
| echo 'Done uploading' | |
| - | |
| name: aggregate metadata | |
| run: | | |
| { | |
| printf '### manifest.txt\n\n' | |
| printf '```\n' | |
| cat manifest.txt | |
| printf '```\n' | |
| } >>"$GITHUB_STEP_SUMMARY" | |
| { | |
| printf '### packages.txt\n\n' | |
| printf '```\n' | |
| cat packages.txt | |
| printf '```\n' | |
| } >>"$GITHUB_STEP_SUMMARY" | |
| - | |
| name: parse metadata for README | |
| run: | | |
| for filename in $(echo dist-images/image-*.tar) | |
| do | |
| IMAGE_URI="$(basename -s .tar $filename | sed 's/image-//g' | base64 -d)" | |
| mkdir -p "$(dirname "$IMAGE_URI")" | |
| mv $filename "${IMAGE_URI}.tar" | |
| filename="${IMAGE_URI}.tar" | |
| PACKAGE_URL="$(grep "${IMAGE_URI}@" packages.txt | cut -d@ -f2-)" | |
| echo '{ | |
| "filename": "'"$(basename $filename)"'", | |
| "url": "'"$IMAGE_URI"'", | |
| "tag": "'$(echo $IMAGE_URI | rev | cut -d: -f1 | rev)'", | |
| "path": "'$(echo $IMAGE_URI | cut -d/ -f2- | cut -d: -f1)'", | |
| "name": "'$(echo $IMAGE_URI | rev | cut -d: -f2- | cut -d/ -f1 | rev)'", | |
| "registry": "'"$(echo $IMAGE_URI | cut -d/ -f1)"'", | |
| "size": '"$(wc -c "$filename" | xargs | cut -f1 -d' ')"', | |
| "digest": "'$(grep "${IMAGE_URI}@" manifest.txt | cut -d@ -f2-)'", | |
| "package_url": "'$PACKAGE_URL'" | |
| }' | |
| done | >./values.json jq -s '. | {"images": .}' | |
| cat ./values.json | |
| >./README.rst python \ | |
| -m jinplate.cli \ | |
| README.rst.tmpl file://$PWD/values.json | |
| - | |
| name: Convert README.rst to markdown | |
| uses: docker://pandoc/core:3.8 | |
| with: | |
| args: >- | |
| -s | |
| --wrap=none | |
| -t gfm | |
| -o README.md | |
| README.rst | |
| - | |
| name: Print out markdown | |
| run: | | |
| cat README.md >>"$GITHUB_STEP_SUMMARY" | |
| - | |
| name: Update repo description | |
| uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0 | |
| with: | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| short-description: ${{ github.event.repository.description }} | |
| # - | |
| # name: Create Release | |
| # id: upload-release-asset | |
| # uses: softprops/action-gh-release@v1 | |
| # with: | |
| # body_path: RELEASE_NOTES.md | |
| # name: Release ${{ steps.version.outputs.CURRENT_VERSION }} | |
| # fail_on_unmatched_files: true | |
| # files: | |
| # dist-images/**/*.tar | |
| # dist-images/**/**/*.tar |