From 398c448fb8fe454b4359e097bde6737cccdf0762 Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Fri, 9 May 2025 09:53:37 +0100 Subject: [PATCH 1/2] Minor improvements to GH workflows This change adds debugging job for vendoring process to investigate why vendoring PRs are not closed automatically (job skipped), while the check expression for it looks correct. It also adds a number of minor cleanups to workflows. --- .github/workflows/Java.yml | 88 ++++++++++++++++++++---------------- .github/workflows/Vendor.yml | 7 +-- 2 files changed, 51 insertions(+), 44 deletions(-) diff --git a/.github/workflows/Java.yml b/.github/workflows/Java.yml index 23b36c8a4..edd0a4006 100644 --- a/.github/workflows/Java.yml +++ b/.github/workflows/Java.yml @@ -2,24 +2,25 @@ name: Java JDBC on: push: pull_request: - workflow_call: workflow_dispatch: - repository_dispatch: - + inputs: + skip_tests: + description: 'Skip test runs' + required: false + default: 'false' + type: 'string' env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} - OVERRIDE_GIT_DESCRIBE: ${{ inputs.override_git_describe }} + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} jobs: - format_check: + format-check: + name: Format Check runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ inputs.git_ref }} - - run: | python3 -m pip install --user clang_format==11.0.1 make format-check @@ -27,6 +28,7 @@ jobs: java-linux-amd64: name: Java Linux (amd64) runs-on: ubuntu-latest + needs: format-check env: MANYLINUX_IMAGE: quay.io/pypa/manylinux_2_28_x86_64 MANYLINUX_PACKAGES: java-1.8.0-openjdk-devel ninja-build @@ -34,7 +36,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ inputs.git_ref }} - name: Build shell: bash @@ -59,12 +60,10 @@ jobs: - name: Deploy shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | cp build/release/duckdb_jdbc.jar duckdb_jdbc-linux-amd64.jar ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-linux-amd64.jar + - uses: actions/upload-artifact@v4 with: name: java-linux-amd64 @@ -82,7 +81,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ inputs.git_ref }} - name: Build shell: bash @@ -107,12 +105,9 @@ jobs: - name: Deploy shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | cp build/release/duckdb_jdbc.jar duckdb_jdbc-linux-aarch64.jar - # ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-linux-aarch64.jar + ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-linux-aarch64.jar - uses: actions/upload-artifact@v4 with: @@ -130,39 +125,34 @@ jobs: with: fetch-depth: 0 ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' || github.repository != 'duckdb/duckdb-java' }} - name: Build shell: bash run: make release + - name: Java Tests if: ${{ inputs.skip_tests != 'true' }} shell: bash run: | ls -R . make test + - name: Deploy shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | cp build/release/duckdb_jdbc.jar duckdb_jdbc-windows-amd64.jar ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-windows-amd64.jar + - uses: actions/upload-artifact@v4 with: name: java-windows-amd64 path: | build/release/duckdb_jdbc.jar - java-osx-universal: name: Java OSX (Universal) runs-on: macos-14 @@ -171,33 +161,30 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ inputs.git_ref }} + - uses: actions/setup-python@v5 with: python-version: "3.12" - - name: Setup Ccache - uses: hendrikmuhs/ccache-action@main - with: - key: ${{ github.job }} - save: ${{ github.ref == 'refs/heads/main' }} + - name: Build shell: bash run: make release + - name: Java Tests if: ${{ inputs.skip_tests != 'true' }} shell: bash run: make test + - name: See if this actually universal shell: bash run: lipo -archs build/release/libduckdb_java.so_osx_universal | grep "x86_64 arm64" + - name: Deploy shell: bash - env: - AWS_ACCESS_KEY_ID: ${{ secrets.S3_DUCKDB_STAGING_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DUCKDB_STAGING_KEY }} run: | cp build/release/duckdb_jdbc.jar duckdb_jdbc-osx-universal.jar ./scripts/upload-assets-to-staging.sh github_release duckdb_jdbc-osx-universal.jar + - uses: actions/upload-artifact@v4 with: name: java-osx-universal @@ -219,7 +206,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - ref: ${{ inputs.git_ref }} - shell: bash run: mkdir jdbc-artifacts @@ -315,7 +301,7 @@ jobs: java-merge-vendoring-pr: name: Merge vendoring PR - if: ${{ github.repository == 'duckdb/duckdb-java' && github.event_name == 'pull_request' && github.head_ref == format('vendoring-{0}', github.ref_name) }} + if: ${{ github.repository == 'duckdb/duckdb-java' && github.event_name == 'pull_request' && startsWith(github.head_ref, 'vendoring-') }} needs: - java-linux-aarch64 - java-linux-amd64 @@ -332,7 +318,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - # echo "Merging PR number: ${{ github.event.pull_request.number }} with message: ${{ github.event.pull_request.title }}" + echo "Merging PR number: ${{ github.event.pull_request.number }} with message: ${{ github.event.pull_request.title }}" gh pr merge vendoring-${{ github.ref_name }} \ --rebase \ --subject "${{ github.event.pull_request.title }}" \ @@ -348,3 +334,27 @@ jobs: git pull --ff-only git branch vendoring-${{ github.ref_name }} git push origin vendoring-${{ github.ref_name }} + + java-merge-vendoring-pr-debug: + name: Merge vendoring PR Debug + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Merge vendoring PR Debug + id: merge_vendoring_pr_debug + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "github.repository: ${{ github.repository }}" + echo "github.event_name: ${{ github.event_name }}" + echo "github.head_ref: ${{ github.head_ref }}" + echo "github.base_ref: ${{ github.base_ref }}" + echo "github.ref_name: ${{ github.ref_name }}" + echo "format('vendoring-{0}', github.ref_name): ${{ format('vendoring-{0}', github.ref_name) }}" + echo "format('vendoring-{0}', github.base_ref): ${{ format('vendoring-{0}', github.base_ref) }}" + echo "startsWith(github.head_ref, 'workflows_minor_'): ${{ startsWith(github.head_ref, 'vendoring-') }}" + echo "github.event.pull_request.number: ${{ github.event.pull_request.number }}" + echo "github.event.pull_request.title: ${{ github.event.pull_request.title }}" diff --git a/.github/workflows/Vendor.yml b/.github/workflows/Vendor.yml index f4fae71d6..a1b4c7082 100644 --- a/.github/workflows/Vendor.yml +++ b/.github/workflows/Vendor.yml @@ -73,7 +73,6 @@ jobs: - name: Commit and push the changes id: commit_and_push - if: ${{ steps.vendor.outcome == 'success' }} run: | MSG="Update vendored DuckDB sources to ${{ steps.vendor.outputs.vendor_rev }}" git commit -m "${MSG}" @@ -89,7 +88,6 @@ jobs: - name: Check PR exists id: check_pr_exists - if: ${{ steps.commit_and_push.outcome == 'success' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -105,7 +103,6 @@ jobs: - name: Prepare PR message id: prepare_pr_message - if: ${{ steps.check_pr_exists.outcome == 'success' }} run: | DATE="$(date +"%Y-%m-%d %H:%M:%S")" CHANGE_LABEL="duckdb/duckdb#${{ steps.vendor.outputs.vendor_rev }}" @@ -116,7 +113,7 @@ jobs: - name: Create PR id: create_pr - if: ${{ steps.prepare_pr_message.outcome == 'success' && steps.check_pr_exists.outputs.pr_exists == 'false' }} + if: ${{ steps.check_pr_exists.outputs.pr_exists == 'false' }} env: # We cannot use default workflow's GITHUB_TOKEN here, because # it is restricted to not trigger 'pull_request' event that @@ -136,7 +133,7 @@ jobs: - name: Update PR id: update_pr - if: ${{ steps.prepare_pr_message.outcome == 'success' && steps.check_pr_exists.outputs.pr_exists == 'true' }} + if: ${{ steps.check_pr_exists.outputs.pr_exists == 'true' }} env: # We cannot use default workflow's GITHUB_TOKEN here, because # it is restricted to not trigger 'pull_request' event that From 725f1f60cbdb17de83faae0e6a36ba30a3aa40ed Mon Sep 17 00:00:00 2001 From: Alex Kasko Date: Fri, 9 May 2025 10:09:08 +0100 Subject: [PATCH 2/2] Remove debug job --- .github/workflows/Java.yml | 36 ++++++------------------------------ 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/.github/workflows/Java.yml b/.github/workflows/Java.yml index edd0a4006..baaff423e 100644 --- a/.github/workflows/Java.yml +++ b/.github/workflows/Java.yml @@ -301,7 +301,7 @@ jobs: java-merge-vendoring-pr: name: Merge vendoring PR - if: ${{ github.repository == 'duckdb/duckdb-java' && github.event_name == 'pull_request' && startsWith(github.head_ref, 'vendoring-') }} + if: ${{ github.repository == 'duckdb/duckdb-java' && github.event_name == 'pull_request' && github.head_ref == format('vendoring-{0}', github.base_ref) }} needs: - java-linux-aarch64 - java-linux-amd64 @@ -319,7 +319,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | echo "Merging PR number: ${{ github.event.pull_request.number }} with message: ${{ github.event.pull_request.title }}" - gh pr merge vendoring-${{ github.ref_name }} \ + gh pr merge vendoring-${{ github.base_ref }} \ --rebase \ --subject "${{ github.event.pull_request.title }}" \ --body "" @@ -328,33 +328,9 @@ jobs: id: update_vendoring_branch if: ${{ steps.merge_vendoring_pr.outcome == 'success' }} run: | - # Delete vendoring-${{ github.ref_name }} branch and re-create it for future PRs - git push --delete origin vendoring-${{ github.ref_name }} + # Delete vendoring-${{ github.base_ref }} branch and re-create it for future PRs + git push --delete origin vendoring-${{ github.base_ref }} git checkout --track origin/main git pull --ff-only - git branch vendoring-${{ github.ref_name }} - git push origin vendoring-${{ github.ref_name }} - - java-merge-vendoring-pr-debug: - name: Merge vendoring PR Debug - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Merge vendoring PR Debug - id: merge_vendoring_pr_debug - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - echo "github.repository: ${{ github.repository }}" - echo "github.event_name: ${{ github.event_name }}" - echo "github.head_ref: ${{ github.head_ref }}" - echo "github.base_ref: ${{ github.base_ref }}" - echo "github.ref_name: ${{ github.ref_name }}" - echo "format('vendoring-{0}', github.ref_name): ${{ format('vendoring-{0}', github.ref_name) }}" - echo "format('vendoring-{0}', github.base_ref): ${{ format('vendoring-{0}', github.base_ref) }}" - echo "startsWith(github.head_ref, 'workflows_minor_'): ${{ startsWith(github.head_ref, 'vendoring-') }}" - echo "github.event.pull_request.number: ${{ github.event.pull_request.number }}" - echo "github.event.pull_request.title: ${{ github.event.pull_request.title }}" + git branch vendoring-${{ github.base_ref }} + git push origin vendoring-${{ github.base_ref }}