CI Linux Complementary Debian Multi-Version x86_64 GNU #85
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # CI Strategy: Complementary Testing for SSL and System Library Regressions | |
| # | |
| # This workflow runs complementary tests that don't need to block PRs but are essential | |
| # for catching regressions in SSL functionality and system library compatibility. | |
| # It complements the main CI by testing stable-but-critical functionality on a nightly | |
| # schedule and when workflow changes are made. | |
| # | |
| # WHY COMPLEMENTARY TESTING: | |
| # - SSL specs have been stable for 3+ years and rarely break due to code changes | |
| # - Integration specs test OS library compatibility, which changes with system updates | |
| # - These tests catch regressions from external changes (OpenSSL updates, system libs) | |
| # - Running every 3 days to prevent these slower tests from blocking PR velocity | |
| # - Manual triggering allows testing workflow changes before they go into schedule | |
| # | |
| # WHY WE TEST DEBIAN BOOKWORM AND TRIXIE: | |
| # - Different OpenSSL versions and SSL library implementations | |
| # - Different system library versions that affect SSL handshakes and compatibility | |
| # - Forward compatibility testing with Trixie (Debian testing/future stable) | |
| # - Catching SSL regressions from package updates in both stable and testing | |
| # | |
| # SSL TESTING (specs_install + specs_precompiled): | |
| # - Tests SSL/TLS connectivity with Kafka using docker-compose-ssl.yml | |
| # - Validates certificate handling and SSL handshakes across Ruby and Debian versions | |
| # - Ensures SSL works with both compiled-from-source and precompiled flows | |
| # - Catches OpenSSL version compatibility issues and SSL library regressions | |
| # - Tests real SSL scenarios that mirror production deployments | |
| # | |
| # INTEGRATION TESTING (integration specs in both jobs): | |
| # - Tests system library compatibility without requiring Kafka infrastructure | |
| # - Validates libssl, libsasl2, libzstd, zlib integration across OS versions | |
| # - Ensures native extensions work with different system library versions | |
| # - Catches regressions from Debian package updates and system changes | |
| # - Tests both compilation and precompiled library compatibility | |
| # | |
| # SCHEDULING STRATEGY: | |
| # - Runs every 3 days at 3 AM to catch system/library changes from base image updates | |
| # - Triggers on workflow file changes to validate CI modifications | |
| # - Manual dispatch available for ad-hoc regression testing | |
| # - Separate artifact naming prevents interference with main CI | |
| # | |
| # This approach ensures comprehensive coverage while keeping PR CI fast and focused | |
| # on code-related issues rather than infrastructure/system regressions. | |
| name: CI Linux Complementary Debian Multi-Version x86_64 GNU | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| schedule: | |
| - cron: '0 3 */3 * *' | |
| workflow_dispatch: | |
| pull_request: | |
| paths: | |
| - '.github/workflows/ci_linux_debian_x86_64_gnu_complementary.yml' | |
| - 'spec/integrations/**' | |
| branches: [ master ] | |
| push: | |
| branches: | |
| - 'v[0-9]+.[0-9]+.*' | |
| tags-ignore: | |
| - '**' | |
| permissions: | |
| contents: read | |
| env: | |
| BUNDLE_RETRY: 6 | |
| BUNDLE_JOBS: 4 | |
| jobs: | |
| build_precompiled: | |
| timeout-minutes: 50 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| debian: | |
| - 'bookworm' | |
| - 'trixie' | |
| runs-on: ubuntu-22.04 # renovate: ignore | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| gcc \ | |
| g++ \ | |
| make \ | |
| tar \ | |
| gzip \ | |
| wget \ | |
| curl \ | |
| file \ | |
| pkg-config \ | |
| autoconf \ | |
| automake \ | |
| libtool \ | |
| python3 \ | |
| git \ | |
| ca-certificates \ | |
| patch \ | |
| libsasl2-dev \ | |
| libssl-dev \ | |
| zlib1g-dev \ | |
| libzstd-dev \ | |
| bison \ | |
| flex \ | |
| perl \ | |
| binutils-dev | |
| - name: Cache build-tmp directory | |
| uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 | |
| with: | |
| path: ext/build-tmp | |
| key: build-tmp-complementary-${{ runner.os }}-${{ matrix.debian }}-${{ hashFiles('ext/*.sh', 'ext/Rakefile') }}-v2 | |
| - name: Build precompiled librdkafka.so | |
| run: | | |
| cd ext | |
| ./build_linux_x86_64_gnu.sh | |
| - name: Upload precompiled library | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 | |
| with: | |
| name: librdkafka-precompiled-linux-complementary-${{ matrix.debian }} | |
| path: ext/ | |
| retention-days: 1 | |
| specs_install: | |
| timeout-minutes: 50 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: | |
| - '3.4' | |
| - '3.3' | |
| - '3.2' | |
| debian: | |
| - 'bookworm' | |
| - 'trixie' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Start Kafka with Docker Compose | |
| run: | | |
| ./ext/generate-ssl-certs.sh | |
| docker compose -f docker-compose-ssl.yml up -d | |
| echo "Waiting for Kafka to be ready..." | |
| sleep 10 | |
| for i in {1..30}; do | |
| if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then | |
| echo "Kafka topics command succeeded!" | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| - name: Install dependencies | |
| env: | |
| RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext | |
| run: | | |
| docker run --rm \ | |
| --network host \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| -e "RDKAFKA_EXT_PATH=/workspace/ext" \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get upgrade -y openssl libssl3 libssl-dev && \ | |
| apt-get install -y git build-essential pkg-config \ | |
| libssl-dev libsasl2-dev zlib1g-dev libzstd-dev && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install' | |
| - name: Build gem with mini_portile | |
| run: | | |
| docker run --rm \ | |
| --network host \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get upgrade -y openssl libssl3 libssl-dev && \ | |
| apt-get install -y git build-essential pkg-config \ | |
| libssl-dev libsasl2-dev zlib1g-dev libzstd-dev && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install && \ | |
| cd ext && bundle exec rake' | |
| - name: Run all specs in SSL (compiled flow) | |
| env: | |
| RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext | |
| KAFKA_SSL_ENABLED: "true" | |
| run: | | |
| docker run --rm \ | |
| --network host \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| -e "RDKAFKA_EXT_PATH=/workspace/ext" \ | |
| -e "KAFKA_SSL_ENABLED=true" \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get upgrade -y openssl libssl3 libssl-dev && \ | |
| apt-get install -y git build-essential pkg-config \ | |
| libssl-dev libsasl2-dev zlib1g-dev libzstd-dev default-jdk && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install && \ | |
| echo "=== SSL Library Versions (${{ matrix.debian }}) ===" && \ | |
| openssl version && \ | |
| dpkg -l | grep -E "(libssl|openssl)" && \ | |
| echo "=== Running SSL Specs (Compiled) ===" && \ | |
| bundle exec rspec' | |
| - name: Verify Kafka warnings | |
| run: bin/verify_kafka_warnings | |
| - name: Run integration specs (compiled flow) | |
| run: | | |
| docker run --rm \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get upgrade -y openssl libssl3 libssl-dev && \ | |
| apt-get install -y git build-essential pkg-config \ | |
| libssl-dev libsasl2-dev zlib1g-dev libzstd-dev libcurl4-openssl-dev libcurl4 && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install && \ | |
| echo "=== OS Library Versions (${{ matrix.debian }}) ===" && \ | |
| openssl version && \ | |
| dpkg -l | grep -E "(libssl|libsasl|libzstd|zlib)" && \ | |
| echo "=== Running Integration Specs (Compiled) ===" && \ | |
| for file in $(ls spec/integrations/*_spec.rb); do \ | |
| echo "Running $file with Ruby ${{ matrix.ruby }} on ${{ matrix.debian }}"; \ | |
| bundle exec ruby "$file" || exit 1; \ | |
| done' | |
| specs_precompiled: | |
| timeout-minutes: 50 | |
| needs: build_precompiled | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| ruby: | |
| - '3.4' | |
| - '3.3' | |
| - '3.2' | |
| debian: | |
| - 'bookworm' | |
| - 'trixie' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download precompiled library | |
| uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 | |
| with: | |
| name: librdkafka-precompiled-linux-complementary-${{ matrix.debian }} | |
| path: ext/ | |
| - name: Start Kafka with Docker Compose | |
| run: | | |
| ./ext/generate-ssl-certs.sh | |
| docker compose -f docker-compose-ssl.yml up -d | |
| echo "Waiting for Kafka to be ready..." | |
| sleep 10 | |
| for i in {1..30}; do | |
| if docker compose exec -T kafka kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1; then | |
| echo "Kafka topics command succeeded!" | |
| break | |
| fi | |
| sleep 2 | |
| done | |
| - name: Run specs with precompiled library and SSL | |
| env: | |
| RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext | |
| KAFKA_SSL_ENABLED: "true" | |
| RDKAFKA_PRECOMPILED: "true" | |
| run: | | |
| docker run --rm \ | |
| --network host \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| -e "RDKAFKA_EXT_PATH=/workspace/ext" \ | |
| -e "KAFKA_SSL_ENABLED=true" \ | |
| -e "RDKAFKA_PRECOMPILED=true" \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get install -y git default-jdk && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install && \ | |
| echo "=== SSL Library Versions (${{ matrix.debian }}) ===" && \ | |
| openssl version && \ | |
| dpkg -l | grep -E "(libssl|openssl)" && \ | |
| echo "=== Running SSL Specs (Precompiled) ===" && \ | |
| bundle exec rspec' | |
| - name: Verify Kafka warnings | |
| run: bin/verify_kafka_warnings | |
| - name: Run integration specs (precompiled flow) | |
| env: | |
| RDKAFKA_EXT_PATH: ${{ github.workspace }}/ext | |
| RDKAFKA_PRECOMPILED: "true" | |
| run: | | |
| docker run --rm \ | |
| -v "${{ github.workspace }}:/workspace" \ | |
| -w /workspace \ | |
| -e "RDKAFKA_EXT_PATH=/workspace/ext" \ | |
| -e "RDKAFKA_PRECOMPILED=true" \ | |
| ruby:${{ matrix.ruby }}-${{ matrix.debian }} \ | |
| sh -c 'apt-get update && \ | |
| apt-get upgrade -y openssl libssl3 libssl-dev && \ | |
| apt-get install -y git build-essential pkg-config \ | |
| libssl-dev libsasl2-dev zlib1g-dev libzstd-dev libcurl4-openssl-dev libcurl4 && \ | |
| git config --global --add safe.directory /workspace && \ | |
| bundle config set --local path vendor/bundle && \ | |
| bundle install && \ | |
| echo "=== OS Library Versions (${{ matrix.debian }}) ===" && \ | |
| openssl version && \ | |
| dpkg -l | grep -E "(libssl|libsasl|libzstd|zlib)" && \ | |
| echo "=== Running Integration Specs (Precompiled) ===" && \ | |
| for file in $(ls spec/integrations/*_spec.rb); do \ | |
| echo "Running $file with Ruby ${{ matrix.ruby }} on ${{ matrix.debian }} (precompiled)"; \ | |
| bundle exec ruby "$file" || exit 1; \ | |
| done' | |
| ci-success: | |
| name: CI Linux Complementary Debian Multi-Version x86_64 GNU Success | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - specs_install | |
| - build_precompiled | |
| - specs_precompiled | |
| steps: | |
| - name: Check all jobs passed | |
| if: | | |
| contains(needs.*.result, 'failure') || | |
| contains(needs.*.result, 'cancelled') || | |
| contains(needs.*.result, 'skipped') | |
| run: exit 1 | |
| - run: echo "All CI checks passed!" |