Extract, transform and alert #4601
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Extract, transform and alert | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "22 14,19,23 * * *" | |
| jobs: | |
| extract: | |
| continue-on-error: true | |
| timeout-minutes: 30 # Sets a 30-minute timeout for this specific job | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| source: | |
| [ | |
| al, | |
| ak, | |
| az, | |
| ca, | |
| co, | |
| ct, | |
| dc, | |
| de, | |
| fl, | |
| ga, | |
| hi, | |
| ia, | |
| id, | |
| il, | |
| in, | |
| ks, | |
| ky, | |
| la, | |
| md, | |
| me, | |
| mi, | |
| mo, | |
| mt, | |
| ne, | |
| nj, | |
| nm, | |
| ny, | |
| oh, | |
| ok, | |
| or, | |
| pa, | |
| ri, | |
| sc, | |
| sd, | |
| tn, | |
| tx, | |
| ut, | |
| va, | |
| vt, | |
| wa, | |
| wi, | |
| ] | |
| name: Extract | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Extract | |
| uses: ./.github/actions/extract | |
| with: | |
| source: ${{ matrix.source }} | |
| bln-api-key: ${{ secrets.BLN_API_TOKEN }} | |
| bln-project-id: ${{ secrets.BLN_PROJECT_ID }} | |
| env: | |
| ZYTE_API_KEY: ${{ secrets.ZYTE_API_KEY }} | |
| - name: Fail alert | |
| id: slack | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_ID }} | |
| slack-message: "🚨Layoff Watch ETL extraction run <https://github.com/biglocalnews/warn-github-flow/actions/runs/${{ github.run_id }}|#${{ github.run_id }}> for `${{ matrix.source }}` failed" | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| transform: | |
| name: Transform | |
| timeout-minutes: 30 # Sets a 30-minute timeout for this specific job | |
| runs-on: ubuntu-latest | |
| needs: [extract] | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Transform | |
| uses: ./.github/actions/transform | |
| with: | |
| bln-api-key: ${{ secrets.BLN_API_TOKEN }} | |
| bln-project-id: ${{ secrets.BLN_PROJECT_ID }} | |
| - name: Fail alert | |
| id: slack | |
| if: ${{ failure() }} | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_ID }} | |
| slack-message: "🚨 Layoff Watch ETL transformation run <https://github.com/biglocalnews/warn-github-flow/actions/runs/${{ github.run_id }}|#${{ github.run_id }}> failed" | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| status-report: | |
| name: Status report | |
| timeout-minutes: 30 # Sets a 30-minute timeout for this specific job | |
| runs-on: ubuntu-latest | |
| needs: [transform] | |
| steps: | |
| - name: Post | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| channel-id: ${{ secrets.SLACK_CHANNEL_ID }} | |
| slack-message: "👍 Layoff Watch ETL run <https://github.com/biglocalnews/warn-github-flow/actions/runs/${{ github.run_id }}|#${{ github.run_id }}> finished successfully" | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| alert: | |
| name: Alert | |
| timeout-minutes: 30 # Sets a 30-minute timeout for this specific job | |
| runs-on: ubuntu-latest | |
| needs: [transform] | |
| permissions: | |
| contents: 'read' | |
| id-token: 'write' | |
| steps: | |
| - name: Clone repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| # - id: install-python3_9 | |
| # name: Install Python 3.9 | |
| # uses: actions/setup-python@v5 | |
| # with: | |
| # python-version: '3.9' | |
| - id: install-python3_14 | |
| name: Install Python 3.14 | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: '3.14' | |
| - id: auth-gcloud | |
| name: Authenticate with Google Cloud | |
| uses: google-github-actions/auth@v3 | |
| with: | |
| workload_identity_provider: ${{ secrets.GOOGLE_WORKLOAD_IDENTITY_PROVIDER }} | |
| service_account: ${{ secrets.GOOGLE_SERVICE_ACCOUNT }} | |
| create_credentials_file: true | |
| - name: Install public dependencies | |
| run: | | |
| pip install keyring keyrings.google-artifactregistry-auth | |
| keyring --list-backends | |
| shell: bash | |
| - name: Install private dependencies | |
| run: pip install warn-bot==0.1.55 -vv --index-url https://us-west2-python.pkg.dev/big-local-news-267923/warn-bot/simple/ --extra-index-url https://pypi.org/simple/ | |
| shell: bash | |
| env: | |
| GOOGLE_APPLICATION_CREDENTIALS: ${{ steps.auth-gcloud.outputs.credentials_file_path }} | |
| - name: Send alert | |
| # run: warn-bot | |
| run: python3 -m warn_bot.cli |