feat: deleting media when redacted events get removed #1048
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tests | |
| # By separating push and pull_request by 'branches', they won't both run if there is a | |
| # push to a pull request. In other words, any change on any branch is considered a 'push', | |
| # so a change to a pull request was triggering both. Also run tests when a tag is pushed. | |
| on: | |
| push: | |
| branches: ["master", "release-*"] | |
| tags: | |
| - '**' | |
| pull_request: | |
| merge_group: | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| check-sampleconfig: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - uses: matrix-org/setup-python-poetry@v2 | |
| with: | |
| python-version: "3.13" | |
| poetry-version: "2.1.1" | |
| extras: "all" | |
| - run: poetry run scripts-dev/generate_sample_config.sh --check | |
| - run: poetry run scripts-dev/config-lint.sh | |
| # We don't update develop currently | |
| #check-schema-delta: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - uses: actions/setup-python@v5 | |
| # with: | |
| # python-version: "3.x" | |
| # - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'" | |
| # - run: scripts-dev/check_schema_delta.py --force-colors | |
| check-lockfile: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - run: .ci/scripts/check_lockfile.py | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Poetry | |
| uses: matrix-org/setup-python-poetry@v2 | |
| with: | |
| poetry-version: "2.1.1" | |
| python-version: "3.13" | |
| install-project: "false" | |
| - name: Run ruff check | |
| run: poetry run ruff check --output-format=github . | |
| - name: Run ruff format | |
| run: poetry run ruff format --check . | |
| lint-mypy: | |
| runs-on: ubuntu-latest | |
| name: Typechecking | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - name: Setup Poetry | |
| uses: matrix-org/setup-python-poetry@v2 | |
| with: | |
| # We want to make use of type hints in optional dependencies too. | |
| extras: all | |
| # We have seen odd mypy failures that were resolved when we started | |
| # installing the project again: | |
| # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775 | |
| # To make CI green, err towards caution and install the project. | |
| install-project: "true" | |
| python-version: "3.13" | |
| poetry-version: "2.1.1" | |
| # Cribbed from | |
| # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17 | |
| - name: Restore/persist mypy's cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| .mypy_cache | |
| key: mypy-cache-${{ github.context.sha }} | |
| restore-keys: mypy-cache- | |
| - name: Run mypy | |
| run: poetry run mypy | |
| lint-crlf: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check line endings | |
| run: scripts-dev/check_line_terminators.sh | |
| lint-pydantic: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - uses: matrix-org/setup-python-poetry@v2 | |
| with: | |
| poetry-version: "2.1.1" | |
| python-version: "3.13" | |
| extras: "all" | |
| - run: poetry run scripts-dev/check_pydantic_models.py | |
| #lint-clippy: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| # - run: cargo clippy -- -D warnings | |
| #lint-rustfmt: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # - name: Install Rust | |
| # uses: dtolnay/rust-toolchain@master | |
| # with: | |
| # # We use nightly so that it correctly groups together imports | |
| # toolchain: nightly-2022-12-01 | |
| # components: rustfmt | |
| # - uses: Swatinem/rust-cache@v2 | |
| # - run: cargo fmt --check | |
| # This is to detect issues with the rst file, which can otherwise cause issues | |
| # when uploading packages to PyPi. | |
| lint-readme: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - run: "pip install rstcheck" | |
| - run: "rstcheck --report-level=WARNING README.rst" | |
| # Dummy step to gate other tests on without repeating the whole list | |
| linting-done: | |
| if: ${{ !cancelled() }} # Run this even if prior jobs were skipped | |
| needs: | |
| - lint | |
| - lint-mypy | |
| - lint-crlf | |
| - lint-pydantic | |
| - check-sampleconfig | |
| #- check-schema-delta | |
| - check-lockfile | |
| #- lint-clippy | |
| #- lint-clippy-nightly | |
| #- lint-rustfmt | |
| - lint-readme | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "done" | |
| calculate-test-jobs: | |
| if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail | |
| needs: linting-done | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.x" | |
| - id: get-matrix | |
| run: .ci/scripts/calculate_jobs.py | |
| outputs: | |
| trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} | |
| sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} | |
| trial: | |
| if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail | |
| needs: | |
| - calculate-test-jobs | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} | |
| env: | |
| # Both of these are used for the coverage system. TOP gives the directory that is | |
| # the root of the source code, and COVERAGE_PROCESS_START points at the coverage | |
| # settings file to use(which also enables multiprocess implicit mode for coverage.py) | |
| TOP: ${{ github.workspace }} | |
| COVERAGE_PROCESS_START: ${{ github.workspace }}/.coveragerc | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: sudo apt-get -qq install xmlsec1 | |
| - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} | |
| if: ${{ matrix.job.postgres-version }} | |
| # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer. | |
| # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections. | |
| run: | | |
| docker run -d -p 5432:5432 \ | |
| --tmpfs /var/lib/postgresql/data:rw,size=6144m \ | |
| --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \ | |
| -e POSTGRES_PASSWORD=postgres \ | |
| -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ | |
| postgres:${{ matrix.job.postgres-version }} | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - uses: matrix-org/setup-python-poetry@v2 | |
| with: | |
| python-version: ${{ matrix.job.python-version }} | |
| poetry-version: "2.1.1" | |
| extras: ${{ matrix.job.extras }} | |
| - name: Await PostgreSQL | |
| if: ${{ matrix.job.postgres-version }} | |
| timeout-minutes: 2 | |
| run: until pg_isready -h localhost; do sleep 1; done | |
| # coverage is already installed from the pyproject.toml file | |
| - run: poetry run pip install coverage-enable-subprocess | |
| # Normally, this Github runner has 2 cores(actually threads) available to it. | |
| # Using only one trial test runner does not saturate the cpu. Rough experimentation in the past led to having 3 | |
| # of these runners per core as optimal, but now coverage is required and there may be the need | |
| # to scale back a little. This should be 6 trial test runners/jobs | |
| - run: poetry run coverage run -m twisted.trial -j$((`nproc` * 2 )) tests | |
| env: | |
| SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} | |
| SYNAPSE_POSTGRES_HOST: /var/run/postgresql | |
| SYNAPSE_POSTGRES_USER: postgres | |
| SYNAPSE_POSTGRES_PASSWORD: postgres | |
| - name: Dump logs | |
| # Logs are most useful when the command fails, always include them. | |
| if: ${{ always() }} | |
| # Note: Dumps to workflow logs instead of using actions/upload-artifact | |
| # This keeps logs colocated with failing jobs | |
| # It also ignores find's exit code; this is a best effort affair | |
| run: >- | |
| find _trial_temp -name '*.log' | |
| -exec echo "::group::{}" \; | |
| -exec cat {} \; | |
| -exec echo "::endgroup::" \; | |
| || true | |
| - run: poetry run coverage combine | |
| - name: Codecov - Upload coverage | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| token: ${{secrets.CODECOV_TOKEN}} | |
| sytest: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: | |
| - calculate-test-jobs | |
| runs-on: ubuntu-latest | |
| container: | |
| image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} | |
| volumes: | |
| - ${{ github.workspace }}:/src | |
| env: | |
| # If this is a pull request to a release branch, use that branch as default branch for sytest, else use develop | |
| # This works because the release script always create a branch on the sytest repo with the same name as the release branch | |
| SYTEST_DEFAULT_BRANCH: ${{ startsWith(github.base_ref, 'release-') && github.base_ref || 'develop' }} | |
| SYTEST_BRANCH: ${{ github.head_ref }} | |
| POSTGRES: ${{ matrix.job.postgres && 1}} | |
| MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }} | |
| ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }} | |
| WORKERS: ${{ matrix.job.workers && 1 }} | |
| BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} | |
| TOP: ${{ github.workspace }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Prepare test blacklist | |
| run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - name: Run SyTest | |
| run: /bootstrap.sh synapse | |
| working-directory: /src | |
| - name: Summarise results.tap | |
| if: ${{ always() }} | |
| run: /sytest/scripts/tap_to_gha.pl /logs/results.tap | |
| - name: Upload SyTest logs | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ always() }} | |
| with: | |
| name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) | |
| path: | | |
| /logs/results.tap | |
| /logs/**/*.log* | |
| complement: | |
| if: "${{ !failure() && !cancelled() }}" | |
| needs: | |
| - linting-done | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - arrangement: monolith | |
| database: SQLite | |
| - arrangement: monolith | |
| database: Postgres | |
| - arrangement: workers | |
| database: Postgres | |
| steps: | |
| - name: Run actions/checkout@v4 for synapse | |
| uses: actions/checkout@v4 | |
| with: | |
| path: synapse | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - name: Prepare Complement's Prerequisites | |
| run: synapse/.ci/scripts/setup_complement_prerequisites.sh | |
| - uses: actions/setup-go@v5 | |
| with: | |
| cache-dependency-path: complement/go.sum | |
| go-version-file: complement/go.mod | |
| # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once. | |
| - run: | | |
| set -o pipefail | |
| COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt | |
| shell: bash | |
| env: | |
| POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }} | |
| WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }} | |
| name: Run Complement Tests | |
| cargo-test: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - linting-done | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@68b3cb7503c78e67dae8373749990a220eb65352 | |
| - run: cargo test | |
| otlp: | |
| if: ${{ !failure() && !cancelled() }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Run actions/checkout@v4 for synapse | |
| uses: actions/checkout@v4 | |
| - run: | | |
| set -e | |
| DOCKER_BUILDKIT=1 docker build -t famedly/synapse -f docker/Dockerfile . | |
| cd otlp-test | |
| success() { [ -s out/traces.json ] && [ -s out/logs.json ]; } | |
| (until success; do sleep 1; done && docker compose down >/dev/null 2>&1) & | |
| timeout 30 docker compose up | |
| ! sudo grep -q '"ERROR"' out/logs.json # ensure no errors occurred | |
| success # ensure both traces and logs are present | |
| sudo grep -q 'Main loop terminated.' out/logs.json # ensure log is complete (to the end) | |
| # a job which marks all the other jobs as complete, thus allowing PRs to be merged. | |
| tests-done: | |
| if: ${{ always() }} | |
| needs: | |
| - trial | |
| - sytest | |
| - complement | |
| - cargo-test | |
| - linting-done | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: echo "done" |