Run tests #9424
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Run tests | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: ['**'] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| jobs: | |
| sdk-tests: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: {fetch-depth: 0} | |
| - name: Detect sdk changes | |
| id: changed | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| files: | | |
| openhands-sdk/** | |
| tests/sdk/** | |
| pyproject.toml | |
| uv.lock | |
| .github/workflows/tests.yml | |
| - name: Install uv | |
| if: steps.changed.outputs.any_changed == 'true' | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: '3.13' | |
| - name: Install deps | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: uv sync --frozen --group dev | |
| - name: Check for openhands.tools imports in sdk tests | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: | | |
| echo "Checking for openhands.tools imports in tests/sdk..." | |
| if grep -r "from openhands\.tools" tests/sdk/ || grep -r "import openhands\.tools" tests/sdk/; then | |
| echo "ERROR: Found openhands.tools imports in tests/sdk/" | |
| echo "SDK tests should only import from openhands.sdk" | |
| echo "Please move tests that use openhands.tools to tests/cross/" | |
| exit 1 | |
| fi | |
| echo "✓ No openhands.tools imports found in tests/sdk/" | |
| - name: Run sdk tests with coverage | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: | | |
| # Clean up any existing coverage file | |
| rm -f .coverage | |
| # Use pytest-xdist (-n auto) for parallel execution with proper | |
| # coverage collection. --forked prevents coverage from child processes. | |
| CI=true uv run python -m pytest -vvs \ | |
| -n auto \ | |
| --cov=openhands-sdk \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=0 \ | |
| --cov-config=pyproject.toml \ | |
| tests/sdk | |
| # Rename coverage file for upload | |
| if [ -f .coverage ]; then | |
| mv .coverage coverage-sdk.dat | |
| echo "SDK coverage file prepared for upload" | |
| fi | |
| - name: Upload sdk coverage | |
| if: steps.changed.outputs.any_changed == 'true' && always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: coverage-sdk | |
| path: coverage-sdk.dat | |
| if-no-files-found: warn | |
| tools-tests: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: {fetch-depth: 0} | |
| - name: Detect tools changes | |
| id: changed | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| files: | | |
| openhands-tools/** | |
| tests/tools/** | |
| pyproject.toml | |
| uv.lock | |
| .github/workflows/tests.yml | |
| - name: Install uv | |
| if: steps.changed.outputs.any_changed == 'true' | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: '3.13' | |
| - name: Install deps | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: uv sync --frozen --group dev | |
| - name: Run tools tests with coverage | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: | | |
| # Clean up any existing coverage file | |
| rm -f .coverage | |
| # Use --forked for tools tests due to terminal test conflicts | |
| # when running in parallel (shared /tmp paths, subprocess management) | |
| CI=true uv run python -m pytest -vvs \ | |
| --forked \ | |
| --cov=openhands-tools \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=0 \ | |
| --cov-config=pyproject.toml \ | |
| tests/tools | |
| # Rename coverage file for upload | |
| if [ -f .coverage ]; then | |
| mv .coverage coverage-tools.dat | |
| echo "Tools coverage file prepared for upload" | |
| fi | |
| - name: Upload tools coverage | |
| if: steps.changed.outputs.any_changed == 'true' && always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: coverage-tools | |
| path: coverage-tools.dat | |
| if-no-files-found: warn | |
| agent-server-tests: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: {fetch-depth: 0} | |
| - name: Detect Agent Server changes | |
| id: changed | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| files: | | |
| openhands-agent-server/** | |
| tests/agent_server/** | |
| pyproject.toml | |
| uv.lock | |
| .github/workflows/tests.yml | |
| - name: Install uv | |
| if: steps.changed.outputs.any_changed == 'true' | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: '3.13' | |
| - name: Install deps | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: uv sync --frozen --group dev | |
| - name: Run Agent Server tests with coverage | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: | | |
| # Clean up any existing coverage file | |
| rm -f .coverage | |
| # Use pytest-xdist (-n auto) for parallel execution with proper | |
| # coverage collection. --forked prevents coverage from child processes. | |
| CI=true uv run python -m pytest -vvs \ | |
| -n auto \ | |
| --cov=openhands-agent-server \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=0 \ | |
| --cov-config=pyproject.toml \ | |
| tests/agent_server | |
| # Rename coverage file for upload | |
| if [ -f .coverage ]; then | |
| mv .coverage coverage-agent-server.dat | |
| echo "Agent Server coverage file prepared for upload" | |
| fi | |
| - name: Upload Agent Server coverage | |
| if: steps.changed.outputs.any_changed == 'true' && always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: coverage-agent-server | |
| path: coverage-agent-server.dat | |
| if-no-files-found: warn | |
| cross-tests: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: {fetch-depth: 0} | |
| - name: Detect cross changes | |
| id: changed | |
| uses: tj-actions/changed-files@v47 | |
| with: | |
| files: | | |
| tests/** | |
| openhands/** | |
| pyproject.toml | |
| uv.lock | |
| .github/workflows/tests.yml | |
| - name: Install uv | |
| if: steps.changed.outputs.any_changed == 'true' | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: '3.13' | |
| - name: Install deps | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: uv sync --frozen --group dev | |
| - name: Run cross tests with coverage | |
| if: steps.changed.outputs.any_changed == 'true' | |
| run: | | |
| # Clean up any existing coverage file | |
| rm -f .coverage | |
| CI=true uv run python -m pytest -vvs \ | |
| --basetemp="${{ runner.temp }}/pytest" \ | |
| -o tmp_path_retention=none \ | |
| -o tmp_path_retention_count=0 \ | |
| --cov=openhands \ | |
| --cov-report=term-missing \ | |
| --cov-fail-under=0 \ | |
| --cov-config=pyproject.toml \ | |
| tests/cross | |
| # Rename coverage file for upload | |
| if [ -f .coverage ]; then | |
| mv .coverage coverage-cross.dat | |
| echo "Cross coverage file prepared for upload" | |
| fi | |
| - name: Upload cross coverage | |
| if: steps.changed.outputs.any_changed == 'true' && always() | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: coverage-cross | |
| path: coverage-cross.dat | |
| if-no-files-found: warn | |
| coverage-report: | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| needs: [sdk-tests, tools-tests, agent-server-tests, cross-tests] | |
| if: always() && github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| python-version: '3.13' | |
| - name: Install deps (for coverage CLI) | |
| run: uv sync --frozen --group dev | |
| - name: Download coverage artifacts | |
| uses: actions/download-artifact@v6 | |
| with: | |
| path: ./cov | |
| continue-on-error: true | |
| - name: Combine coverage data | |
| run: | | |
| shopt -s nullglob | |
| # For some reason, the github action won't properly upload the original | |
| # .converage* files | |
| # Convert uploaded .dat files back to .coverage format for coverage tool | |
| for dat_file in cov/**/coverage-*.dat; do | |
| if [[ "$dat_file" == *coverage-sdk.dat ]]; then | |
| cp "$dat_file" .coverage.sdk | |
| elif [[ "$dat_file" == *coverage-tools.dat ]]; then | |
| cp "$dat_file" .coverage.tools | |
| elif [[ "$dat_file" == *coverage-agent-server.dat ]]; then | |
| cp "$dat_file" .coverage.agent-server | |
| elif [[ "$dat_file" == *coverage-cross.dat ]]; then | |
| cp "$dat_file" .coverage.cross | |
| fi | |
| done | |
| # Check if we have any coverage files | |
| coverage_files=(.coverage.*) | |
| if [ ${#coverage_files[@]} -eq 0 ]; then | |
| echo "No coverage files found; skipping combined report." | |
| exit 0 | |
| fi | |
| echo "Found ${#coverage_files[@]} coverage files" | |
| uv run coverage combine | |
| uv run coverage xml -i -o coverage.xml | |
| uv run coverage report -m | |
| - name: Pytest coverage PR comment | |
| if: always() | |
| continue-on-error: true | |
| uses: MishaKav/pytest-coverage-comment@v1 | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| pytest-xml-coverage-path: coverage.xml | |
| title: Coverage Report | |
| create-new-comment: false | |
| hide-report: false | |
| xml-skip-covered: true | |
| report-only-changed-files: true | |
| remove-links-to-files: true | |
| remove-links-to-lines: true |