feat: Add include_default_tools option to control built-in tools #344
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Run Examples Scripts | |
| on: | |
| pull_request: | |
| types: [labeled] | |
| workflow_dispatch: | |
| inputs: | |
| reason: | |
| description: Reason for manual trigger | |
| required: true | |
| default: '' | |
| schedule: | |
| - cron: 30 22 * * * # Runs at 10:30pm UTC every day | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| issues: write | |
| jobs: | |
| test-examples: | |
| # Schedule trigger only runs in the main repository, not in forks | |
| if: github.event.label.name == 'test-examples' || github.event_name == 'workflow_dispatch' || (github.event_name == 'schedule' && | |
| github.repository == 'OpenHands/software-agent-sdk') | |
| runs-on: blacksmith-2vcpu-ubuntu-2404 | |
| timeout-minutes: 60 | |
| steps: | |
| - name: Wait for agent server to finish build | |
| if: github.event_name == 'pull_request' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| check-name: Build & Push (python-amd64) | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| wait-interval: 10 | |
| - name: Checkout | |
| uses: actions/checkout@v5 | |
| with: | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - name: Install Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| - name: Setup Apptainer | |
| uses: eWaterCycle/setup-apptainer@v2 | |
| with: | |
| apptainer-version: 1.3.6 | |
| - name: Install dependencies | |
| run: uv sync --frozen --group dev | |
| - name: Run examples | |
| shell: bash | |
| env: | |
| LLM_API_KEY: ${{ secrets.LLM_API_KEY }} | |
| LLM_MODEL: openhands/claude-haiku-4-5-20251001 | |
| RUNTIME_API_KEY: ${{ secrets.RUNTIME_API_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPO_OWNER: ${{ github.repository_owner }} | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| GITHUB_SHA: ${{ github.event.pull_request.head.sha }} | |
| OPENHANDS_CLOUD_API_KEY: ${{ secrets.ALLHANDS_BOT_OPENHANDS_SAAS_API_KEY }} | |
| run: | | |
| RESULTS_DIR=".example-test-results" | |
| REPORT_PATH="examples_report.md" | |
| rm -rf "$RESULTS_DIR" | |
| mkdir -p "$RESULTS_DIR" | |
| update_comment() { | |
| if [ -z "$API_URL" ]; then | |
| echo "Skipping PR comment update because API_URL is unset." | |
| return | |
| fi | |
| local comment_body="$1" | |
| local payload | |
| local response | |
| payload=$(jq -n --arg body "$comment_body" '{body: $body}') | |
| if [ -z "$COMMENT_ID" ]; then | |
| echo "Creating PR comment..." | |
| if ! response=$(curl -sSf -X POST \ | |
| -H "Authorization: token ${GITHUB_TOKEN}" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| -H "Content-Type: application/json" \ | |
| "${API_URL}" \ | |
| -d "$payload"); then | |
| echo "::error::Failed to create PR comment." | |
| exit 1 | |
| fi | |
| COMMENT_ID=$(echo "$response" | jq -r '.id // ""') | |
| if [ -z "$COMMENT_ID" ]; then | |
| echo "::error::GitHub API response did not include a comment id: $response" | |
| exit 1 | |
| fi | |
| echo "Created comment with ID: $COMMENT_ID" | |
| else | |
| echo "Updating PR comment (ID: $COMMENT_ID)..." | |
| if ! curl -sSf -X PATCH \ | |
| -H "Authorization: token ${GITHUB_TOKEN}" \ | |
| -H "Accept: application/vnd.github.v3+json" \ | |
| -H "Content-Type: application/json" \ | |
| "https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/issues/comments/${COMMENT_ID}" \ | |
| -d "$payload" > /dev/null; then | |
| echo "::error::Failed to update PR comment (ID: $COMMENT_ID)." | |
| exit 1 | |
| fi | |
| fi | |
| } | |
| API_URL="" | |
| COMMENT_ID="" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| API_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/issues/${PR_NUMBER}/comments" | |
| initial_comment="## 🔄 Running Examples with \`${LLM_MODEL}\`" | |
| initial_comment+=$'\n\n' | |
| initial_comment+="_Run in progress..._" | |
| initial_comment+=$'\n' | |
| update_comment "$initial_comment" | |
| fi | |
| EXIT_CODE=0 | |
| uv run pytest tests/examples/test_examples.py \ | |
| --run-examples \ | |
| --examples-results-dir "$RESULTS_DIR" \ | |
| -n 4 || EXIT_CODE=$? | |
| TIMESTAMP="$(date -u '+%Y-%m-%d %H:%M:%S UTC')" | |
| WORKFLOW_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| uv run python scripts/render_examples_report.py \ | |
| --results-dir "$RESULTS_DIR" \ | |
| --model "$LLM_MODEL" \ | |
| --workflow-url "$WORKFLOW_URL" \ | |
| --timestamp "$TIMESTAMP" \ | |
| --output "$REPORT_PATH" | |
| COMMENT_BODY="$(cat "$REPORT_PATH")" | |
| echo "$COMMENT_BODY" | |
| if [ "${{ github.event_name }}" = "pull_request" ]; then | |
| echo "Publishing PR comment..." | |
| update_comment "$COMMENT_BODY" | |
| fi | |
| if [ $EXIT_CODE -ne 0 ]; then | |
| exit $EXIT_CODE | |
| fi | |
| - name: Read examples report for issue comment | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| id: read_report | |
| shell: bash | |
| run: | | |
| if [ -f examples_report.md ]; then | |
| REPORT_CONTENT=$(cat examples_report.md) | |
| echo "report<<EOF" >> "$GITHUB_OUTPUT" | |
| echo "$REPORT_CONTENT" >> "$GITHUB_OUTPUT" | |
| echo "EOF" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "report=Report file not found" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Comment with results on tracker issue | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| uses: KeisukeYamashita/create-comment@v1 | |
| with: | |
| number: 976 | |
| unique: false | |
| comment: | | |
| **Trigger:** ${{ github.event_name == 'schedule' && 'Nightly Scheduled Run' || format('Manual Trigger: {0}', github.event.inputs.reason) }} | |
| **Commit:** ${{ github.sha }} | |
| **Workflow Run:** ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| ${{ steps.read_report.outputs.report }} |