diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 217b51eb4..40964e0e9 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -33,23 +33,72 @@ jobs: fi integration-tests: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: + - name: scan-basic + pattern: "^Test(CreateScan|ScanCreate|ScansE2E|FastScan|LightQueries|Recommended)" + timeout: "50m" + - name: scan-advanced + pattern: "^Test(Incremental|Cancel|BranchPrimary|ScansUpdate|Invalid|ScanShow|ScanTimeout|BrokenLink|ScanWorkflow|ScanLog|PartialScan|FailedScan|ScanFilter|RunKics|ScanType|ScanGenerating|ScanWith|ScanList)" + timeout: "50m" + - name: results + pattern: "^Test(Result|CodeBashing|BFL|RiskManagement)" + timeout: "40m" + - name: auth-project + pattern: "^Test(Auth|Project|Root|Groups|Tenant|SetLog|Main)" + timeout: "30m" + - name: realtime-containers + pattern: "^Test(Container|Realtime|Iac|Oss|Secrets|Asca|Sca)" + timeout: "40m" + - name: utilities + pattern: "^Test(PR|Predicate|Import|LearnMore|Configuration|UserCount|Telemetry|PreCommit|PreReceive|Azure|Bitbucket|GitHub|GitLab|Remediation|Mask)" + timeout: "40m" + - name: catch-all + pattern: "." + timeout: "60m" + + name: Integration Tests (${{ matrix.shard.name }}) steps: - name: Checkout the repository uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + - name: Set up Go version uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 with: go-version-file: go.mod + - run: go version + - name: Go Build run: go build -o ./bin/cx ./cmd - - name: Install gocovmerge - run: go install github.com/wadey/gocovmerge@latest + + - name: Install gotestsum + run: go install gotest.tools/gotestsum@latest + - name: Install pre-commit run: | pip install pre-commit pre-commit install - - name: Go Integration test + + - name: Start Squid Proxy + run: | + docker run \ + --name squid \ + -d \ + -p 3128:3128 \ + -v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \ + -v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \ + ubuntu/squid:5.2-22.04_beta + + - name: Download ScaResolver + run: | + wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz + tar -xzvf ScaResolver-linux64.tar.gz -C /tmp + rm -rf ScaResolver-linux64.tar.gz + + - name: Run Integration Tests (${{ matrix.shard.name }}) shell: bash env: CX_BASE_URI: ${{ secrets.CX_BASE_URI }} @@ -94,30 +143,218 @@ jobs: PR_BITBUCKET_NAMESPACE: "AstSystemTest" PR_BITBUCKET_REPO_NAME: "cliIntegrationTest" PR_BITBUCKET_ID: 1 + SCA_RESOLVER: /tmp/ScaResolver run: | - sudo chmod +x ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh - ./internal/commands/.scripts/integration_up.sh - ./internal/commands/.scripts/integration_down.sh - - - name: Coverage report + gotestsum \ + --format standard-verbose \ + --junitfile test-results-${{ matrix.shard.name }}.xml \ + --jsonfile test-results-${{ matrix.shard.name }}.json \ + -- \ + -tags integration \ + -v \ + -timeout ${{ matrix.shard.timeout }} \ + -run '${{ matrix.shard.pattern }}' \ + -coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \ + -coverprofile cover-${{ matrix.shard.name }}.out \ + github.com/checkmarx/ast-cli/test/integration + + - name: Extract Test Names for Validation + if: always() + run: | + grep 'testcase name=' test-results-${{ matrix.shard.name }}.xml | \ + sed 's/.*name="\([^"]*\)".*/\1/' > tests-ran-${{ matrix.shard.name }}.txt || true + echo "Tests run in ${{ matrix.shard.name }} shard:" + wc -l tests-ran-${{ matrix.shard.name }}.txt + + - name: Upload Test Results + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() + with: + name: test-results-${{ matrix.shard.name }} + path: | + test-results-${{ matrix.shard.name }}.xml + test-results-${{ matrix.shard.name }}.json + tests-ran-${{ matrix.shard.name }}.txt + + - name: Upload Coverage uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + if: always() with: - name: ${{ runner.os }}-coverage-latest - path: coverage.html + name: coverage-${{ matrix.shard.name }} + path: cover-${{ matrix.shard.name }}.out - - name: Check if total coverage is greater then 75 - shell: bash + - name: Stop Squid Proxy + if: always() + run: docker stop squid || true + + # Aggregate results from all shards + aggregate-test-results: + needs: integration-tests + runs-on: ubuntu-latest + if: always() + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Download All Test Results + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4 + with: + pattern: test-results-* + merge-multiple: true + + - name: Download All Coverage Files + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4 + with: + pattern: coverage-* + merge-multiple: true + + - name: Validate All Tests Ran run: | - CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + echo "## 🔍 Test Completeness Validation" >> $GITHUB_STEP_SUMMARY + + # Count expected tests from source + EXPECTED=$(grep -r "^func Test" test/integration/*.go | wc -l) + echo "Expected tests from source: $EXPECTED" + + # Count unique tests that actually ran (excluding duplicates from catch-all) + cat tests-ran-*.txt | sort -u > all-tests-ran.txt + ACTUAL=$(wc -l < all-tests-ran.txt) + echo "Unique tests executed: $ACTUAL" + + echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY + echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Expected Tests | $EXPECTED |" >> $GITHUB_STEP_SUMMARY + echo "| Unique Tests Run | $ACTUAL |" >> $GITHUB_STEP_SUMMARY + + if [ "$EXPECTED" -ne "$ACTUAL" ]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY + echo "> Test count mismatch! Expected $EXPECTED but ran $ACTUAL" >> $GITHUB_STEP_SUMMARY + + # Show missing tests + grep -rh "^func Test" test/integration/*.go | \ + sed 's/func \(Test[^(]*\).*/\1/' | sort > expected-tests.txt + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Missing tests:**" >> $GITHUB_STEP_SUMMARY + comm -23 expected-tests.txt all-tests-ran.txt >> $GITHUB_STEP_SUMMARY || true + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "> [!NOTE]" >> $GITHUB_STEP_SUMMARY + echo "> ✅ All $EXPECTED tests were executed successfully!" >> $GITHUB_STEP_SUMMARY + fi + + - name: Check for Duplicate Tests + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📊 Shard Distribution" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Count tests per shard + echo "| Shard | Tests Run |" >> $GITHUB_STEP_SUMMARY + echo "|-------|-----------|" >> $GITHUB_STEP_SUMMARY + for file in tests-ran-*.txt; do + shard=$(echo $file | sed 's/tests-ran-\(.*\)\.txt/\1/') + count=$(wc -l < "$file") + echo "| $shard | $count |" >> $GITHUB_STEP_SUMMARY + done + + # Show duplicates (tests that ran in multiple shards) + echo "" >> $GITHUB_STEP_SUMMARY + DUPLICATES=$(cat tests-ran-*.txt | sort | uniq -d | wc -l) + if [ "$DUPLICATES" -gt 0 ]; then + echo "> [!TIP]" >> $GITHUB_STEP_SUMMARY + echo "> $DUPLICATES tests ran in multiple shards (including catch-all). This is expected for catch-all coverage." >> $GITHUB_STEP_SUMMARY + fi + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@latest + + - name: Merge Coverage Profiles + run: | + echo "" >> $GITHUB_STEP_SUMMARY + echo "## 📈 Coverage Report" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "Found coverage files:" + ls -la cover-*.out + + gocovmerge cover-*.out > merged-coverage.out + + # Show coverage summary + go tool cover -func=merged-coverage.out | tail -20 >> $GITHUB_STEP_SUMMARY + + - name: Generate HTML Coverage Report + run: go tool cover -html=merged-coverage.out -o coverage.html + + - name: Check Coverage Threshold (75%) + run: | + COVERAGE=$(go tool cover -func=merged-coverage.out | grep total | awk '{print substr($3, 1, length($3)-1)}') + echo "Total coverage: $COVERAGE%" + EXPECTED_CODE_COV=75 - var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }') - if [ "$var" -eq 1 ];then - echo "Your code coverage is too low. Coverage precentage is: $CODE_COV" + var=$(awk 'BEGIN{ print "'$COVERAGE'"<"'$EXPECTED_CODE_COV'" }') + if [ "$var" -eq 1 ]; then + echo "❌ Coverage $COVERAGE% is below threshold $EXPECTED_CODE_COV%" + echo "" >> $GITHUB_STEP_SUMMARY + echo "> [!CAUTION]" >> $GITHUB_STEP_SUMMARY + echo "> Coverage $COVERAGE% is below the required $EXPECTED_CODE_COV% threshold!" >> $GITHUB_STEP_SUMMARY exit 1 else - echo "Your code coverage test passed! Coverage precentage is: $CODE_COV" - exit 0 + echo "✅ Coverage $COVERAGE% meets threshold" + echo "" >> $GITHUB_STEP_SUMMARY + echo "> [!NOTE]" >> $GITHUB_STEP_SUMMARY + echo "> ✅ Coverage $COVERAGE% meets the $EXPECTED_CODE_COV% threshold!" >> $GITHUB_STEP_SUMMARY fi + + - name: Upload Combined Coverage Report + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4 + with: + name: merged-coverage + path: | + merged-coverage.out + coverage.html + + - name: Publish Test Report + uses: dorny/test-reporter@v1 + if: always() + with: + name: Integration Tests Summary + path: 'test-results-*.xml' + reporter: java-junit + fail-on-error: false + + # Run cleandata to delete test projects + cleanup-test-data: + needs: aggregate-test-results + runs-on: ubuntu-latest + if: always() + steps: + - name: Checkout the repository + uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0 + + - name: Set up Go version + uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4 + with: + go-version-file: go.mod + + - name: Run Cleanup + env: + CX_BASE_URI: ${{ secrets.CX_BASE_URI }} + CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }} + CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }} + CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }} + CX_APIKEY: ${{ secrets.CX_APIKEY }} + CX_TENANT: ${{ secrets.CX_TENANT }} + run: | + echo "Running cleandata to clean up test projects..." + go test -v github.com/checkmarx/ast-cli/test/cleandata || true + + lint: name: lint runs-on: ubuntu-latest