Skip to content

Enhance integration tests with sharding strategy and improved reporting(AST-22323) #1404

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
cx-amol-mane wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Enhance integration tests with sharding strategy and improved reporting(AST-22323) #1404

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
273 changes: 255 additions & 18 deletions .github/workflows/ci-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,23 +33,72 @@ jobs:
fi
integration-tests:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
shard:
- name: scan-basic
pattern: "^Test(CreateScan|ScanCreate|ScansE2E|FastScan|LightQueries|Recommended)"
timeout: "50m"
- name: scan-advanced
pattern: "^Test(Incremental|Cancel|BranchPrimary|ScansUpdate|Invalid|ScanShow|ScanTimeout|BrokenLink|ScanWorkflow|ScanLog|PartialScan|FailedScan|ScanFilter|RunKics|ScanType|ScanGenerating|ScanWith|ScanList)"
timeout: "50m"
- name: results
pattern: "^Test(Result|CodeBashing|BFL|RiskManagement)"
timeout: "40m"
- name: auth-project
pattern: "^Test(Auth|Project|Root|Groups|Tenant|SetLog|Main)"
timeout: "30m"
- name: realtime-containers
pattern: "^Test(Container|Realtime|Iac|Oss|Secrets|Asca|Sca)"
timeout: "40m"
- name: utilities
pattern: "^Test(PR|Predicate|Import|LearnMore|Configuration|UserCount|Telemetry|PreCommit|PreReceive|Azure|Bitbucket|GitHub|GitLab|Remediation|Mask)"
timeout: "40m"
- name: catch-all
pattern: "."
timeout: "60m"

name: Integration Tests (${{ matrix.shard.name }})
steps:
- name: Checkout the repository
uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

- name: Set up Go version
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4
with:
go-version-file: go.mod

- run: go version

- name: Go Build
run: go build -o ./bin/cx ./cmd
- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest

- name: Install gotestsum
run: go install gotest.tools/gotestsum@latest

- name: Install pre-commit
run: |
pip install pre-commit
pre-commit install
- name: Go Integration test

- name: Start Squid Proxy
run: |
docker run \
--name squid \
-d \
-p 3128:3128 \
-v $(pwd)/internal/commands/.scripts/squid/squid.conf:/etc/squid/squid.conf \
-v $(pwd)/internal/commands/.scripts/squid/passwords:/etc/squid/passwords \
ubuntu/squid:5.2-22.04_beta

- name: Download ScaResolver
run: |
wget https://sca-downloads.s3.amazonaws.com/cli/latest/ScaResolver-linux64.tar.gz
tar -xzvf ScaResolver-linux64.tar.gz -C /tmp
rm -rf ScaResolver-linux64.tar.gz

- name: Run Integration Tests (${{ matrix.shard.name }})
shell: bash
env:
CX_BASE_URI: ${{ secrets.CX_BASE_URI }}
Expand Down Expand Up @@ -94,30 +143,218 @@ jobs:
PR_BITBUCKET_NAMESPACE: "AstSystemTest"
PR_BITBUCKET_REPO_NAME: "cliIntegrationTest"
PR_BITBUCKET_ID: 1
SCA_RESOLVER: /tmp/ScaResolver
run: |
sudo chmod +x ./internal/commands/.scripts/integration_up.sh ./internal/commands/.scripts/integration_down.sh
./internal/commands/.scripts/integration_up.sh
./internal/commands/.scripts/integration_down.sh

- name: Coverage report
gotestsum \
--format standard-verbose \
--junitfile test-results-${{ matrix.shard.name }}.xml \
--jsonfile test-results-${{ matrix.shard.name }}.json \
-- \
-tags integration \
-v \
-timeout ${{ matrix.shard.timeout }} \
-run '${{ matrix.shard.pattern }}' \
-coverpkg github.com/checkmarx/ast-cli/internal/commands,github.com/checkmarx/ast-cli/internal/services,github.com/checkmarx/ast-cli/internal/wrappers \
-coverprofile cover-${{ matrix.shard.name }}.out \
github.com/checkmarx/ast-cli/test/integration

- name: Extract Test Names for Validation
if: always()
run: |
grep 'testcase name=' test-results-${{ matrix.shard.name }}.xml | \
sed 's/.*name="\([^"]*\)".*/\1/' > tests-ran-${{ matrix.shard.name }}.txt || true
echo "Tests run in ${{ matrix.shard.name }} shard:"
wc -l tests-ran-${{ matrix.shard.name }}.txt

- name: Upload Test Results
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
if: always()
with:
name: test-results-${{ matrix.shard.name }}
path: |
test-results-${{ matrix.shard.name }}.xml
test-results-${{ matrix.shard.name }}.json
tests-ran-${{ matrix.shard.name }}.txt

- name: Upload Coverage
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
if: always()
with:
name: ${{ runner.os }}-coverage-latest
path: coverage.html
name: coverage-${{ matrix.shard.name }}
path: cover-${{ matrix.shard.name }}.out

- name: Check if total coverage is greater then 75
shell: bash
- name: Stop Squid Proxy
if: always()
run: docker stop squid || true

# Aggregate results from all shards
aggregate-test-results:
needs: integration-tests
runs-on: ubuntu-latest
if: always()
steps:
- name: Checkout the repository
uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

- name: Set up Go version
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4
with:
go-version-file: go.mod

- name: Download All Test Results
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4
with:
pattern: test-results-*
merge-multiple: true

- name: Download All Coverage Files
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 #v4
with:
pattern: coverage-*
merge-multiple: true

- name: Validate All Tests Ran
run: |
CODE_COV=$(go tool cover -func cover.out | grep total | awk '{print substr($3, 1, length($3)-1)}')
echo "## 🔍 Test Completeness Validation" >> $GITHUB_STEP_SUMMARY

# Count expected tests from source
EXPECTED=$(grep -r "^func Test" test/integration/*.go | wc -l)
echo "Expected tests from source: $EXPECTED"

# Count unique tests that actually ran (excluding duplicates from catch-all)
cat tests-ran-*.txt | sort -u > all-tests-ran.txt
ACTUAL=$(wc -l < all-tests-ran.txt)
echo "Unique tests executed: $ACTUAL"

echo "| Metric | Count |" >> $GITHUB_STEP_SUMMARY
echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY
echo "| Expected Tests | $EXPECTED |" >> $GITHUB_STEP_SUMMARY
echo "| Unique Tests Run | $ACTUAL |" >> $GITHUB_STEP_SUMMARY

if [ "$EXPECTED" -ne "$ACTUAL" ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
echo "> Test count mismatch! Expected $EXPECTED but ran $ACTUAL" >> $GITHUB_STEP_SUMMARY

# Show missing tests
grep -rh "^func Test" test/integration/*.go | \
sed 's/func \(Test[^(]*\).*/\1/' | sort > expected-tests.txt
echo "" >> $GITHUB_STEP_SUMMARY
echo "**Missing tests:**" >> $GITHUB_STEP_SUMMARY
comm -23 expected-tests.txt all-tests-ran.txt >> $GITHUB_STEP_SUMMARY || true
else
echo "" >> $GITHUB_STEP_SUMMARY
echo "> [!NOTE]" >> $GITHUB_STEP_SUMMARY
echo "> ✅ All $EXPECTED tests were executed successfully!" >> $GITHUB_STEP_SUMMARY
fi

- name: Check for Duplicate Tests
run: |
echo "" >> $GITHUB_STEP_SUMMARY
echo "## 📊 Shard Distribution" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY

# Count tests per shard
echo "| Shard | Tests Run |" >> $GITHUB_STEP_SUMMARY
echo "|-------|-----------|" >> $GITHUB_STEP_SUMMARY
for file in tests-ran-*.txt; do
shard=$(echo $file | sed 's/tests-ran-\(.*\)\.txt/\1/')
count=$(wc -l < "$file")
echo "| $shard | $count |" >> $GITHUB_STEP_SUMMARY
done

# Show duplicates (tests that ran in multiple shards)
echo "" >> $GITHUB_STEP_SUMMARY
DUPLICATES=$(cat tests-ran-*.txt | sort | uniq -d | wc -l)
if [ "$DUPLICATES" -gt 0 ]; then
echo "> [!TIP]" >> $GITHUB_STEP_SUMMARY
echo "> $DUPLICATES tests ran in multiple shards (including catch-all). This is expected for catch-all coverage." >> $GITHUB_STEP_SUMMARY
fi

- name: Install gocovmerge
run: go install github.com/wadey/gocovmerge@latest

- name: Merge Coverage Profiles
run: |
echo "" >> $GITHUB_STEP_SUMMARY
echo "## 📈 Coverage Report" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY

echo "Found coverage files:"
ls -la cover-*.out

gocovmerge cover-*.out > merged-coverage.out

# Show coverage summary
go tool cover -func=merged-coverage.out | tail -20 >> $GITHUB_STEP_SUMMARY

- name: Generate HTML Coverage Report
run: go tool cover -html=merged-coverage.out -o coverage.html

- name: Check Coverage Threshold (75%)
run: |
COVERAGE=$(go tool cover -func=merged-coverage.out | grep total | awk '{print substr($3, 1, length($3)-1)}')
echo "Total coverage: $COVERAGE%"

EXPECTED_CODE_COV=75
var=$(awk 'BEGIN{ print "'$CODE_COV'"<"'$EXPECTED_CODE_COV'" }')
if [ "$var" -eq 1 ];then
echo "Your code coverage is too low. Coverage precentage is: $CODE_COV"
var=$(awk 'BEGIN{ print "'$COVERAGE'"<"'$EXPECTED_CODE_COV'" }')
if [ "$var" -eq 1 ]; then
echo "❌ Coverage $COVERAGE% is below threshold $EXPECTED_CODE_COV%"
echo "" >> $GITHUB_STEP_SUMMARY
echo "> [!CAUTION]" >> $GITHUB_STEP_SUMMARY
echo "> Coverage $COVERAGE% is below the required $EXPECTED_CODE_COV% threshold!" >> $GITHUB_STEP_SUMMARY
exit 1
else
echo "Your code coverage test passed! Coverage precentage is: $CODE_COV"
exit 0
echo "✅ Coverage $COVERAGE% meets threshold"
echo "" >> $GITHUB_STEP_SUMMARY
echo "> [!NOTE]" >> $GITHUB_STEP_SUMMARY
echo "> ✅ Coverage $COVERAGE% meets the $EXPECTED_CODE_COV% threshold!" >> $GITHUB_STEP_SUMMARY
fi

- name: Upload Combined Coverage Report
uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 #v4
with:
name: merged-coverage
path: |
merged-coverage.out
coverage.html

- name: Publish Test Report
uses: dorny/test-reporter@v1
if: always()
with:
name: Integration Tests Summary
path: 'test-results-*.xml'
reporter: java-junit
fail-on-error: false

# Run cleandata to delete test projects
cleanup-test-data:
needs: aggregate-test-results
runs-on: ubuntu-latest
if: always()
steps:
- name: Checkout the repository
uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2 #v4.0.0

- name: Set up Go version
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 #v4
with:
go-version-file: go.mod

- name: Run Cleanup
env:
CX_BASE_URI: ${{ secrets.CX_BASE_URI }}
CX_CLIENT_ID: ${{ secrets.CX_CLIENT_ID }}
CX_CLIENT_SECRET: ${{ secrets.CX_CLIENT_SECRET }}
CX_BASE_AUTH_URI: ${{ secrets.CX_BASE_AUTH_URI }}
CX_APIKEY: ${{ secrets.CX_APIKEY }}
CX_TENANT: ${{ secrets.CX_TENANT }}
run: |
echo "Running cleandata to clean up test projects..."
go test -v github.com/checkmarx/ast-cli/test/cleandata || true


lint:
name: lint
runs-on: ubuntu-latest
Expand Down
Loading