Skip to content

Commit 6d2e27e

Browse files
juliojimenezjuliojimenez
authored
lib/github.sh, GitHub SBOM Download Retry, Licenses 2025-08-01 (#32)
* lib/github.sh, GitHub SBOM Download Retry, Licenses 2025-08-01 Signed-off-by: Julio Jimenez <julio@clickhouse.com> * lib/github.sh, GitHub SBOM Download Retry, Licenses 2025-08-01 Signed-off-by: Julio Jimenez <julio@clickhouse.com> * github download retry logic Signed-off-by: Julio Jimenez <julio@clickhouse.com> * licenses Signed-off-by: Julio Jimenez <julio@clickhouse.com> --------- Signed-off-by: Julio Jimenez <julio@clickhouse.com>
1 parent 9415e12 commit 6d2e27e

File tree

6 files changed

+218
-82
lines changed

6 files changed

+218
-82
lines changed

‎README.md‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,7 @@ jobs:
137137
aws-region: us-east-1
138138

139139
- name: Upload SBOM
140-
uses: ClickHouse/ClickBom@v1.0.6
140+
uses: ClickHouse/ClickBom@v1.0.7
141141
with:
142142
github-token: ${{ secrets.GITHUB_TOKEN }}
143143
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -180,7 +180,7 @@ jobs:
180180
aws-region: us-east-1
181181

182182
- name: Upload SBOM
183-
uses: ClickHouse/ClickBom@v1.0.6
183+
uses: ClickHouse/ClickBom@v1.0.7
184184
with:
185185
github-token: ${{ secrets.GITHUB_TOKEN }}
186186
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -234,7 +234,7 @@ jobs:
234234
aws-region: us-east-1
235235

236236
- name: Upload SBOM
237-
uses: ClickHouse/ClickBom@v1.0.6
237+
uses: ClickHouse/ClickBom@v1.0.7
238238
with:
239239
github-token: ${{ steps.generate-token.outputs.token }}
240240
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -299,7 +299,7 @@ jobs:
299299
aws-region: us-east-1
300300

301301
- name: Upload SBOM
302-
uses: ClickHouse/ClickBom@v1.0.6
302+
uses: ClickHouse/ClickBom@v1.0.7
303303
with:
304304
github-token: ${{ steps.generate-token.outputs.token }}
305305
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -363,7 +363,7 @@ jobs:
363363
aws-region: us-east-1
364364

365365
- name: Upload SBOM
366-
uses: ClickHouse/ClickBom@v1.0.6
366+
uses: ClickHouse/ClickBom@v1.0.7
367367
with:
368368
github-token: ${{ steps.generate-token.outputs.token }}
369369
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -405,7 +405,7 @@ jobs:
405405
aws-region: us-east-1
406406

407407
- name: Upload SBOM
408-
uses: ClickHouse/ClickBom@v1.0.6
408+
uses: ClickHouse/ClickBom@v1.0.7
409409
with:
410410
github-token: ${{ steps.generate-token.outputs.token }}
411411
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -459,7 +459,7 @@ jobs:
459459
aws-region: us-east-1
460460
461461
- name: Merge Production SBOMs Only
462-
uses: ClickHouse/ClickBom@v1.0.6
462+
uses: ClickHouse/ClickBom@v1.0.7
463463
with:
464464
github-token: ${{ steps.generate-token.outputs.token }}
465465
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
@@ -514,7 +514,7 @@ jobs:
514514
aws-region: us-east-1
515515
516516
- name: Upload SBOM from Mend
517-
uses: ClickHouse/ClickBom@v1.0.6
517+
uses: ClickHouse/ClickBom@v1.0.7
518518
with:
519519
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
520520
aws-secret-access-key: ${{ steps.aws-creds.outputs.aws-secret-access-key }}
@@ -565,7 +565,7 @@ jobs:
565565
aws-region: us-east-1
566566
567567
- name: Upload SBOM from Wiz
568-
uses: ClickHouse/ClickBom@v1.0.6
568+
uses: ClickHouse/ClickBom@v1.0.7
569569
with:
570570
aws-access-key-id: ${{ steps.aws-creds.outputs.aws-access-key-id }}
571571
aws-secret-access-key: ${{ steps.aws-creds.outputs.aws-secret-access-key }}

‎entrypoint.sh‎

Lines changed: 1 addition & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -9,79 +9,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
99
source "$SCRIPT_DIR/lib/sanitize.sh"
1010
source "$SCRIPT_DIR/lib/common.sh"
1111
source "$SCRIPT_DIR/lib/validation.sh"
12-
13-
# Download SBOM from GitHub repository
14-
download_sbom() {
15-
local repo="$1"
16-
local output_file="$2"
17-
18-
log_info "Downloading SBOM from $repo"
19-
20-
# GitHub API URL for SBOM
21-
local api_url="https://api.github.com/repos/$repo/dependency-graph/sbom"
22-
23-
# Authentication header
24-
local auth_header="Authorization: Bearer $GITHUB_TOKEN"
25-
26-
# Download SBOM file with optimizations for large files
27-
log_info "Starting SBOM download (may take time for large files)..."
28-
29-
if curl -L \
30-
--max-time 300 \
31-
--connect-timeout 30 \
32-
--retry 3 \
33-
--retry-delay 5 \
34-
--retry-max-time 180 \
35-
--silent \
36-
--show-error \
37-
--compressed \
38-
-H "Accept: application/vnd.github+json" \
39-
-H "$auth_header" \
40-
-H "X-GitHub-Api-Version: 2022-11-28" \
41-
"$api_url" \
42-
-o "$output_file"; then
43-
# Verify the download
44-
if [[ -f "$output_file" && -s "$output_file" ]]; then
45-
local file_size
46-
file_size=$(du -h "$output_file" | cut -f1)
47-
log_success "SBOM downloaded successfully ($file_size)"
48-
49-
# Debug: Show first few lines of downloaded content
50-
log_debug "First 200 characters of downloaded content:"
51-
if [[ "${DEBUG:-false}" == "true" ]]; then
52-
head -c 200 "$output_file" | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g'
53-
echo ""
54-
fi
55-
56-
# Quick validation that it's JSON
57-
if ! jq . "$output_file" > /dev/null 2>&1; then
58-
log_error "Downloaded file is not valid JSON"
59-
log_error "Content preview:"
60-
head -n 5 "$output_file" || cat "$output_file"
61-
exit 1
62-
fi
63-
64-
# Check if it looks like an error response
65-
if jq -e '.message' "$output_file" > /dev/null 2>&1; then
66-
local error_message
67-
error_message=$(jq -r '.message' "$output_file")
68-
log_error "GitHub API returned error: $error_message"
69-
exit 1
70-
fi
71-
else
72-
log_error "Downloaded file is empty or missing"
73-
exit 1
74-
fi
75-
else
76-
log_error "Failed to download SBOM file"
77-
log_error "This could be due to:"
78-
log_error " - Network timeout (file too large)"
79-
log_error " - Authentication issues"
80-
log_error " - Repository doesn't have dependency graph enabled"
81-
log_error " - SBOM not available for this repository"
82-
exit 1
83-
fi
84-
}
12+
source "$SCRIPT_DIR/lib/github.sh"
8513

8614
# Authenticate with Mend API and get JWT token
8715
authenticate_mend() {

‎lib/github.sh‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#!/bin/bash
2+
# GitHub SBOM download and processing
3+
4+
source "$(dirname "${BASH_SOURCE[0]}")/common.sh"
5+
6+
# Download SBOM from GitHub repository
7+
download_sbom() {
8+
local repo="$1"
9+
local output_file="$2"
10+
local max_attempts=3
11+
local base_delay=30
12+
13+
log_info "Downloading SBOM from $repo"
14+
15+
# GitHub API URL for SBOM
16+
local api_url="https://api.github.com/repos/$repo/dependency-graph/sbom"
17+
18+
# Authentication header
19+
local auth_header="Authorization: Bearer $GITHUB_TOKEN"
20+
21+
for attempt in $(seq 1 $max_attempts); do
22+
# Download SBOM file with optimizations for large files
23+
log_info "Starting SBOM download $attempt/$max_attempts (may take time for large files)..."
24+
25+
# Calculate delay for this attempt (exponential backoff)
26+
local delay=$((base_delay * attempt))
27+
28+
if curl -L \
29+
--max-time 600 \
30+
--connect-timeout 60 \
31+
--retry 2 \
32+
--retry-delay 10 \
33+
--retry-max-time 120 \
34+
--silent \
35+
--show-error \
36+
--compressed \
37+
-H "Accept: application/vnd.github+json" \
38+
-H "$auth_header" \
39+
-H "X-GitHub-Api-Version: 2022-11-28" \
40+
"$api_url" \
41+
-o "$output_file"; then
42+
# Verify the download
43+
if [[ -f "$output_file" && -s "$output_file" ]]; then
44+
local file_size
45+
file_size=$(du -h "$output_file" | cut -f1)
46+
log_success "SBOM downloaded successfully ($file_size) on attempt $attempt"
47+
48+
# Debug: Show first few lines of downloaded content
49+
log_debug "First 200 characters of downloaded content:"
50+
if [[ "${DEBUG:-false}" == "true" ]]; then
51+
head -c 200 "$output_file" | tr '\n' ' ' | sed 's/[[:space:]]\+/ /g'
52+
echo ""
53+
fi
54+
55+
# Quick validation that it's JSON
56+
if ! jq . "$output_file" > /dev/null 2>&1; then
57+
log_warning "Downloaded file is not valid JSON on attempt $attempt"
58+
log_error "Content preview:"
59+
head -n 5 "$output_file" || cat "$output_file"
60+
61+
# If not last attempt, continue to retry
62+
if [[ $attempt -lt $max_attempts ]]; then
63+
log_info "Invalid JSON received, waiting ${delay} seconds before retry..."
64+
sleep $delay
65+
continue
66+
else
67+
log_error "Downloaded file is not valid JSON after all attempts"
68+
exit 1
69+
fi
70+
fi
71+
72+
# Check if it looks like an error response
73+
if jq -e '.message' "$output_file" > /dev/null 2>&1; then
74+
local error_message
75+
error_message=$(jq -r '.message' "$output_file")
76+
# Check if it's a timeout or generation error that we can retry
77+
if [[ "$error_message" =~ "Request timed out" ]] || [[ "$error_message" =~ "Failed to generate SBOM" ]] || [[ "$error_message" =~ "timeout" ]]; then
78+
log_warning "GitHub SBOM generation timed out on attempt $attempt: $error_message"
79+
80+
if [[ $attempt -lt $max_attempts ]]; then
81+
log_info "GitHub's SBOM generation timed out, waiting ${delay} seconds before retry..."
82+
sleep $delay
83+
continue
84+
else
85+
log_error "GitHub SBOM generation failed after $max_attempts attempts"
86+
log_error "Final error: $error_message"
87+
log_error "This repository may be too large or complex for GitHub's SBOM generation"
88+
log_error "Possible solutions:"
89+
log_error " - Try again later when GitHub's service load is lower"
90+
log_error " - Consider using alternative SBOM sources (Mend, Wiz)"
91+
log_error " - Break down the repository analysis into smaller components"
92+
exit 1
93+
fi
94+
else
95+
# Non-retryable error
96+
log_error "GitHub API returned error: $error_message"
97+
exit 1
98+
fi
99+
fi
100+
101+
# Success - SBOM downloaded and validated
102+
log_success "SBOM download completed successfully"
103+
return 0
104+
else
105+
log_error "Downloaded file is empty or missing on attempt $attempt"
106+
if [[ $attempt -lt $max_attempts ]]; then
107+
log_info "Empty file received, waiting ${delay} seconds before retry..."
108+
sleep $delay
109+
continue
110+
else
111+
log_error "Downloaded file is empty or missing after all attempts"
112+
exit 1
113+
fi
114+
fi
115+
else
116+
local curl_exit_code=$?
117+
log_warning "Curl failed on attempt $attempt with exit code: $curl_exit_code"
118+
119+
if [[ $attempt -lt $max_attempts ]]; then
120+
log_info "Network request failed, waiting ${delay} seconds before retry..."
121+
sleep $delay
122+
continue
123+
else
124+
log_error "Failed to download SBOM file after $max_attempts attempts"
125+
log_error "This could be due to:"
126+
log_error " - Repository is too large for GitHub's SBOM generation (common cause)"
127+
log_error " - GitHub's SBOM service is experiencing high load or issues"
128+
log_error " - Network connectivity problems"
129+
log_error " - Authentication issues with the provided token"
130+
log_error " - Repository doesn't have dependency graph enabled"
131+
log_error " - SBOM feature not available for this repository type"
132+
exit 1
133+
fi
134+
fi
135+
done
136+
}

‎license-mappings.json‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -616,18 +616,88 @@
616616
"github.com/mattn/go-tty": "MIT",
617617
"github.com/matttproud/golang_protobuf_extensions": "Apache-2.0",
618618
"github.com/mdlayher/netlink": "MIT",
619+
"github.com/mdlayher/socket": "MIT",
620+
"github.com/mdlayher/vsock": "MIT",
621+
"github.com/mfridman/interpolate": "MIT",
622+
"github.com/mfridman/xflag": "MIT",
623+
"github.com/mgechev/revive": "MIT",
624+
"github.com/microcosm-cc/bluemonday": "BSD-3-Clause",
625+
"github.com/microsoft/go-mssqldb": "BSD-3-Clause",
619626
"github.com/Microsoft/go-winio": "MIT",
620627
"github.com/Microsoft/hcsshim": "MIT",
628+
"github.com/miekg/dns": "BSD-3-Clause",
629+
"github.com/minio/md5-simd": "Apache-2.0",
630+
"github.com/minio/minio-go/v7": "Apache-2.0",
631+
"github.com/minio/sha256-simd": "Apache-2.0",
632+
"github.com/mitchellh/colorstring": "MIT",
633+
"github.com/mitchellh/copystructure": "MIT",
634+
"github.com/mitchellh/go-homedir": "MIT",
635+
"github.com/mitchellh/go-ps": "MIT",
636+
"github.com/mitchellh/go-wordwrap": "MIT",
637+
"github.com/mitchellh/hashstructure": "MIT",
638+
"github.com/mitchellh/mapstructure": "MIT",
639+
"github.com/mitchellh/reflectwalk": "MIT",
621640
"github.com/mkevac/debugcharts": "MIT",
622641
"github.com/moby/docker-image-spec": "Apache-2.0",
623642
"github.com/moby/go-archive": "Apache-2.0",
643+
"github.com/moby/locker": "Apache-2.0",
624644
"github.com/moby/patternmatcher": "Apache-2.0",
645+
"github.com/moby/spdystream": "Apache-2.0",
646+
"github.com/moby/sys/atomicwriter": "Apache-2.0",
625647
"github.com/moby/sys/sequential": "Apache-2.0",
626648
"github.com/moby/sys/user": "Apache-2.0",
627649
"github.com/moby/sys/userns": "Apache-2.0",
628650
"github.com/moby/term": "Apache-2.0",
651+
"github.com/modern-go/concurrent": "Apache-2.0",
652+
"github.com/modern-go/reflect2": "Apache-2.0",
653+
"github.com/monochromegane/go-gitignore": "MIT",
654+
"github.com/moricho/tparallel": "MIT",
629655
"github.com/morikuni/aec": "MIT",
656+
"github.com/mostynb/go-grpc-compression": "MIT",
657+
"github.com/motemen/go-loghttp": "MIT",
658+
"github.com/motemen/go-nuts": "MIT",
659+
"github.com/mozillazg/go-httpheader": "MIT",
660+
"github.com/muesli/ansi": "MIT",
661+
"github.com/muesli/cancelreader": "MIT",
662+
"github.com/muesli/reflow": "MIT",
663+
"github.com/muesli/termenv": "MIT",
664+
"github.com/munnerz/goautoneg": "BSD-3-Clause",
665+
"github.com/mwitkow/go-conntrack": "Apache-2.0",
666+
"github.com/mxk/go-flowrate": "BSD-3-Clause",
667+
"github.com/nakabonne/nestif": "BSD-2-Clause",
668+
"github.com/ncruces/go-strftime": "MIT",
669+
"github.com/ncw/swift": "MIT",
670+
"github.com/neilotoole/jsoncolor": "MIT",
671+
"github.com/nexus-rpc/sdk-go": "MIT",
672+
"github.com/nicksnyder/go-i18n/v2": "MIT",
630673
"github.com/NimbleMarkets/ntcharts": "MIT",
674+
"github.com/nishanths/exhaustive": "BSD-2-Clause",
675+
"github.com/nishanths/predeclared": "BSD-3-Clause",
676+
"github.com/nqd/flat": "MIT",
677+
"github.com/nunnatsa/ginkgolinter": "MIT",
678+
"github.com/oapi-codegen/runtime": "Apache-2.0",
679+
"github.com/oklog/run": "Apache-2.0",
680+
"github.com/oklog/ulid": "Apache-2.0",
681+
"github.com/olekukonko/tablewriter": "MIT",
682+
"github.com/olivere/elastic/v7": "MIT",
683+
"github.com/onsi/ginkgo": "MIT",
684+
"github.com/onsi/ginkgo/v2": "MIT",
685+
"github.com/onsi/gomega": "MIT",
686+
"github.com/open-telemetry/opentelemetry-collector-contrib/connector/routingconnector": "Apache-2.0",
687+
"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter": "Apache-2.0",
688+
"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/fileexporter": "Apache-2.0",
689+
"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusexporter": "Apache-2.0",
690+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/healthcheckextension": "Apache-2.0",
691+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/k8sleaderelector": "Apache-2.0",
692+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/oidcauthextension": "Apache-2.0",
693+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/pprofextension": "Apache-2.0",
694+
"github.com/open-telemetry/opentelemetry-collector-contrib/extension/storage/filestorage": "Apache-2.0",
695+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil": "Apache-2.0",
696+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/common": "Apache-2.0",
697+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal": "Apache-2.0",
698+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/filter": "Apache-2.0",
699+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/k8sconfig": "Apache-2.0",
700+
"github.com/open-telemetry/opentelemetry-collector-contrib/internal/metadataproviders": "Apache-2.0",
631701
"github.com/opencontainers/go-digest": "Apache-2.0",
632702
"github.com/opencontainers/image-spec": "Apache-2.0",
633703
"github.com/OpenPeeDeeP/depguard/v2": "GPL-3.0",

‎test/advanced.bats‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ setup() {
1919
sed -i "s|source \"\$SCRIPT_DIR/lib/sanitize.sh\"|source \"$PROJECT_ROOT/lib/sanitize.sh\"|" "$TEST_SCRIPT"
2020
sed -i "s|source \"\$SCRIPT_DIR/lib/common.sh\"|source \"$PROJECT_ROOT/lib/common.sh\"|" "$TEST_SCRIPT"
2121
sed -i "s|source \"\$SCRIPT_DIR/lib/validation.sh\"|source \"$PROJECT_ROOT/lib/validation.sh\"|" "$TEST_SCRIPT"
22+
sed -i "s|source \"\$SCRIPT_DIR/lib/github.sh\"|source \"$PROJECT_ROOT/lib/github.sh\"|" "$TEST_SCRIPT"
2223

2324
# Source the functions
2425
source "$TEST_SCRIPT"

0 commit comments

Comments
 (0)