Added paralelism to the offline archive build pipeline

Hahihula · Hahihula · commit 3a1f9bcba358 · 2025-09-23T09:07:58.000+02:00
diff --git a/.github/workflows/build_offline_installer_archives.yaml b/.github/workflows/build_offline_installer_archives.yaml
@@ -41,25 +41,91 @@ on:
     types: [published]
 
 jobs:
+  # New setup job to determine build matrix
+  setup-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.generate-matrix.outputs.matrix }}
+      should_purge: ${{ steps.generate-matrix.outputs.should_purge }}
+    steps:
+      - name: Checkout (for scripts if needed)
+        uses: actions/checkout@v4
+
+      - name: Download offline_installer_builder artifact
+        uses: actions/download-artifact@v5
+        with:
+          pattern: offline_installer_builder-linux-x64-*
+          merge-multiple: true
+          path: ./
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ inputs.run_id || github.run_id }}
+
+      - name: Make binary executable
+        run: chmod +x ./offline_installer_builder
+
+      - name: Install UV (Python package manager)
+        run: cargo install --git https://github.com/astral-sh/uv uv
+
+      - name: Generate build matrix
+        id: generate-matrix
+        run: |
+          # Get list of available IDF versions from the builder
+          if [ -n "${{ inputs.idf_version }}" ]; then
+            # Single version specified
+            IDF_VERSIONS='["${{ inputs.idf_version }}"]'
+          else
+            # Get all available versions from the builder
+            echo "Getting available IDF versions..."
+            VERSIONS_OUTPUT=$(./offline_installer_builder --list-versions)
+            echo "Raw versions output:"
+            echo "$VERSIONS_OUTPUT"
+
+            # Convert newline-separated to JSON array
+            IDF_VERSIONS=$(echo "$VERSIONS_OUTPUT" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+            echo "Parsed versions JSON:"
+            echo "$IDF_VERSIONS"
+          fi
+
+          # Platform configurations
+          PLATFORMS='[
+            {"os": "ubuntu-latest", "package_name": "linux-x64"},
+            {"os": "ubuntu-24.04-arm", "package_name": "linux-aarch64"},
+            {"os": "windows-latest", "package_name": "windows-x64"},
+            {"os": "macos-latest", "package_name": "macos-aarch64"},
+            {"os": "macos-13", "package_name": "macos-x64"}
+          ]'
+
+          # Create matrix combining versions and platforms
+          MATRIX=$(jq -n \
+            --argjson versions "$IDF_VERSIONS" \
+            --argjson platforms "$PLATFORMS" \
+            '{
+              include: [
+                $versions[] as $version |
+                $platforms[] |
+                . + {"idf_version": $version}
+              ]
+            }')
+
+          echo "Generated matrix:"
+          echo "$MATRIX" | jq .
+
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+
+          # Determine if we should purge
+          SHOULD_PURGE=$([ "${{ github.event_name }}" = "release" ] || [ "${{ inputs.purge_all }}" = "true" ] && echo "true" || echo "false")
+          echo "should_purge=$SHOULD_PURGE" >> $GITHUB_OUTPUT
+
+  # Modified build job with dynamic matrix
   build-and-upload:
-    name: Build & Upload (${{ matrix.package_name }})
+    name: Build & Upload (${{ matrix.package_name }}, ${{ matrix.idf_version }})
+    needs: setup-matrix
     runs-on: ${{ matrix.os }}
     continue-on-error: false
 
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - os: ubuntu-latest
-            package_name: linux-x64
-          - os: ubuntu-24.04-arm
-            package_name: linux-aarch64
-          - os: windows-latest
-            package_name: windows-x64
-          - os: macos-latest
-            package_name: macos-aarch64
-          - os: macos-13
-            package_name: macos-x64
+      matrix: ${{ fromJson(needs.setup-matrix.outputs.matrix) }}
 
     steps:
       - name: Checkout (for scripts if needed)
@@ -76,12 +142,10 @@ jobs:
 
       - name: Make binary executable (Unix)
         if: runner.os != 'Windows'
-        run: |
-          chmod +x ./offline_installer_builder
+        run: chmod +x ./offline_installer_builder
 
       - name: Install UV (Python package manager)
-        run: |
-          cargo install --git https://github.com/astral-sh/uv uv
+        run: cargo install --git https://github.com/astral-sh/uv uv
 
       - name: Set up AWS CLI
         uses: aws-actions/configure-aws-credentials@v4
@@ -127,18 +191,36 @@ jobs:
           Write-Host "Current offline_archives.json:"
           Get-Content "./offline_archives.json"
 
-      - name: Purge existing archives (if requested)
-        if: github.event_name == 'release' || inputs.purge_all == true
+      - name: Purge existing archives for this version (if requested)
+        if: needs.setup-matrix.outputs.should_purge == 'true'
         run: |
-          echo "Purging existing archives from S3..."
-          jq -r '.[] | .filename' ./offline_archives.json | while read filename; do
-            if [ -n "$filename" ]; then
-              echo "Deleting s3://espdldata/dl/eim/$filename"
-              aws s3 rm "s3://espdldata/dl/eim/$filename" || echo "Failed to delete $filename, continuing..."
-            fi
-          done
+          echo "Purging existing archives for version ${{ matrix.idf_version }} from S3..."
 
-      - name: Build archives
+          # Only purge archives for THIS specific version and platform
+          PATTERN="archive_v${{ matrix.idf_version }}_${{ matrix.package_name }}.zst"
+
+          if [ "${{ runner.os }}" = "Windows" ]; then
+            # PowerShell version
+            $archives = aws s3 ls s3://espdldata/dl/eim/ | Where-Object { $_.Split()[-1] -match "archive_v${{ matrix.idf_version }}_${{ matrix.package_name }}\.zst" }
+            foreach ($archive in $archives) {
+              $filename = $archive.Split()[-1]
+              if ($filename) {
+                Write-Host "Deleting s3://espdldata/dl/eim/$filename"
+                aws s3 rm "s3://espdldata/dl/eim/$filename"
+              }
+            }
+          else
+            # Bash version
+            aws s3 ls s3://espdldata/dl/eim/ | grep "archive_v${{ matrix.idf_version }}_${{ matrix.package_name }}\.zst" | awk '{print $4}' | while read filename; do
+              if [ -n "$filename" ]; then
+                echo "Deleting s3://espdldata/dl/eim/$filename"
+                aws s3 rm "s3://espdldata/dl/eim/$filename" || echo "Failed to delete $filename, continuing..."
+              fi
+            done
+          fi
+        shell: ${{ runner.os == 'Windows' && 'pwsh' || 'bash' }}
+
+      - name: Build archive for specific version
         id: build
         run: |
           # Determine binary name
@@ -148,19 +230,12 @@ jobs:
             BINARY="./offline_installer_builder"
           fi
 
-          # Determine arguments
-          if [ -n "${{ inputs.idf_version }}" ]; then
-            echo "Building specific version: ${{ inputs.idf_version }}"
-            ARGS="-c default --idf-version-override ${{ inputs.idf_version }}"
-          else
-            echo "Building ALL versions"
-            ARGS="-c default --build-all-versions"
-          fi
+          echo "Building specific version: ${{ matrix.idf_version }}"
 
-          # Run it
-          $BINARY $ARGS
+          # Build only this specific version
+          $BINARY -c default --idf-version-override ${{ matrix.idf_version }}
 
-          # Collect and rename all built .zst files to include platform
+          # Rename the built file to include platform
           PLATFORM="${{ matrix.package_name }}"
           mkdir -p built_archives
 
@@ -184,50 +259,52 @@ jobs:
             echo "ERROR: No archives built!" >&2
             exit 1
           fi
-
         shell: bash
 
-      - name: Upload archives to S3 and generate build info
+      - name: Upload archive to S3 and generate build info
         id: upload
         run: |
           PLATFORM="${{ matrix.package_name }}"
-          mkdir -p build-infos
+          VERSION="${{ matrix.idf_version }}"
 
-          # Upload each .zst and record metadata
-          for archive in built_archives/*.zst; do
-            if [ ! -f "$archive" ]; then continue; fi
+          # Should have exactly one file
+          archive=$(ls built_archives/*.zst | head -1)
 
-            # Extract version from new filename: archive_v5.1.2_linux-x64.zst → 5.1.2
-            FILENAME=$(basename "$archive")
-            VERSION=$(echo "$FILENAME" | sed -E 's/archive_v([^_]+)_.*/\1/')
+          if [ ! -f "$archive" ]; then
+            echo "ERROR: No archive found!" >&2
+            exit 1
+          fi
 
-            # Get file size
-            if [ "${{ runner.os }}" = "macOS" ]; then
-              SIZE=$(stat -f %z "$archive")
-            else
-              SIZE=$(stat -c %s "$archive")
-            fi
+          FILENAME=$(basename "$archive")
 
-            # Upload to S3
-            aws s3 cp --acl=public-read "$archive" "s3://espdldata/dl/eim/$FILENAME"
+          # Get file size
+          if [ "${{ runner.os }}" = "macOS" ]; then
+            SIZE=$(stat -f %z "$archive")
+          else
+            SIZE=$(stat -c %s "$archive")
+          fi
 
-            # Generate build info
-            jq -n \
-              --arg version "$VERSION" \
-              --arg platform "$PLATFORM" \
-              --arg filename "$FILENAME" \
-              --argjson size $SIZE \
-              '{"version": $version, "platform": $platform, "filename": $filename, "size": $size}' \
-              > "build-infos/build-info-${VERSION}-${PLATFORM}.json"
+          # Upload to S3
+          aws s3 cp --acl=public-read "$archive" "s3://espdldata/dl/eim/$FILENAME"
 
-            echo "Uploaded $FILENAME ($SIZE bytes)"
-          done
+          # Generate build info
+          mkdir -p build-info
+          jq -n \
+            --arg version "$VERSION" \
+            --arg platform "$PLATFORM" \
+            --arg filename "$FILENAME" \
+            --argjson size $SIZE \
+            '{"version": $version, "platform": $platform, "filename": $filename, "size": $size}' \
+            > "build-info/build-info.json"
+
+          echo "Uploaded $FILENAME ($SIZE bytes)"
+        shell: bash
 
-      - name: Save build infos as artifacts
+      - name: Save build info as artifact
         uses: actions/upload-artifact@v4
         with:
-          name: build-infos-${{ matrix.package_name }}
-          path: build-infos/
+          name: build-info-${{ matrix.idf_version }}-${{ matrix.package_name }}
+          path: build-info/
           retention-days: 7
 
   update-json:
@@ -251,15 +328,15 @@ jobs:
       - name: Download all build infos
         uses: actions/download-artifact@v4
         with:
-          pattern: build-infos-*
+          pattern: build-info-*
           path: ./all-build-infos/
           merge-multiple: true
 
       - name: Merge and update JSON
         run: |
           # Collect all new entries
           NEW_ENTRIES="[]"
-          for info_file in all-build-infos/build-infos/*/build-info-*.json; do
+          for info_file in all-build-infos/build-info/build-info.json; do
             if [ -f "$info_file" ]; then
               echo "Processing: $info_file"
               entry=$(cat "$info_file")
@@ -271,7 +348,6 @@ jobs:
           CURRENT=$(cat offline_archives.json)
 
           # Remove existing entries that are being replaced (same platform + version)
-          # Keep entries for platforms/versions NOT rebuilt
           UPDATED=$(jq --argjson new "$NEW_ENTRIES" '
             . as $current |
             ($new | map({version, platform})) as $to_replace |
diff --git a/src-tauri/src/offline_installer_builder.rs b/src-tauri/src/offline_installer_builder.rs
@@ -368,17 +368,31 @@ struct Args {
     /// Each version will create its own .zst archive file
     #[arg(long)]
     build_all_versions: bool,
+
+    /// List all supported IDF versions in machine-readable format and exit
+    /// Output format: one version per line
+    #[arg(long)]
+    list_versions: bool,
 }
 
 #[tokio::main]
 async fn main() {
     let args = Args::parse();
+    if args.list_versions {
+        let versions = idf_im_lib::idf_versions::get_idf_names().await;
+        for version in versions {
+            println!("{}", version);
+        }
+        return;
+    }
 
     // Setup logging
     if let Err(e) = setup_logging(args.verbose) {
         error!("Failed to initialize logging: {e}");
     }
 
+
+
     if args.create_from_config.is_some() {
         info!(
             "Creating installation data from configuration file: {:?}",
