[main] Update common Docker engineering infrastructure with latest (#1282)

Co-authored-by: Logan Bussell <[email protected]>

Author: dotnet-docker-bot

eng/docker-tools/DEV-GUIDE.md

@@ -25,4 +25,6 @@
 
 !!! Changes made in this directory are subject to being overwritten by automation !!!
 
-The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools.
+The files in this directory are shared by all .NET Docker repos. If you need to make changes to these files, open an issue or submit a pull request in https://github.com/dotnet/docker-tools.
+
+For guidance on using this infrastructure, see the [Developer Guide](DEV-GUIDE.md).

eng/docker-tools/readme.md

@@ -39,6 +39,15 @@ parameters:
 - name: enableSbom
   type: boolean
   default: false
+# Network isolation policy that will be enabled for jobs. The default policy
+# allows all outbound connections except for public package feeds and known
+# malicious endpoints. If this policy breaks the build, then it can be set to
+# "Permissive" temporarily until external dependencies are resolved.
+# See the network isolation documentation for more details:
+# https://eng.ms/docs/coreai/devdiv/one-engineering-system-1es/1es-build/cloudbuild/security/1espt-network-isolation
+- name: networkIsolationPolicy
+  type: string
+  default: Permissive,CFSClean
 
 resources:
   repositories:
@@ -53,6 +62,8 @@ extends:
     baseTemplate: v1/1ES.${{ iif(contains(variables['Build.DefinitionName'], '-official'), 'Official', 'Unofficial') }}.PipelineTemplate.yml@1ESPipelineTemplates
     templateParameters:
       pool: ${{ parameters.pool }}
+      settings:
+        networkIsolationPolicy: ${{ parameters.networkIsolationPolicy }}
       sdl:
         sbom:
           enabled: ${{ parameters.enableSbom }}

eng/docker-tools/templates/1es.yml

@@ -32,6 +32,7 @@ jobs:
   - ${{ parameters.commonInitStepsForMatrixAndBuild }}
   - template: /eng/docker-tools/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
     parameters:
+      publishConfig: ${{ parameters.publishConfig }}
       cleanupDocker: true
   - ${{ parameters.customInitSteps }}
   - template: /eng/docker-tools/templates/steps/set-image-info-path-var.yml@self
@@ -51,7 +52,7 @@ jobs:
       # the environment variable for us.
       $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $env:IMAGEBUILDER_QUEUEARGS --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
       if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.buildAcr.repoPrefix }} --push"
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix ${{ parameters.publishConfig.BuildRegistry.repoPrefix }} --push"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
@@ -66,14 +67,8 @@ jobs:
     parameters:
       name: BuildImages
       displayName: Build Images
-      serviceConnections:
-      # "name" here refers to the argument name, not the service connection name.
-      # It should probably be changed to "argName".
-      - name: acr
-        id: ${{ parameters.publishConfig.buildAcr.serviceConnection.id }}
-        tenantId: ${{ parameters.publishConfig.buildAcr.serviceConnection.tenantId }}
-        clientId: ${{ parameters.publishConfig.buildAcr.serviceConnection.clientId }}
-      - ${{ if parameters.storageAccountServiceConnection }}:
+      ${{ if parameters.storageAccountServiceConnection }}:
+        serviceConnections:
         - name: storage
           id: ${{ parameters.storageAccountServiceConnection.id }}
           tenantId: ${{ parameters.storageAccountServiceConnection.tenantId }}
@@ -89,8 +84,8 @@ jobs:
         --architecture $(architecture)
         --retry
         --digests-out-var 'builtImages'
-        --acr-subscription '${{ parameters.publishConfig.buildAcr.subscription }}'
-        --acr-resource-group '${{ parameters.publishConfig.buildAcr.resourceGroup }}'
+        --acr-subscription '${{ parameters.publishConfig.BuildRegistry.subscription }}'
+        --acr-resource-group '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
         $(manifestVariables)
         $(imageBuilderBuildArgs)
   - template: /eng/docker-tools/templates/steps/publish-artifact.yml@self
@@ -104,12 +99,12 @@ jobs:
     - powershell: |
         $images = "$(BuildImages.builtImages)"
         if (-not $images) { return 0 }
-        $syftImageName = "${{ parameters.publishConfig.publicMirrorAcr.server }}/$(imageNames.syft)"
+        $syftImageName = "${{ parameters.publishConfig.PublicMirrorRegistry.server }}/$(imageNames.syft)"
         & $(engDockerToolsPath)/Pull-Image.ps1 $syftImageName
         $images -Split ',' | ForEach-Object {
           echo "Generating SBOM for $_";
           $targetImageName = "$_";
-          $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.buildAcr.server }}/${{ parameters.publishConfig.buildAcr.repoPrefix }}', "").Replace('/', '_').Replace(':', '_');
+          $formattedImageName = $targetImageName.Replace('${{ parameters.publishConfig.BuildRegistry.server }}/${{ parameters.publishConfig.BuildRegistry.repoPrefix }}', "").Replace('/', '_').Replace(':', '_');
           $sbomChildDir = "$(sbomDirectory)/$formattedImageName";
           New-Item -Type Directory -Path $sbomChildDir > $null;
           docker build --output=$sbomChildDir -f $(engDockerToolsPath)/Dockerfile.syft --build-arg SYFT_IMAGE_NAME=$syftImageName --build-arg TARGET_IMAGE_NAME=$targetImageName -t syft-sbom $(engDockerToolsPath);

eng/docker-tools/templates/jobs/build-images.yml

@@ -22,10 +22,10 @@ jobs:
     os: linux
   steps:
   - powershell: >
-      ./eng/docker-tools/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "/usr/share/.dotnet"
+      ./eng/docker-tools/Install-DotNetSdk.ps1 -Channel ${{ parameters.dotnetVersionChannel }} -InstallPath "$(Build.SourcesDirectory)/.dotnet"
     displayName: Run Dotnet Install Script
   - script: >
-      find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 /usr/share/.dotnet/dotnet build
+      find . -name '*.csproj' | grep $(cgBuildGrepArgs) | xargs -n 1 $(Build.SourcesDirectory)/.dotnet/dotnet build
     displayName: Build Projects
 
     # Component Detection is only automatically run on production branches.

eng/docker-tools/templates/jobs/cg-build-projects.yml

@@ -23,7 +23,7 @@ jobs:
   parameters:
     name: ${{ parameters.name }}
     pool: ${{ parameters.pool }}
+    publishConfig: ${{ parameters.publishConfig }}
     customInitSteps: ${{ parameters.customInitSteps }}
     additionalOptions: ${{ parameters.additionalOptions }}
-    acr: ${{ parameters.publishConfig.internalMirrorAcr }}
-    repoPrefix: ${{ parameters.publishConfig.internalMirrorAcr.repoPrefix }}
+    acr: ${{ parameters.publishConfig.InternalMirrorRegistry }}

eng/docker-tools/templates/jobs/copy-base-images-staging.yml

@@ -5,11 +5,11 @@ parameters:
 - name: pool
   type: object
   default: {}
-- name: acr
+- name: publishConfig
   type: object
   default: null
-- name: repoPrefix
-  type: string
+- name: acr
+  type: object
   default: null
 - name: customInitSteps
   type: stepList
@@ -29,11 +29,12 @@ jobs:
   pool: ${{ parameters.pool }}
   steps:
   - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
   - ${{ parameters.customInitSteps }}
   - template: /eng/docker-tools/templates/steps/copy-base-images.yml@self
     parameters:
       acr: ${{ parameters.acr }}
-      repoPrefix: ${{ parameters.repoPrefix }}
       additionalOptions: ${{ parameters.additionalOptions }}
       continueOnError: ${{ parameters.continueOnError }}
       forceDryRun: ${{ parameters.forceDryRun }}

eng/docker-tools/templates/jobs/copy-base-images.yml

@@ -63,10 +63,5 @@ jobs:
     parameters:
       name: matrix
       displayName: Generate ${{ parameters.matrixType }} Matrix
-      serviceConnections:
-      - name: acr
-        tenantId: $(build.serviceConnection.tenantId)
-        clientId: $(build.serviceConnection.clientId)
-        id: $(build.serviceConnection.id)
       internalProjectName: internal
       args: $(generateBuildMatrixCommand)

eng/docker-tools/templates/jobs/generate-matrix.yml

@@ -21,7 +21,7 @@ jobs:
   - name: imageBuilder.commonCmdArgs
     value: >-
       --manifest '$(manifest)'
-      --registry-override '${{ parameters.publishConfig.publishAcr.server }}'
+      --registry-override '${{ parameters.publishConfig.PublishRegistry.server }}'
       $(manifestVariables)
       $(imageBuilder.queueArgs)
   - name: publishNotificationRepoName
@@ -53,6 +53,8 @@ jobs:
   - template: /eng/docker-tools/templates/steps/retain-build.yml@self
 
   - template: /eng/docker-tools/templates/steps/init-docker-linux.yml@self
+    parameters:
+      publishConfig: ${{ parameters.publishConfig }}
 
   - pwsh: |
       $azdoOrgName = Split-Path -Leaf $Env:SYSTEM_COLLECTIONURI
@@ -95,21 +97,16 @@ jobs:
   - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Copy Images
-      serviceConnections:
-      - name: acr
-        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
-        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
-        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       args: >-
         copyAcrImages
-        '${{ parameters.publishConfig.buildAcr.subscription }}'
-        '${{ parameters.publishConfig.buildAcr.resourceGroup }}'
-        '${{ parameters.publishConfig.buildAcr.repoPrefix }}'
-        '${{ parameters.publishConfig.buildAcr.server }}'
+        '${{ parameters.publishConfig.BuildRegistry.subscription }}'
+        '${{ parameters.publishConfig.BuildRegistry.resourceGroup }}'
+        '${{ parameters.publishConfig.BuildRegistry.repoPrefix }}'
+        '${{ parameters.publishConfig.BuildRegistry.server }}'
         --os-type '*'
         --architecture '*'
-        --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
+        --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
         --image-info '$(imageInfoContainerDir)/image-info.json'
         $(dryRunArg)
         $(imageBuilder.pathArgs)
@@ -118,17 +115,12 @@ jobs:
   - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Publish Manifest
-      serviceConnections:
-      - name: acr
-        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
-        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
-        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: ${{ parameters.internalProjectName }}
       dockerClientOS: ${{ parameters.dockerClientOS }}
       args: >-
         publishManifest
         '$(imageInfoContainerDir)/image-info.json'
-        --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
+        --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
         --os-type '*'
         --architecture '*'
         $(dryRunArg)
@@ -208,17 +200,12 @@ jobs:
   - template: /eng/docker-tools/templates/steps/run-imagebuilder.yml@self
     parameters:
       displayName: Generate EOL Annotation Data
-      serviceConnections:
-      - name: acr
-        id: ${{ parameters.publishConfig.publishAcr.serviceConnection.id }}
-        tenantId: ${{ parameters.publishConfig.publishAcr.serviceConnection.tenantId }}
-        clientId: ${{ parameters.publishConfig.publishAcr.serviceConnection.clientId }}
       internalProjectName: internal
       condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true'))
       args: >-
         generateEolAnnotationDataForPublish
-        '${{ parameters.publishConfig.publishAcr.server }}'
-        '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
+        '${{ parameters.publishConfig.PublishRegistry.server }}'
+        '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
         '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json'
         '$(imageInfoContainerDir)/full-image-info-orig.json'
         '$(imageInfoContainerDir)/full-image-info-new.json'
@@ -236,7 +223,7 @@ jobs:
 
   - template: /eng/docker-tools/templates/steps/annotate-eol-digests.yml@self
     parameters:
-      acr: ${{ parameters.publishConfig.publishAcr }}
+      acr: ${{ parameters.publishConfig.PublishRegistry }}
       dataFile: $(artifactsPath)/eol-annotation-data/eol-annotation-data.json
 
   - script: >
@@ -275,7 +262,7 @@ jobs:
       $(gitHubNotificationsRepoInfo.authArgs)
       '$(gitHubNotificationsRepoInfo.org)'
       '$(gitHubNotificationsRepoInfo.repo)'
-      --repo-prefix '${{ parameters.publishConfig.publishAcr.repoPrefix }}'
+      --repo-prefix '${{ parameters.publishConfig.PublishRegistry.repoPrefix }}'
       --task "🟪 Copy Images"
       --task "🟪 Publish Manifest"
       --task "🟪 Wait for Image Ingestion"
@@ -284,8 +271,8 @@ jobs:
       --task "🟪 Publish Image Info"
       --task "🟪 Ingest Kusto Image Info"
       --task "🟪 Generate EOL Annotation Data"
-      --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.publishAcr.server }})"
-      --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.publishAcr.server }})"
+      --task "🟪 Annotate EOL Images (${{ parameters.publishConfig.PublishRegistry.server }})"
+      --task "🟪 Wait for Annotation Ingestion (${{ parameters.publishConfig.PublishRegistry.server }})"
       $(dryRunArg)
       $(imageBuilder.commonCmdArgs)
     displayName: Post Publish Notification

eng/docker-tools/templates/jobs/publish.yml

@@ -47,10 +47,12 @@ stages:
     internalProjectName: "internal"
     publicProjectName: "public"
 
+    # publishConfig schema is defined in src/ImageBuilder/Configuration/PublishConfiguration.cs.
+    # This will get converted to JSON and placed in appsettings.json to be loaded by ImageBuilder at runtime.
     publishConfig:
-      internalMirrorAcr:
+      InternalMirrorRegistry:
         server: $(acr-staging-test.server)
-        repoPrefix: $(mirrorRepoPrefix)
+        repoPrefix: $(internalMirrorRepoPrefix)
         resourceGroup: $(testResourceGroup)
         subscription: $(testSubscription)
         serviceConnection:
@@ -59,8 +61,9 @@ stages:
           clientId: $(internal-mirror-test.serviceConnection.clientId)
           tenantId: $(testTenant)
 
-      publicMirrorAcr:
+      PublicMirrorRegistry:
         server: $(public-mirror.server)
+        repoPrefix: $(publicMirrorRepoPrefix)
         resourceGroup: $(public-mirror.resourceGroup)
         subscription: $(public-mirror.subscription)
         serviceConnection:
@@ -69,7 +72,7 @@ stages:
           tenantId: $(public-mirror.serviceConnection.tenantId)
           clientId: $(public-mirror.serviceConnection.clientId)
 
-      buildAcr:
+      BuildRegistry:
         server: $(acr-staging-test.server)
         resourceGroup: $(testResourceGroup)
         subscription: $(testSubscription)
@@ -92,7 +95,7 @@ stages:
         clientId: $(test-nonprod.serviceConnection.clientId)
         tenantId: $(testTenant)
 
-      publishAcr:
+      PublishRegistry:
         server: $(acr-test.server)
         resourceGroup: $(testResourceGroup)
         subscription: $(testSubscription)