Skip to content

Commit c0987a5

Browse files
b-barthelBrian Barthel
authored and
merlynomsft
committed
Add multiple domain support to Build Artifacts upload/download (#4617)
* Add initial support for multiple domains in Build Artifacts * Add knob to override the storage domain used. * Split domain knob into separate build|pipeline artifact knobs. * Add fallback for socket exception as well --------- Co-authored-by: Brian Barthel <[email protected]>
1 parent fdd6916 commit c0987a5

File tree

9 files changed

+285
-184
lines changed

9 files changed

+285
-184
lines changed

‎src/Agent.Plugins/Artifact/FileContainerProvider.cs‎

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using Microsoft.VisualStudio.Services.Agent.Util;
1212
using Microsoft.VisualStudio.Services.BlobStore.Common;
1313
using Microsoft.VisualStudio.Services.BlobStore.WebApi;
14+
using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
1415
using Microsoft.VisualStudio.Services.Content.Common;
1516
using Microsoft.VisualStudio.Services.Content.Common.Tracing;
1617
using Microsoft.VisualStudio.Services.FileContainer;
@@ -149,27 +150,44 @@ private async Task DownloadFileContainerAsync(IEnumerable<FileContainerItem> ite
149150
// Only initialize these clients if we know we need to download from Blobstore
150151
// If a client cannot connect to Blobstore, we shouldn't stop them from downloading from FCS
151152
var downloadFromBlob = !AgentKnobs.DisableBuildArtifactsToBlob.GetValue(context).AsBoolean();
152-
DedupStoreClient dedupClient = null;
153+
Dictionary<IDomainId,DedupStoreClient> dedupClientTable = new Dictionary<IDomainId, DedupStoreClient>();
153154
BlobStoreClientTelemetryTfs clientTelemetry = null;
154155
if (downloadFromBlob && fileItems.Any(x => x.BlobMetadata != null))
155156
{
157+
// this is not the most efficient but good enough for now:
158+
var domains = fileItems.Select(x => GetDomainIdAndDedupIdFromArtifactHash(x.BlobMetadata.ArtifactHash).domainId).Distinct();
159+
DedupStoreClient dedupClient = null;
156160
try
157161
{
158-
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupClientAsync(
159-
false,
160-
(str) => this.tracer.Info(str),
161-
this.connection,
162-
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
162+
BlobstoreClientSettings clientSettings = await BlobstoreClientSettings.GetClientSettingsAsync(
163+
connection,
163164
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.BuildArtifact,
165+
tracer,
164166
cancellationToken);
167+
168+
foreach(var domainId in domains)
169+
{
170+
(dedupClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance.CreateDedupClient(
171+
this.connection,
172+
domainId,
173+
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
174+
clientSettings.GetRedirectTimeout(),
175+
false,
176+
(str) => this.tracer.Info(str),
177+
cancellationToken);
178+
179+
dedupClientTable.Add(domainId, dedupClient);
180+
}
165181
}
166182
catch (SocketException e)
167183
{
168184
ExceptionsUtil.HandleSocketException(e, connection.Uri.ToString(), context.Warning);
185+
// Fall back to streaming through TFS if we cannot reach blobstore for any reason
186+
downloadFromBlob = false;
169187
}
170188
catch
171189
{
172-
var blobStoreHost = dedupClient.Client.BaseAddress.Host;
190+
var blobStoreHost = dedupClient?.Client.BaseAddress.Host;
173191
var allowListLink = BlobStoreWarningInfoProvider.GetAllowListLinkForCurrentPlatform();
174192
var warningMessage = StringUtil.Loc("BlobStoreDownloadWarning", blobStoreHost, allowListLink);
175193

@@ -191,7 +209,8 @@ await AsyncHttpRetryHelper.InvokeVoidAsync(
191209
tracer.Info($"Downloading: {targetPath}");
192210
if (item.BlobMetadata != null && downloadFromBlob)
193211
{
194-
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, dedupClient, clientTelemetry, cancellationToken);
212+
var client = dedupClientTable[GetDomainIdAndDedupIdFromArtifactHash(item.BlobMetadata.ArtifactHash).domainId];
213+
await this.DownloadFileFromBlobAsync(context, containerIdAndRoot, targetPath, projectId, item, client, clientTelemetry, cancellationToken);
195214
}
196215
else
197216
{
@@ -336,6 +355,22 @@ private async Task<Stream> DownloadFileAsync(
336355
return responseStream;
337356
}
338357

358+
private static (IDomainId domainId, DedupIdentifier dedupId) GetDomainIdAndDedupIdFromArtifactHash(string artifactHash)
359+
{
360+
string[] parts = artifactHash.Split(',');
361+
if(parts.Length == 1)
362+
{
363+
// legacy format is always in the default domain:
364+
return (WellKnownDomainIds.DefaultDomainId, DedupIdentifier.Deserialize(parts[0]));
365+
}
366+
else if(parts.Length==2)
367+
{
368+
// Multidomain format is in the form of <domainId>,<dedupId>
369+
return (DomainIdFactory.Create(parts[0]), DedupIdentifier.Deserialize(parts[1]));
370+
}
371+
throw new ArgumentException($"Invalid artifact hash: {artifactHash}", nameof(artifactHash));
372+
}
373+
339374
private async Task DownloadFileFromBlobAsync(
340375
AgentTaskPluginExecutionContext context,
341376
(long, string) containerIdAndRoot,
@@ -346,7 +381,7 @@ private async Task DownloadFileFromBlobAsync(
346381
BlobStoreClientTelemetryTfs clientTelemetry,
347382
CancellationToken cancellationToken)
348383
{
349-
var dedupIdentifier = DedupIdentifier.Deserialize(item.BlobMetadata.ArtifactHash);
384+
(var domainId, var dedupIdentifier) = GetDomainIdAndDedupIdFromArtifactHash(item.BlobMetadata.ArtifactHash);
350385

351386
var downloadRecord = clientTelemetry.CreateRecord<BuildArtifactActionRecord>((level, uri, type) =>
352387
new BuildArtifactActionRecord(level, uri, type, nameof(DownloadFileContainerAsync), context));

‎src/Agent.Plugins/Artifact/PipelineArtifactServer.cs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
using Microsoft.VisualStudio.Services.WebApi;
1919
using Microsoft.VisualStudio.Services.Agent.Util;
2020
using Microsoft.VisualStudio.Services.BlobStore.Common;
21+
using Agent.Sdk.Knob;
2122

2223
namespace Agent.Plugins
2324
{
@@ -44,14 +45,15 @@ internal async Task UploadAsync(
4445
// Get the client settings, if any.
4546
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose: false, (str) => context.Output(str));
4647
VssConnection connection = context.VssConnection;
47-
var clientSettings = await DedupManifestArtifactClientFactory.GetClientSettingsAsync(
48+
var clientSettings = await BlobstoreClientSettings.GetClientSettingsAsync(
4849
connection,
4950
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
5051
tracer,
5152
cancellationToken);
5253

53-
// Get the default domain to use:
54-
IDomainId domainId = DedupManifestArtifactClientFactory.GetDefaultDomainId(clientSettings, tracer);
54+
// Check if the pipeline has an override domain set, if not, use the default domain from the client settings.
55+
string overrideDomain = AgentKnobs.SendPipelineArtifactsToBlobstoreDomain.GetValue(context).AsString();
56+
IDomainId domainId = String.IsNullOrWhiteSpace(overrideDomain) ? clientSettings.GetDefaultDomainId() : DomainIdFactory.Create(overrideDomain);
5557

5658
var (dedupManifestClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
5759
.CreateDedupManifestClient(

‎src/Agent.Sdk/Knob/AgentKnobs.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -381,6 +381,18 @@ public class AgentKnobs
381381
new RuntimeKnobSource("DISABLE_BUILD_ARTIFACTS_TO_BLOB"),
382382
new EnvironmentKnobSource("DISABLE_BUILD_ARTIFACTS_TO_BLOB"),
383383
new BuiltInDefaultKnobSource("false"));
384+
public static readonly Knob SendBuildArtifactsToBlobstoreDomain = new Knob(
385+
nameof(SendBuildArtifactsToBlobstoreDomain),
386+
"When set, defines the domain to use to send Build artifacts to.",
387+
new RuntimeKnobSource("SEND_BUILD_ARTIFACTS_TO_BLOBSTORE_DOMAIN"),
388+
new EnvironmentKnobSource("SEND_BUILD_ARTIFACT_ARTIFACTS_TO_BLOBSTORE_DOMAIN"),
389+
new BuiltInDefaultKnobSource(string.Empty));
390+
public static readonly Knob SendPipelineArtifactsToBlobstoreDomain = new Knob(
391+
nameof(SendPipelineArtifactsToBlobstoreDomain),
392+
"When set, defines the domain to use to send Pipeline artifacts to.",
393+
new RuntimeKnobSource("SEND_PIPELINE_ARTIFACTS_TO_BLOBSTORE_DOMAIN"),
394+
new EnvironmentKnobSource("SEND_PIPELINE_ARTIFACT_ARTIFACTS_TO_BLOBSTORE_DOMAIN"),
395+
new BuiltInDefaultKnobSource(string.Empty));
384396

385397
public static readonly Knob EnableIncompatibleBuildArtifactsPathResolution = new Knob(
386398
nameof(EnableIncompatibleBuildArtifactsPathResolution),

‎src/Agent.Worker/Build/FileContainerServer.cs‎

Lines changed: 39 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,13 @@
33

44
using Agent.Sdk.Knob;
55
using Agent.Sdk.Util;
6+
using BuildXL.Cache.ContentStore.Hashing;
7+
using Microsoft.TeamFoundation.DistributedTask.WebApi;
68
using Microsoft.VisualStudio.Services.Agent.Blob;
79
using Microsoft.VisualStudio.Services.Agent.Util;
10+
using Microsoft.VisualStudio.Services.BlobStore.WebApi;
11+
using Microsoft.VisualStudio.Services.BlobStore.Common;
12+
using Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts;
813
using Microsoft.VisualStudio.Services.FileContainer.Client;
914
using System;
1015
using System.Collections.Concurrent;
@@ -18,8 +23,6 @@
1823
using System.Net.Http;
1924
using System.Net;
2025
using System.Net.Sockets;
21-
using Microsoft.TeamFoundation.DistributedTask.WebApi;
22-
using Microsoft.VisualStudio.Services.BlobStore.WebApi;
2326

2427

2528
namespace Microsoft.VisualStudio.Services.Agent.Worker.Build
@@ -326,6 +329,16 @@ private async Task<UploadResult> UploadAsync(IAsyncCommandContext context, int u
326329

327330
return new UploadResult(failedFiles, uploadedSize);
328331
}
332+
public static string CreateDomainHash(IDomainId domainId, DedupIdentifier dedupId)
333+
{
334+
if (domainId != WellKnownDomainIds.DefaultDomainId)
335+
{
336+
// Only use the new format domainId,dedupId if we aren't going to the default domain as this is a breaking change:
337+
return $"{domainId.Serialize()},{dedupId.ValueString}";
338+
}
339+
// We are still uploading to the default domain so use the don't use the new format:
340+
return dedupId.ValueString;
341+
}
329342

330343
private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, IReadOnlyList<string> files, int concurrentUploads, CancellationToken token)
331344
{
@@ -342,20 +355,33 @@ private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, I
342355
BlobStoreClientTelemetryTfs clientTelemetry = null;
343356
try
344357
{
358+
345359
var verbose = String.Equals(context.GetVariableValueOrDefault("system.debug"), "true", StringComparison.InvariantCultureIgnoreCase);
346-
int maxParallelism = context.GetHostContext().GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism;
347-
(dedupClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
348-
.CreateDedupClientAsync(
360+
Action<string> tracer = (str) => context.Output(str);
361+
362+
var clientSettings = await BlobstoreClientSettings.GetClientSettingsAsync(
363+
_connection,
364+
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.BuildArtifact,
365+
DedupManifestArtifactClientFactory.CreateArtifactsTracer(verbose, tracer),
366+
token);
367+
368+
// Check if the pipeline has an override domain set, if not, use the default domain from the client settings.
369+
string overrideDomain = AgentKnobs.SendBuildArtifactsToBlobstoreDomain.GetValue(context).AsString();
370+
IDomainId domainId = String.IsNullOrWhiteSpace(overrideDomain) ? clientSettings.GetDefaultDomainId() : DomainIdFactory.Create(overrideDomain);
371+
372+
(dedupClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
373+
.CreateDedupClient(
374+
_connection,
375+
domainId,
376+
context.GetHostContext().GetService<IConfigurationStore>().GetSettings().MaxDedupParallelism,
377+
clientSettings.GetRedirectTimeout(),
349378
verbose,
350-
(str) => context.Output(str),
351-
this._connection,
352-
maxParallelism,
353-
BlobStore.WebApi.Contracts.Client.BuildArtifact,
379+
tracer,
354380
token);
355381

356382
// Upload to blobstore
357383
var results = await BlobStoreUtils.UploadBatchToBlobstore(verbose, files, (level, uri, type) =>
358-
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), (str) => context.Output(str), dedupClient, clientTelemetry, token, enableReporting: true);
384+
new BuildArtifactActionRecord(level, uri, type, nameof(BlobUploadAsync), context), tracer, dedupClient, clientTelemetry, token, enableReporting: true);
359385

360386
// Associate with TFS
361387
context.Output(StringUtil.Loc("AssociateFiles"));
@@ -373,7 +399,7 @@ private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, I
373399
var parallelAssociateTasks = new List<Task<UploadResult>>();
374400
for (int uploader = 0; uploader < concurrentUploads; uploader++)
375401
{
376-
parallelAssociateTasks.Add(AssociateAsync(context, queue, token));
402+
parallelAssociateTasks.Add(AssociateAsync(context, domainId, queue, token));
377403
}
378404

379405
// Wait for parallel associate tasks to finish.
@@ -419,7 +445,7 @@ private async Task<UploadResult> BlobUploadAsync(IAsyncCommandContext context, I
419445
return uploadResult;
420446
}
421447

422-
private async Task<UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue<BlobFileInfo> associateQueue, CancellationToken token)
448+
private async Task<UploadResult> AssociateAsync(IAsyncCommandContext context, IDomainId domainId, ConcurrentQueue<BlobFileInfo> associateQueue, CancellationToken token)
423449
{
424450
var uploadResult = new UploadResult();
425451

@@ -443,7 +469,7 @@ private async Task<UploadResult> AssociateAsync(IAsyncCommandContext context, Co
443469
{
444470
var length = (long)file.Node.TransitiveContentBytes;
445471
response = await retryHelper.Retry(async () => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId,
446-
file.DedupId.ValueString, length, token),
472+
CreateDomainHash(domainId, file.DedupId), length, token),
447473
(retryCounter) => (int)Math.Pow(retryCounter, 2) * 5,
448474
(exception) => true);
449475
uploadResult.TotalFileSizeUploaded += length;

‎src/Microsoft.VisualStudio.Services.Agent/Blob/BlobStoreUtils.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ private static async Task<List<BlobFileInfo>> GenerateHashes(IReadOnlyList<strin
120120
var itemPath = filePaths[i];
121121
try
122122
{
123-
var dedupNode = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, cancellationToken, false);
123+
var dedupNode = await ChunkerHelper.CreateFromFileAsync(FileSystem.Instance, itemPath, false, ChunkerHelper.DefaultChunkHashType, cancellationToken);
124124
nodes[i] = new BlobFileInfo
125125
{
126126
Path = itemPath,

0 commit comments

Comments
 (0)