Skip to content

Commit da5ca14

Browse files
TingluoHuangTingluoHuang
committed
Merge pull request #43 from Microsoft/users/tihuang/tests
L0 tests for GitSourceProvider
2 parents cbdea2f + 4eb2e62 commit da5ca14

File tree

4 files changed

+819
-421
lines changed

4 files changed

+819
-421
lines changed

‎src/Agent.Worker/Build/GitCommandManager.cs‎

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,355 @@
1+
using Microsoft.VisualStudio.Services.Agent.Util;
2+
using System;
3+
using System.Collections.Generic;
4+
using System.Diagnostics;
5+
using System.Linq;
6+
using System.Text.RegularExpressions;
7+
using System.Threading;
8+
using System.Threading.Tasks;
9+
10+
namespace Microsoft.VisualStudio.Services.Agent.Worker.Build
11+
{
12+
[ServiceLocator(Default = typeof(GitCommandManager))]
13+
public interface IGitCommandManager : IAgentService
14+
{
15+
string GitPath { get; set; }
16+
17+
Version Version { get; set; }
18+
19+
// git clone --progress --no-checkout <URL> <LocalDir>
20+
Task<int> GitClone(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string username, string password, bool exposeCred, CancellationToken cancellationToken);
21+
22+
// git fetch --tags --prune --progress origin [+refs/pull/*:refs/remote/pull/*]
23+
Task<int> GitFetch(IExecutionContext context, string repositoryPath, string remoteName, List<string> refSpec, string username, string password, bool exposeCred, CancellationToken cancellationToken);
24+
25+
// git checkout -f --progress <commitId/branch>
26+
Task<int> GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken);
27+
28+
// git clean -fdx
29+
Task<int> GitClean(IExecutionContext context, string repositoryPath);
30+
31+
// git reset --hard HEAD
32+
Task<int> GitReset(IExecutionContext context, string repositoryPath);
33+
34+
// get remote set-url <origin> <url>
35+
Task<int> GitRemoteSetUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl);
36+
37+
// get remote set-url --push <origin> <url>
38+
Task<int> GitRemoteSetPushUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl);
39+
40+
// git submodule init
41+
Task<int> GitSubmoduleInit(IExecutionContext context, string repositoryPath);
42+
43+
// git submodule update -f
44+
Task<int> GitSubmoduleUpdate(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken);
45+
46+
// git config --get remote.origin.url
47+
Task<Uri> GitGetFetchUrl(IExecutionContext context, string repositoryPath);
48+
49+
// git config --get-regexp submodule.*.url
50+
Task<Dictionary<string, Uri>> GitGetSubmoduleUrls(IExecutionContext context, string repoRoot);
51+
52+
// git config <key> <value>
53+
Task<int> GitUpdateSubmoduleUrls(IExecutionContext context, string repositoryPath, Dictionary<string, Uri> updateSubmoduleUrls);
54+
55+
// git config gc.auto 0
56+
Task<int> GitDisableAutoGC(IExecutionContext context, string repositoryPath);
57+
58+
// git version
59+
Task<Version> GitVersion(IExecutionContext context);
60+
}
61+
62+
public class GitCommandManager : AgentService, IGitCommandManager
63+
{
64+
private readonly Dictionary<string, Dictionary<Version, string>> _gitCommands = new Dictionary<string, Dictionary<Version, string>>(StringComparer.OrdinalIgnoreCase)
65+
{
66+
{
67+
"checkout", new Dictionary<Version, string> ()
68+
{
69+
{ new Version(1,8), "--force {0}" },
70+
{ new Version(2,7), "--progress --force {0}" }
71+
}
72+
}
73+
};
74+
75+
public string GitPath { get; set; }
76+
public Version Version { get; set; }
77+
78+
// git clone --progress --no-checkout <URL> <LocalDir>
79+
public async Task<int> GitClone(IExecutionContext context, string repositoryPath, Uri repositoryUrl, string username, string password, bool exposeCred, CancellationToken cancellationToken)
80+
{
81+
context.Debug($"Clone git repository: {repositoryUrl.AbsoluteUri} into: {repositoryPath}.");
82+
string repoRootEscapeSpace = StringUtil.Format(@"""{0}""", repositoryPath.Replace(@"""", @"\"""));
83+
return await ExecuteGitCommandAsync(context, repositoryPath, "clone", StringUtil.Format($"--progress --no-checkout {repositoryUrl.AbsoluteUri} {repoRootEscapeSpace}"), cancellationToken);
84+
}
85+
86+
// git fetch --tags --prune --progress origin [+refs/pull/*:refs/remote/pull/*]
87+
public async Task<int> GitFetch(IExecutionContext context, string repositoryPath, string remoteName, List<string> refSpec, string username, string password, bool exposeCred, CancellationToken cancellationToken)
88+
{
89+
context.Debug($"Fetch git repository at: {repositoryPath} remote: {remoteName}.");
90+
if (refSpec != null && refSpec.Count > 0)
91+
{
92+
refSpec = refSpec.Where(r => !string.IsNullOrEmpty(r)).ToList();
93+
}
94+
95+
return await ExecuteGitCommandAsync(context, repositoryPath, "fetch", StringUtil.Format($"--tags --prune --progress {remoteName} {string.Join(" ", refSpec)}"), cancellationToken);
96+
}
97+
98+
// git checkout -f --progress <commitId/branch>
99+
public async Task<int> GitCheckout(IExecutionContext context, string repositoryPath, string committishOrBranchSpec, CancellationToken cancellationToken = default(CancellationToken))
100+
{
101+
context.Debug($"Checkout {committishOrBranchSpec}.");
102+
string checkoutOption = GetCommandOption("checkout");
103+
return await ExecuteGitCommandAsync(context, repositoryPath, "checkout", StringUtil.Format(checkoutOption, committishOrBranchSpec), cancellationToken);
104+
}
105+
106+
// git clean -fdx
107+
public async Task<int> GitClean(IExecutionContext context, string repositoryPath)
108+
{
109+
context.Debug($"Delete untracked files/folders for repository at {repositoryPath}.");
110+
return await ExecuteGitCommandAsync(context, repositoryPath, "clean", "-fdx");
111+
}
112+
113+
// git reset --hard HEAD
114+
public async Task<int> GitReset(IExecutionContext context, string repositoryPath)
115+
{
116+
context.Debug($"Undo any changes to tracked files in the working tree for repository at {repositoryPath}.");
117+
return await ExecuteGitCommandAsync(context, repositoryPath, "reset", "--hard HEAD");
118+
}
119+
120+
// get remote set-url <origin> <url>
121+
public async Task<int> GitRemoteSetUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl)
122+
{
123+
context.Debug($"Set git fetch url to: {remoteUrl} for remote: {remoteName}.");
124+
return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url {remoteName} {remoteUrl}"));
125+
}
126+
127+
// get remote set-url --push <origin> <url>
128+
public async Task<int> GitRemoteSetPushUrl(IExecutionContext context, string repositoryPath, string remoteName, string remoteUrl)
129+
{
130+
context.Debug($"Set git push url to: {remoteUrl} for remote: {remoteName}.");
131+
return await ExecuteGitCommandAsync(context, repositoryPath, "remote", StringUtil.Format($"set-url --push {remoteName} {remoteUrl}"));
132+
}
133+
134+
// git submodule init
135+
public async Task<int> GitSubmoduleInit(IExecutionContext context, string repositoryPath)
136+
{
137+
context.Debug("Initialize the git submodules.");
138+
return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "init");
139+
}
140+
141+
// git submodule update -f
142+
public async Task<int> GitSubmoduleUpdate(IExecutionContext context, string repositoryPath, CancellationToken cancellationToken = default(CancellationToken))
143+
{
144+
context.Debug("Update the registered git submodules.");
145+
return await ExecuteGitCommandAsync(context, repositoryPath, "submodule", "update -f", cancellationToken);
146+
}
147+
148+
// git config --get remote.origin.url
149+
public async Task<Uri> GitGetFetchUrl(IExecutionContext context, string repositoryPath)
150+
{
151+
context.Debug($"Inspect remote.origin.url for repository under {repositoryPath}");
152+
Uri fetchUrl = null;
153+
154+
List<string> outputStrings = new List<string>();
155+
int exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", "--get remote.origin.url", outputStrings);
156+
157+
if (exitCode != 0)
158+
{
159+
context.Warning($"'git config --get remote.origin.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'");
160+
}
161+
else
162+
{
163+
// remove empty strings
164+
outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList();
165+
if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First()))
166+
{
167+
string remoteFetchUrl = outputStrings.First();
168+
if (Uri.IsWellFormedUriString(remoteFetchUrl, UriKind.Absolute))
169+
{
170+
context.Debug($"Get remote origin fetch url from git config: {remoteFetchUrl}");
171+
fetchUrl = new Uri(remoteFetchUrl);
172+
}
173+
else
174+
{
175+
context.Debug($"The Origin fetch url from git config: {remoteFetchUrl} is not a absolute well formed url.");
176+
}
177+
}
178+
else
179+
{
180+
context.Debug($"Unable capture git remote fetch uri from 'git config --get remote.origin.url' command's output, the command's output is not expected: {string.Join(Environment.NewLine, outputStrings)}.");
181+
}
182+
}
183+
184+
return fetchUrl;
185+
}
186+
187+
// git config --get-regexp submodule.*.url
188+
public async Task<Dictionary<string, Uri>> GitGetSubmoduleUrls(IExecutionContext context, string repoRoot)
189+
{
190+
context.Debug($"Inspect all submodule.<name>.url for submodules under {repoRoot}");
191+
192+
Dictionary<string, Uri> submoduleUrls = new Dictionary<string, Uri>(StringComparer.OrdinalIgnoreCase);
193+
194+
List<string> outputStrings = new List<string>();
195+
int exitCode = await ExecuteGitCommandAsync(context, repoRoot, "config", "--get-regexp submodule.?*.url", outputStrings);
196+
197+
if (exitCode != 0)
198+
{
199+
context.Debug($"'git config --get-regexp submodule.?*.url' failed with exit code: {exitCode}, output: '{string.Join(Environment.NewLine, outputStrings)}'");
200+
}
201+
else
202+
{
203+
// remove empty strings
204+
outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList();
205+
foreach (var urlString in outputStrings)
206+
{
207+
context.Debug($"Potential git submodule name and fetch url: {urlString}.");
208+
string[] submoduleUrl = urlString.Split(new Char[] { ' ' }, 2, StringSplitOptions.RemoveEmptyEntries);
209+
if (submoduleUrl.Length == 2 && Uri.IsWellFormedUriString(submoduleUrl[1], UriKind.Absolute))
210+
{
211+
submoduleUrls[submoduleUrl[0]] = new Uri(submoduleUrl[1]);
212+
}
213+
else
214+
{
215+
context.Debug($"Can't parse git submodule name and submodule fetch url from output: '{urlString}'.");
216+
}
217+
}
218+
}
219+
220+
return submoduleUrls;
221+
}
222+
223+
// git config <key> <value>
224+
public async Task<int> GitUpdateSubmoduleUrls(IExecutionContext context, string repositoryPath, Dictionary<string, Uri> updateSubmoduleUrls)
225+
{
226+
context.Debug("Update all submodule.<name>.url with credential embeded url.");
227+
228+
int overallExitCode = 0;
229+
foreach (var submodule in updateSubmoduleUrls)
230+
{
231+
Int32 exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"{submodule.Key} {submodule.Value.ToString()}"));
232+
if (exitCode != 0)
233+
{
234+
context.Debug($"Unable update: {submodule.Key}.");
235+
overallExitCode = exitCode;
236+
}
237+
}
238+
239+
return overallExitCode;
240+
}
241+
242+
// git config gc.auto 0
243+
public async Task<int> GitDisableAutoGC(IExecutionContext context, string repositoryPath)
244+
{
245+
context.Debug("Disable git auto garbage collection.");
246+
return await ExecuteGitCommandAsync(context, repositoryPath, "config", "gc.auto 0");
247+
}
248+
249+
// git version
250+
public async Task<Version> GitVersion(IExecutionContext context)
251+
{
252+
context.Debug("Get git version.");
253+
Version version = null;
254+
List<string> outputStrings = new List<string>();
255+
int exitCode = await ExecuteGitCommandAsync(context, IOUtil.GetWorkPath(HostContext), "version", null, outputStrings);
256+
if (exitCode == 0)
257+
{
258+
// remove any empty line.
259+
outputStrings = outputStrings.Where(o => !string.IsNullOrEmpty(o)).ToList();
260+
if (outputStrings.Count == 1 && !string.IsNullOrEmpty(outputStrings.First()))
261+
{
262+
string verString = outputStrings.First();
263+
// we might only interested about major.minor version
264+
Regex verRegex = new Regex("\\d+\\.\\d+", RegexOptions.IgnoreCase);
265+
var matchResult = verRegex.Match(verString);
266+
if (matchResult.Success && !string.IsNullOrEmpty(matchResult.Value))
267+
{
268+
if (!Version.TryParse(matchResult.Value, out version))
269+
{
270+
version = null;
271+
}
272+
}
273+
}
274+
}
275+
276+
return version;
277+
}
278+
279+
private string GetCommandOption(string command)
280+
{
281+
if (string.IsNullOrEmpty(command))
282+
{
283+
throw new ArgumentNullException("command");
284+
}
285+
286+
if (!_gitCommands.ContainsKey(command))
287+
{
288+
throw new NotSupportedException($"Unsupported git command: {command}");
289+
}
290+
291+
Dictionary<Version, string> options = _gitCommands[command];
292+
foreach (var versionOption in options.OrderByDescending(o => o.Key))
293+
{
294+
if (Version >= versionOption.Key)
295+
{
296+
return versionOption.Value;
297+
}
298+
}
299+
300+
var earliestVersion = options.OrderByDescending(o => o.Key).Last();
301+
Trace.Info($"Fallback to version {earliestVersion.Key.ToString()} command option for git {command}.");
302+
return earliestVersion.Value;
303+
}
304+
305+
private async Task<int> ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, CancellationToken cancellationToken = default(CancellationToken))
306+
{
307+
string arg = StringUtil.Format($"{command} {options}").Trim();
308+
context.Command($"git {arg}");
309+
310+
var processInvoker = HostContext.CreateService<IProcessInvoker>();
311+
processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message)
312+
{
313+
context.Output(message.Data);
314+
};
315+
316+
processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message)
317+
{
318+
context.Output(message.Data);
319+
};
320+
321+
return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, null, cancellationToken);
322+
}
323+
324+
private async Task<int> ExecuteGitCommandAsync(IExecutionContext context, string repoRoot, string command, string options, IList<string> output)
325+
{
326+
string arg = StringUtil.Format($"{command} {options}").Trim();
327+
context.Command($"git {arg}");
328+
329+
if (output == null)
330+
{
331+
output = new List<string>();
332+
}
333+
334+
object outputLock = new object();
335+
var processInvoker = HostContext.CreateService<IProcessInvoker>();
336+
processInvoker.OutputDataReceived += delegate (object sender, DataReceivedEventArgs message)
337+
{
338+
lock (outputLock)
339+
{
340+
output.Add(message.Data);
341+
}
342+
};
343+
344+
processInvoker.ErrorDataReceived += delegate (object sender, DataReceivedEventArgs message)
345+
{
346+
lock (outputLock)
347+
{
348+
output.Add(message.Data);
349+
}
350+
};
351+
352+
return await processInvoker.ExecuteAsync(repoRoot, GitPath, arg, null, default(CancellationToken));
353+
}
354+
}
355+
}

0 commit comments

Comments
 (0)