Skip to content

Commit

Permalink
[DownloadBuildArtifactsV1] Tar extraction on download (#3467)
Browse files Browse the repository at this point in the history 
* add tar extraction feature

* fixes

* fix download parameters passing

* add no tars found warning

also some other minor logging improvements

* fix 'StandardError has not been redirected'

* fix invalid StreamReader method

* extract `GetArtifactItems()`

* rename string for tar search start

* get items outside of `DownloadFileContainerAsync`

* move tar extraction to DownloadArtifact methods

* fix 'directory with the same name already exists'

* account for incompatible path resolution too

* fix paths

* trim relative artifact path for slash

apparently `Path.Combine` doesn't work properly if the second argument looks like an absolute path

* only move extracted tars directory if tars found

* fix `MoveDirectory`

* replace files in destination

* fix destination folder

* create target directory

* add comments

Co-authored-by: Anatoly Bolshakov <[email protected]>
  • Loading branch information
@DaniilShmelev
DaniilShmelev and Anatoly Bolshakov committed Aug 23, 2021
1 parent a82b01b commit d896461
Show file tree
Hide file tree
Showing 4 changed files with 172 additions and 14 deletions.
2 changes: 2 additions & 0 deletions src/Agent.Plugins/Artifact/ArtifactDownloadParameters.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ internal class ArtifactDownloadParameters
public int RetryDownloadCount { get; set; } = 4;
public bool CheckDownloadedFiles { get; set; } = false;
public Options CustomMinimatchOptions { get; set; } = null;
public bool ExtractTars { get; set; } = false;
public string ExtractedTarsTempPath { get; set; }
public bool AppendArtifactNameToTargetPath { get; set; } = true;
}

Expand Down
155 changes: 143 additions & 12 deletions src/Agent.Plugins/Artifact/FileContainerProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using Microsoft.VisualStudio.Services.WebApi;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
Expand Down Expand Up @@ -44,17 +45,41 @@ public FileContainerProvider(VssConnection connection, IAppTraceSource tracer)

public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
await this.DownloadFileContainerAsync(downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken);
IEnumerable<FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact);
await this.DownloadFileContainerAsync(items, downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken);

IEnumerable<string> fileArtifactPaths = items
.Where((item) => item.ItemType == ContainerItemType.File)
.Select((fileItem) => Path.Combine(downloadParameters.TargetDirectory, fileItem.Path));

if (downloadParameters.ExtractTars)
{
ExtractTarsIfPresent(context, fileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath);
}
}

public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable<BuildArtifact> buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context)
{
var allFileArtifactPaths = new List<string>();

foreach (var buildArtifact in buildArtifacts)
{
var dirPath = downloadParameters.AppendArtifactNameToTargetPath
? Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name)
: downloadParameters.TargetDirectory;
await DownloadFileContainerAsync(downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload: false);

IEnumerable<FileContainerItem> items = await GetArtifactItems(downloadParameters, buildArtifact);
IEnumerable<string> fileArtifactPaths = items
.Where((item) => item.ItemType == ContainerItemType.File)
.Select((fileItem) => Path.Combine(dirPath, fileItem.Path));
allFileArtifactPaths.AddRange(fileArtifactPaths);

await DownloadFileContainerAsync(items, downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload: false);
}

if (downloadParameters.ExtractTars)
{
ExtractTarsIfPresent(context, allFileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath);
}
}

Expand Down Expand Up @@ -84,21 +109,12 @@ public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters down
}
}

private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
private async Task DownloadFileContainerAsync(IEnumerable<FileContainerItem> items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true)
{
var containerIdAndRoot = ParseContainerId(artifact.Resource.Data);
var projectId = downloadParameters.ProjectId;
var minimatchPatterns = downloadParameters.MinimatchFilters;

var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow: false, includeBlobMetadata: true, containerIdAndRoot.Item2);

tracer.Info($"Start downloading FCS artifact- {artifact.Name}");
IEnumerable<Func<string, bool>> minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions);

if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}

if (!isSingleArtifactDownload && items.Any())
{
Expand Down Expand Up @@ -190,6 +206,35 @@ await AsyncHttpRetryHelper.InvokeVoidAsync(
}
}

// Returns all artifact items. Uses minimatch filters specified in downloadParameters.
private async Task<IEnumerable<FileContainerItem>> GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact)
{
(long, string) containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data);
Guid projectId = downloadParameters.ProjectId;
string[] minimatchPatterns = downloadParameters.MinimatchFilters;

List<FileContainerItem> items = await containerClient.QueryContainerItemsAsync(
containerIdAndRoot.Item1,
projectId,
isShallow: false,
includeBlobMetadata: true,
containerIdAndRoot.Item2
);

IEnumerable<Func<string, bool>> minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(
minimatchPatterns,
tracer,
downloadParameters.CustomMinimatchOptions
);

if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0)
{
items = this.GetFilteredItems(items, minimatcherFuncs);
}

return items;
}

private void CheckDownloads(IEnumerable<FileContainerItem> items, string rootPath, string artifactName, bool includeArtifactName)
{
tracer.Info(StringUtil.Loc("BeginArtifactItemsIntegrityCheck"));
Expand Down Expand Up @@ -323,5 +368,91 @@ private List<FileContainerItem> GetFilteredItems(List<FileContainerItem> items,
}
return filteredItems;
}

// Checks all specified artifact paths, searches for files ending with '.tar'.
// If any files were found, extracts them to extractedTarsTempPath and moves to rootPath/extracted_tars.
private void ExtractTarsIfPresent(AgentTaskPluginExecutionContext context, IEnumerable<string> fileArtifactPaths, string rootPath, string extractedTarsTempPath)
{
tracer.Info(StringUtil.Loc("TarSearchStart"));

int tarsFoundCount = 0;

foreach (var fileArtifactPath in fileArtifactPaths)
{
if (fileArtifactPath.EndsWith(".tar"))
{
tarsFoundCount += 1;

// fileArtifactPath is a combination of rootPath and the relative artifact path
string relativeFileArtifactPath = fileArtifactPath.Substring(rootPath.Length);
string relativeFileArtifactDirPath = Path.GetDirectoryName(relativeFileArtifactPath).TrimStart('/');
string extractedFilesDir = Path.Combine(extractedTarsTempPath, relativeFileArtifactDirPath);

ExtractTar(fileArtifactPath, extractedFilesDir);

File.Delete(fileArtifactPath);
}
}

if (tarsFoundCount == 0) {
context.Warning(StringUtil.Loc("TarsNotFound"));
} else {
tracer.Info(StringUtil.Loc("TarsFound", tarsFoundCount));

string targetDirectory = Path.Combine(rootPath, "extracted_tars");
Directory.CreateDirectory(targetDirectory);
MoveDirectory(extractedTarsTempPath, targetDirectory);
}
}

// Extracts tar archive at tarArchivePath to extractedFilesDir.
// Uses 'tar' utility like this: tar xf `tarArchivePath` --directory `extractedFilesDir`.
// Throws if any errors are encountered.
private void ExtractTar(string tarArchivePath, string extractedFilesDir)
{
tracer.Info(StringUtil.Loc("TarExtraction", tarArchivePath));

Directory.CreateDirectory(extractedFilesDir);
var extractionProcessInfo = new ProcessStartInfo("tar")
{
Arguments = $"xf {tarArchivePath} --directory {extractedFilesDir}",
UseShellExecute = false,
RedirectStandardError = true
};
Process extractionProcess = Process.Start(extractionProcessInfo);
extractionProcess.WaitForExit();

var extractionStderr = extractionProcess.StandardError.ReadToEnd();
if (extractionStderr.Length != 0 || extractionProcess.ExitCode != 0)
{
throw new Exception(StringUtil.Loc("TarExtractionError", tarArchivePath, extractionStderr));
}
}

// Recursively moves sourcePath directory to targetPath
private void MoveDirectory(string sourcePath, string targetPath) {
var sourceDirectoryInfo = new DirectoryInfo(sourcePath);
foreach (FileInfo file in sourceDirectoryInfo.GetFiles("*", SearchOption.TopDirectoryOnly))
{
file.MoveTo(Path.Combine(targetPath, file.Name), true);
}
foreach (DirectoryInfo subdirectory in sourceDirectoryInfo.GetDirectories("*", SearchOption.TopDirectoryOnly))
{
string subdirectoryDestinationPath = Path.Combine(targetPath, subdirectory.Name);
var subdirectoryDestination = new DirectoryInfo(subdirectoryDestinationPath);

if (subdirectoryDestination.Exists)
{
MoveDirectory(
Path.Combine(sourcePath, subdirectory.Name),
Path.Combine(targetPath, subdirectory.Name)
);
}
else
{
subdirectory.MoveTo(Path.Combine(targetPath, subdirectory.Name));
}
}
}
}
}
23 changes: 21 additions & 2 deletions src/Agent.Plugins/BuildArtifact/BuildArtifactPluginV1.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ protected static class TaskProperties
public static readonly string RetryDownloadCount = "retryDownloadCount";
public static readonly string ParallelizationLimit = "parallelizationLimit";
public static readonly string CheckDownloadedFiles = "checkDownloadedFiles";
public static readonly string ExtractTars = "extractTars";
}
}

Expand All @@ -73,6 +74,7 @@ public class DownloadBuildArtifactTaskV1_0_0 : BuildArtifactTaskPluginBaseV1
static readonly string buildVersionToDownloadLatest = "latest";
static readonly string buildVersionToDownloadSpecific = "specific";
static readonly string buildVersionToDownloadLatestFromBranch = "latestFromBranch";
static readonly string extractedTarsTempDir = "extracted_tars";
static readonly Options minimatchOptions = new Options() {
Dot = true,
NoBrace = true,
Expand Down Expand Up @@ -106,6 +108,9 @@ protected override async Task ProcessCommandInternalAsync(
string retryDownloadCount = context.GetInput(TaskProperties.RetryDownloadCount, required: false);
string parallelizationLimit = context.GetInput(TaskProperties.ParallelizationLimit, required: false);
string checkDownloadedFiles = context.GetInput(TaskProperties.CheckDownloadedFiles, required: false);
string extractTars = context.GetInput(TaskProperties.ExtractTars, required: false);

string extractedTarsTempPath = Path.Combine(context.Variables.GetValueOrDefault("Agent.TempDirectory")?.Value, extractedTarsTempDir);

targetPath = Path.IsPathFullyQualified(targetPath) ? targetPath : Path.GetFullPath(Path.Combine(defaultWorkingDirectory, targetPath));

Expand Down Expand Up @@ -139,6 +144,16 @@ protected override async Task ProcessCommandInternalAsync(

var resultFilter = GetResultFilter(allowPartiallySucceededBuildsBool, allowFailedBuildsBool, allowCanceledBuildsBool);

if (!bool.TryParse(extractTars, out var extractTarsBool))
{
extractTarsBool = false;
}

if (extractTarsBool && PlatformUtil.RunningOnWindows)
{
throw new ArgumentException(StringUtil.Loc("TarExtractionNotSupportedInWindows"));
}

PipelineArtifactServer server = new PipelineArtifactServer(tracer);
ArtifactDownloadParameters downloadParameters;

Expand Down Expand Up @@ -190,7 +205,9 @@ protected override async Task ProcessCommandInternalAsync(
ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8,
RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4,
CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads,
CustomMinimatchOptions = minimatchOptions
CustomMinimatchOptions = minimatchOptions,
ExtractTars = extractTarsBool,
ExtractedTarsTempPath = extractedTarsTempPath
};
}
else if (buildType == buildTypeSpecific)
Expand Down Expand Up @@ -285,7 +302,9 @@ protected override async Task ProcessCommandInternalAsync(
ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8,
RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4,
CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads,
CustomMinimatchOptions = minimatchOptions
CustomMinimatchOptions = minimatchOptions,
ExtractTars = extractTarsBool,
ExtractedTarsTempPath = extractedTarsTempPath
};
}
else
Expand Down
6 changes: 6 additions & 0 deletions src/Misc/layoutbin/en-US/strings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -599,6 +599,12 @@
"SvnMappingIgnored": "The entire mapping set is ignored. Proceeding with the full branch mapping.",
"SvnNotInstalled": "Can't find installed svn command line utility",
"SvnSyncingRepo": "Syncing repository: {0} (Svn)",
"TarExtraction": "Extracting tar archive: {0}",
"TarExtractionError": "Failed to extract tar archive {0}: {1}",
"TarExtractionNotSupportedInWindows": "Tar extraction is not supported on Windows",
"TarSearchStart": "Starting to search for tar archives to extract",
"TarsFound": "Found {0} tar archives to extract",
"TarsNotFound": "No tar archives were found to extract",
"TaskDownloadFailed": "Failed to download task '{0}'. Error {1}",
"TaskDownloadTimeout": "Task '{0}' didn't finish download within {1} seconds.",
"TaskSignatureVerificationFailed": "Task signature verification failed.",
Expand Down

0 comments on commit d896461

Please sign in to comment.