From d896461e80664934b94ca03b7808561c61a9e2e9 Mon Sep 17 00:00:00 2001 From: DaniilShmelev <72494759+DaniilShmelev@users.noreply.github.com> Date: Mon, 23 Aug 2021 11:55:51 +0300 Subject: [PATCH] [DownloadBuildArtifactsV1] Tar extraction on download (#3467) * add tar extraction feature * fixes * fix download parameters passing * add no tars found warning also some other minor logging improvements * fix 'StandardError has not been redirected' * fix invalid StreamReader method * extract `GetArtifactItems()` * rename string for tar search start * get items outside of `DownloadFileContainerAsync` * move tar extraction to DownloadArtifact methods * fix 'directory with the same name already exists' * account for incompatible path resolution too * fix paths * trim relative artifact path for slash apparently `Path.Combine` doesn't work properly if the second argument looks like an absolute path * only move extracted tars directory if tars found * fix `MoveDirectory` * replace files in destination * fix destination folder * create target directory * add comments Co-authored-by: Anatoly Bolshakov --- .../Artifact/ArtifactDownloadParameters.cs | 2 + .../Artifact/FileContainerProvider.cs | 155 ++++++++++++++++-- .../BuildArtifact/BuildArtifactPluginV1.cs | 23 ++- src/Misc/layoutbin/en-US/strings.json | 6 + 4 files changed, 172 insertions(+), 14 deletions(-) diff --git a/src/Agent.Plugins/Artifact/ArtifactDownloadParameters.cs b/src/Agent.Plugins/Artifact/ArtifactDownloadParameters.cs index 7a31146b23..2d2c3a46e4 100644 --- a/src/Agent.Plugins/Artifact/ArtifactDownloadParameters.cs +++ b/src/Agent.Plugins/Artifact/ArtifactDownloadParameters.cs @@ -31,6 +31,8 @@ internal class ArtifactDownloadParameters public int RetryDownloadCount { get; set; } = 4; public bool CheckDownloadedFiles { get; set; } = false; public Options CustomMinimatchOptions { get; set; } = null; + public bool ExtractTars { get; set; } = false; + public string ExtractedTarsTempPath { get; set; } public bool AppendArtifactNameToTargetPath { get; set; } = true; } diff --git a/src/Agent.Plugins/Artifact/FileContainerProvider.cs b/src/Agent.Plugins/Artifact/FileContainerProvider.cs index b082368b39..5ca76a856a 100644 --- a/src/Agent.Plugins/Artifact/FileContainerProvider.cs +++ b/src/Agent.Plugins/Artifact/FileContainerProvider.cs @@ -17,6 +17,7 @@ using Microsoft.VisualStudio.Services.WebApi; using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.IO.Compression; using System.Linq; @@ -44,17 +45,41 @@ public FileContainerProvider(VssConnection connection, IAppTraceSource tracer) public async Task DownloadSingleArtifactAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { - await this.DownloadFileContainerAsync(downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken); + IEnumerable items = await GetArtifactItems(downloadParameters, buildArtifact); + await this.DownloadFileContainerAsync(items, downloadParameters, buildArtifact, downloadParameters.TargetDirectory, context, cancellationToken); + + IEnumerable fileArtifactPaths = items + .Where((item) => item.ItemType == ContainerItemType.File) + .Select((fileItem) => Path.Combine(downloadParameters.TargetDirectory, fileItem.Path)); + + if (downloadParameters.ExtractTars) + { + ExtractTarsIfPresent(context, fileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath); + } } public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters downloadParameters, IEnumerable buildArtifacts, CancellationToken cancellationToken, AgentTaskPluginExecutionContext context) { + var allFileArtifactPaths = new List(); + foreach (var buildArtifact in buildArtifacts) { var dirPath = downloadParameters.AppendArtifactNameToTargetPath ? Path.Combine(downloadParameters.TargetDirectory, buildArtifact.Name) : downloadParameters.TargetDirectory; - await DownloadFileContainerAsync(downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload: false); + + IEnumerable items = await GetArtifactItems(downloadParameters, buildArtifact); + IEnumerable fileArtifactPaths = items + .Where((item) => item.ItemType == ContainerItemType.File) + .Select((fileItem) => Path.Combine(dirPath, fileItem.Path)); + allFileArtifactPaths.AddRange(fileArtifactPaths); + + await DownloadFileContainerAsync(items, downloadParameters, buildArtifact, dirPath, context, cancellationToken, isSingleArtifactDownload: false); + } + + if (downloadParameters.ExtractTars) + { + ExtractTarsIfPresent(context, allFileArtifactPaths, downloadParameters.TargetDirectory, downloadParameters.ExtractedTarsTempPath); } } @@ -84,21 +109,12 @@ public async Task DownloadMultipleArtifactsAsync(ArtifactDownloadParameters down } } - private async Task DownloadFileContainerAsync(ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) + private async Task DownloadFileContainerAsync(IEnumerable items, ArtifactDownloadParameters downloadParameters, BuildArtifact artifact, string rootPath, AgentTaskPluginExecutionContext context, CancellationToken cancellationToken, bool isSingleArtifactDownload = true) { var containerIdAndRoot = ParseContainerId(artifact.Resource.Data); var projectId = downloadParameters.ProjectId; - var minimatchPatterns = downloadParameters.MinimatchFilters; - - var items = await containerClient.QueryContainerItemsAsync(containerIdAndRoot.Item1, projectId, isShallow: false, includeBlobMetadata: true, containerIdAndRoot.Item2); tracer.Info($"Start downloading FCS artifact- {artifact.Name}"); - IEnumerable> minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs(minimatchPatterns, tracer, downloadParameters.CustomMinimatchOptions); - - if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) - { - items = this.GetFilteredItems(items, minimatcherFuncs); - } if (!isSingleArtifactDownload && items.Any()) { @@ -190,6 +206,35 @@ await AsyncHttpRetryHelper.InvokeVoidAsync( } } + // Returns all artifact items. Uses minimatch filters specified in downloadParameters. + private async Task> GetArtifactItems(ArtifactDownloadParameters downloadParameters, BuildArtifact buildArtifact) + { + (long, string) containerIdAndRoot = ParseContainerId(buildArtifact.Resource.Data); + Guid projectId = downloadParameters.ProjectId; + string[] minimatchPatterns = downloadParameters.MinimatchFilters; + + List items = await containerClient.QueryContainerItemsAsync( + containerIdAndRoot.Item1, + projectId, + isShallow: false, + includeBlobMetadata: true, + containerIdAndRoot.Item2 + ); + + IEnumerable> minimatcherFuncs = MinimatchHelper.GetMinimatchFuncs( + minimatchPatterns, + tracer, + downloadParameters.CustomMinimatchOptions + ); + + if (minimatcherFuncs != null && minimatcherFuncs.Count() != 0) + { + items = this.GetFilteredItems(items, minimatcherFuncs); + } + + return items; + } + private void CheckDownloads(IEnumerable items, string rootPath, string artifactName, bool includeArtifactName) { tracer.Info(StringUtil.Loc("BeginArtifactItemsIntegrityCheck")); @@ -323,5 +368,91 @@ private List GetFilteredItems(List items, } return filteredItems; } + + // Checks all specified artifact paths, searches for files ending with '.tar'. + // If any files were found, extracts them to extractedTarsTempPath and moves to rootPath/extracted_tars. + private void ExtractTarsIfPresent(AgentTaskPluginExecutionContext context, IEnumerable fileArtifactPaths, string rootPath, string extractedTarsTempPath) + { + tracer.Info(StringUtil.Loc("TarSearchStart")); + + int tarsFoundCount = 0; + + foreach (var fileArtifactPath in fileArtifactPaths) + { + if (fileArtifactPath.EndsWith(".tar")) + { + tarsFoundCount += 1; + + // fileArtifactPath is a combination of rootPath and the relative artifact path + string relativeFileArtifactPath = fileArtifactPath.Substring(rootPath.Length); + string relativeFileArtifactDirPath = Path.GetDirectoryName(relativeFileArtifactPath).TrimStart('/'); + string extractedFilesDir = Path.Combine(extractedTarsTempPath, relativeFileArtifactDirPath); + + ExtractTar(fileArtifactPath, extractedFilesDir); + + File.Delete(fileArtifactPath); + } + } + + if (tarsFoundCount == 0) { + context.Warning(StringUtil.Loc("TarsNotFound")); + } else { + tracer.Info(StringUtil.Loc("TarsFound", tarsFoundCount)); + + string targetDirectory = Path.Combine(rootPath, "extracted_tars"); + Directory.CreateDirectory(targetDirectory); + MoveDirectory(extractedTarsTempPath, targetDirectory); + } + } + + // Extracts tar archive at tarArchivePath to extractedFilesDir. + // Uses 'tar' utility like this: tar xf `tarArchivePath` --directory `extractedFilesDir`. + // Throws if any errors are encountered. + private void ExtractTar(string tarArchivePath, string extractedFilesDir) + { + tracer.Info(StringUtil.Loc("TarExtraction", tarArchivePath)); + + Directory.CreateDirectory(extractedFilesDir); + var extractionProcessInfo = new ProcessStartInfo("tar") + { + Arguments = $"xf {tarArchivePath} --directory {extractedFilesDir}", + UseShellExecute = false, + RedirectStandardError = true + }; + Process extractionProcess = Process.Start(extractionProcessInfo); + extractionProcess.WaitForExit(); + + var extractionStderr = extractionProcess.StandardError.ReadToEnd(); + if (extractionStderr.Length != 0 || extractionProcess.ExitCode != 0) + { + throw new Exception(StringUtil.Loc("TarExtractionError", tarArchivePath, extractionStderr)); + } + } + + // Recursively moves sourcePath directory to targetPath + private void MoveDirectory(string sourcePath, string targetPath) { + var sourceDirectoryInfo = new DirectoryInfo(sourcePath); + foreach (FileInfo file in sourceDirectoryInfo.GetFiles("*", SearchOption.TopDirectoryOnly)) + { + file.MoveTo(Path.Combine(targetPath, file.Name), true); + } + foreach (DirectoryInfo subdirectory in sourceDirectoryInfo.GetDirectories("*", SearchOption.TopDirectoryOnly)) + { + string subdirectoryDestinationPath = Path.Combine(targetPath, subdirectory.Name); + var subdirectoryDestination = new DirectoryInfo(subdirectoryDestinationPath); + + if (subdirectoryDestination.Exists) + { + MoveDirectory( + Path.Combine(sourcePath, subdirectory.Name), + Path.Combine(targetPath, subdirectory.Name) + ); + } + else + { + subdirectory.MoveTo(Path.Combine(targetPath, subdirectory.Name)); + } + } + } } } diff --git a/src/Agent.Plugins/BuildArtifact/BuildArtifactPluginV1.cs b/src/Agent.Plugins/BuildArtifact/BuildArtifactPluginV1.cs index 42f8e3065d..208e89aadb 100644 --- a/src/Agent.Plugins/BuildArtifact/BuildArtifactPluginV1.cs +++ b/src/Agent.Plugins/BuildArtifact/BuildArtifactPluginV1.cs @@ -60,6 +60,7 @@ protected static class TaskProperties public static readonly string RetryDownloadCount = "retryDownloadCount"; public static readonly string ParallelizationLimit = "parallelizationLimit"; public static readonly string CheckDownloadedFiles = "checkDownloadedFiles"; + public static readonly string ExtractTars = "extractTars"; } } @@ -73,6 +74,7 @@ public class DownloadBuildArtifactTaskV1_0_0 : BuildArtifactTaskPluginBaseV1 static readonly string buildVersionToDownloadLatest = "latest"; static readonly string buildVersionToDownloadSpecific = "specific"; static readonly string buildVersionToDownloadLatestFromBranch = "latestFromBranch"; + static readonly string extractedTarsTempDir = "extracted_tars"; static readonly Options minimatchOptions = new Options() { Dot = true, NoBrace = true, @@ -106,6 +108,9 @@ protected override async Task ProcessCommandInternalAsync( string retryDownloadCount = context.GetInput(TaskProperties.RetryDownloadCount, required: false); string parallelizationLimit = context.GetInput(TaskProperties.ParallelizationLimit, required: false); string checkDownloadedFiles = context.GetInput(TaskProperties.CheckDownloadedFiles, required: false); + string extractTars = context.GetInput(TaskProperties.ExtractTars, required: false); + + string extractedTarsTempPath = Path.Combine(context.Variables.GetValueOrDefault("Agent.TempDirectory")?.Value, extractedTarsTempDir); targetPath = Path.IsPathFullyQualified(targetPath) ? targetPath : Path.GetFullPath(Path.Combine(defaultWorkingDirectory, targetPath)); @@ -139,6 +144,16 @@ protected override async Task ProcessCommandInternalAsync( var resultFilter = GetResultFilter(allowPartiallySucceededBuildsBool, allowFailedBuildsBool, allowCanceledBuildsBool); + if (!bool.TryParse(extractTars, out var extractTarsBool)) + { + extractTarsBool = false; + } + + if (extractTarsBool && PlatformUtil.RunningOnWindows) + { + throw new ArgumentException(StringUtil.Loc("TarExtractionNotSupportedInWindows")); + } + PipelineArtifactServer server = new PipelineArtifactServer(tracer); ArtifactDownloadParameters downloadParameters; @@ -190,7 +205,9 @@ protected override async Task ProcessCommandInternalAsync( ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8, RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4, CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads, - CustomMinimatchOptions = minimatchOptions + CustomMinimatchOptions = minimatchOptions, + ExtractTars = extractTarsBool, + ExtractedTarsTempPath = extractedTarsTempPath }; } else if (buildType == buildTypeSpecific) @@ -285,7 +302,9 @@ protected override async Task ProcessCommandInternalAsync( ParallelizationLimit = int.TryParse(parallelizationLimit, out var parallelLimit) ? parallelLimit : 8, RetryDownloadCount = int.TryParse(retryDownloadCount, out var retryCount) ? retryCount : 4, CheckDownloadedFiles = bool.TryParse(checkDownloadedFiles, out var checkDownloads) && checkDownloads, - CustomMinimatchOptions = minimatchOptions + CustomMinimatchOptions = minimatchOptions, + ExtractTars = extractTarsBool, + ExtractedTarsTempPath = extractedTarsTempPath }; } else diff --git a/src/Misc/layoutbin/en-US/strings.json b/src/Misc/layoutbin/en-US/strings.json index 1be2932121..84951d15de 100644 --- a/src/Misc/layoutbin/en-US/strings.json +++ b/src/Misc/layoutbin/en-US/strings.json @@ -599,6 +599,12 @@ "SvnMappingIgnored": "The entire mapping set is ignored. Proceeding with the full branch mapping.", "SvnNotInstalled": "Can't find installed svn command line utility", "SvnSyncingRepo": "Syncing repository: {0} (Svn)", + "TarExtraction": "Extracting tar archive: {0}", + "TarExtractionError": "Failed to extract tar archive {0}: {1}", + "TarExtractionNotSupportedInWindows": "Tar extraction is not supported on Windows", + "TarSearchStart": "Starting to search for tar archives to extract", + "TarsFound": "Found {0} tar archives to extract", + "TarsNotFound": "No tar archives were found to extract", "TaskDownloadFailed": "Failed to download task '{0}'. Error {1}", "TaskDownloadTimeout": "Task '{0}' didn't finish download within {1} seconds.", "TaskSignatureVerificationFailed": "Task signature verification failed.",