Skip to content

Commit

Permalink
Fix 'Self-hosted agent cleaning source folder' issue without affectin…
Browse files Browse the repository at this point in the history 
…g env variables (#3318)

* Add L0 test

* Added tests

* Remove extra change

* Fix tests

* Fix one more case when variable value changed

* Fixed test

* Use HasMultipleCheckouts method instead of own implementation

* Fix nit

* Some code style improvements

* Added some comments

* Changed comments

* Code cleanup in tests

* Added check that the entered path does not correspond to the default location for this repository

* Fixed tests

* Added additional test

* Fixed nit

* Self-hosted agent cleaning source folder (#3237)

* Resolved review points

* Removed extra braces

* Rename method

Co-authored-by: Anatoly Bolshakov <[email protected]>
  • Loading branch information
@EzzhevNikita
EzzhevNikita and Anatoly Bolshakov authored Apr 6, 2021
1 parent 4a2d4ee commit 394684b
Show file tree
Hide file tree
Showing 4 changed files with 326 additions and 10 deletions.
22 changes: 18 additions & 4 deletions src/Agent.Worker/Build/BuildDirectoryManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ public TrackingConfig PrepareDirectory(
// Set the default clone path for each repository (the Checkout task may override this later)
foreach (var repository in repositories)
{
var repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig.SourcesDirectory);
string repoPath = GetDefaultRepositoryPath(executionContext, repository, newConfig);

if (!string.Equals(repoPath, defaultSourceDirectory, StringComparison.Ordinal))
{
Expand Down Expand Up @@ -290,18 +290,32 @@ private BuildCleanOption GetBuildDirectoryCleanOption(IExecutionContext executio
private string GetDefaultRepositoryPath(
IExecutionContext executionContext,
RepositoryResource repository,
string defaultSourcesDirectory)
TrackingConfig newConfig
)
{
string repoPath = String.Empty;
string workDirectory = HostContext.GetDirectory(WellKnownDirectory.Work);

if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// If we have multiple checkouts they should all be rooted to the sources directory (_work/1/s/repo1)
return Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory, RepositoryUtil.GetCloneDirectory(repository));
var repoSourceDirectory = newConfig?.RepositoryTrackingInfo.Where(item => string.Equals(item.Identifier, repository.Alias, StringComparison.OrdinalIgnoreCase)).Select(item => item.SourcesDirectory).FirstOrDefault();
if (repoSourceDirectory != null)
{
repoPath = Path.Combine(workDirectory, repoSourceDirectory);
}
else
{
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory, RepositoryUtil.GetCloneDirectory(repository));
}
}
else
{
// For single checkouts, the repository is rooted to the sources folder (_work/1/s)
return Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Work), defaultSourcesDirectory);
repoPath = Path.Combine(workDirectory, newConfig.SourcesDirectory);
}

return repoPath;
}
}
}
57 changes: 56 additions & 1 deletion src/Agent.Worker/Build/BuildJobExtension.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ public override void InitializeJobExtension(IExecutionContext executionContext,
string pipelineWorkspaceDirectory = Path.Combine(_workDirectory, trackingConfig.BuildDirectory);

UpdateCheckoutTasksAndVariables(executionContext, steps, pipelineWorkspaceDirectory);

// Get default value for RepoLocalPath variable
string selfRepoPath = GetDefaultRepoLocalPathValue(executionContext, steps, trackingConfig, repoInfo);

// Set the directory variables.
executionContext.Output(StringUtil.Loc("SetBuildVars"));
Expand All @@ -177,7 +180,7 @@ public override void InitializeJobExtension(IExecutionContext executionContext,
executionContext.SetVariable(Constants.Variables.Build.SourcesDirectory, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.StagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.ArtifactStagingDirectory, Path.Combine(_workDirectory, trackingConfig.ArtifactsDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, trackingConfig.SourcesDirectory), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, selfRepoPath), isFilePath: true);
executionContext.SetVariable(Constants.Variables.Pipeline.Workspace, pipelineWorkspaceDirectory, isFilePath: true);
}

Expand Down Expand Up @@ -339,6 +342,58 @@ private string ConvertToLegacyRepositoryType(string pipelineRepositoryType)
}
}

private string GetDefaultRepoLocalPathValue(IExecutionContext executionContext, IList<Pipelines.JobStep> steps, TrackingConfig trackingConfig, RepositoryInfo repoInfo)
{
string selfRepoPath = null;
// For saving backward compatibility with the behavior of the Build.RepoLocalPath that was before this PR https://github.com/microsoft/azure-pipelines-agent/pull/3237
// We need to change how we set the default value of this variable
// We need to allow the setting of paths from RepositoryTrackingInfo for checkout tasks where path input was provided by the user
// and this input is not point to the default location for this repository
// This is the only case where the value of Build.RepoLocalPath variable is not pointing to the root of sources directory /s.
// The new logic is not affecting single checkout jobs and jobs with multiple checkouts and default paths for Self repository
if (RepositoryUtil.HasMultipleCheckouts(executionContext.JobSettings))
{
// get checkout task for self repo
var selfCheckoutTask = GetSelfCheckoutTask(steps);

// Check if the task has path input with custom path, if so we need to set as a value of selfRepoPath the value of SourcesDirectory from RepositoryTrackingInfo
if (IsCheckoutToCustomPath(trackingConfig, repoInfo, selfCheckoutTask))
{
selfRepoPath = trackingConfig.RepositoryTrackingInfo
.Where(repo => RepositoryUtil.IsPrimaryRepositoryName(repo.Identifier))
.Select(props => props.SourcesDirectory).FirstOrDefault();
}
}
// For single checkout jobs and multicheckout jobs with default paths set selfRepoPath to the default sources directory
if (selfRepoPath == null)
{
selfRepoPath = trackingConfig.SourcesDirectory;
}

return selfRepoPath;
}

private bool IsCheckoutToCustomPath(TrackingConfig trackingConfig, RepositoryInfo repoInfo, TaskStep selfCheckoutTask)
{
string path;
string selfRepoName = RepositoryUtil.GetCloneDirectory(repoInfo.PrimaryRepository.Properties.Get<string>(Pipelines.RepositoryPropertyNames.Name));
string defaultRepoCheckoutPath = Path.GetFullPath(Path.Combine(trackingConfig.SourcesDirectory, selfRepoName));

return selfCheckoutTask != null
&& selfCheckoutTask.Inputs.TryGetValue(PipelineConstants.CheckoutTaskInputs.Path, out path)
&& !string.Equals(Path.GetFullPath(Path.Combine(trackingConfig.BuildDirectory, path)),
defaultRepoCheckoutPath,
IOUtil.FilePathStringComparison);
}

private TaskStep GetSelfCheckoutTask(IList<JobStep> steps)
{
return steps.Select(x => x as TaskStep)
.Where(task => task.IsCheckoutTask()
&& task.Inputs.TryGetValue(PipelineConstants.CheckoutTaskInputs.Repository, out string repositoryAlias)
&& RepositoryUtil.IsPrimaryRepositoryName(repositoryAlias)).FirstOrDefault();
}

private class RepositoryInfo
{
public Pipelines.RepositoryResource PrimaryRepository { set; get; }
Expand Down
21 changes: 16 additions & 5 deletions src/Agent.Worker/PluginInternalCommandExtension.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,19 +60,30 @@ public void Execute(IExecutionContext context, Command command)
bool isSelfRepo = RepositoryUtil.IsPrimaryRepositoryName(repository.Alias);
bool hasMultipleCheckouts = RepositoryUtil.HasMultipleCheckouts(context.JobSettings);

var directoryManager = context.GetHostContext().GetService<IBuildDirectoryManager>();
string _workDirectory = context.GetHostContext().GetDirectory(WellKnownDirectory.Work);
var trackingConfig = directoryManager.UpdateDirectory(context, repository);

if (isSelfRepo || !hasMultipleCheckouts)
{
var directoryManager = context.GetHostContext().GetService<IBuildDirectoryManager>();
string _workDirectory = context.GetHostContext().GetDirectory(WellKnownDirectory.Work);

var trackingConfig = directoryManager.UpdateDirectory(context, repository);
if (hasMultipleCheckouts)
{
// In Multi-checkout, we don't want to reset sources dir or default working dir.
// So, we will just reset the repo local path
string buildDirectory = context.Variables.Get(Constants.Variables.Pipeline.Workspace);
string repoRelativePath = directoryManager.GetRelativeRepositoryPath(buildDirectory, repositoryPath);
context?.SetVariable(Constants.Variables.Build.RepoLocalPath, Path.Combine(_workDirectory, repoRelativePath), isFilePath: true);

string sourcesDirectory = context.Variables.Get(Constants.Variables.Build.SourcesDirectory);
string repoLocalPath = context.Variables.Get(Constants.Variables.Build.RepoLocalPath);
string newRepoLocation = Path.Combine(_workDirectory, repoRelativePath);
// For saving backward compatibility with the behavior of the Build.RepoLocalPath that was before this PR https://github.com/microsoft/azure-pipelines-agent/pull/3237
// we need to deny updating of the variable in case the new path is the default location for the repository that is equal to sourcesDirectory/repository.Name
// since the variable already has the right value in this case and pointing to the default sources location
if (repoLocalPath == null
|| !string.Equals(newRepoLocation, Path.Combine(sourcesDirectory, repository.Name), IOUtil.FilePathStringComparison))
{
context?.SetVariable(Constants.Variables.Build.RepoLocalPath, newRepoLocation, isFilePath: true);
}
}
else
{
Expand Down
Loading

0 comments on commit 394684b

Please sign in to comment.