From 821adebb37488118ec836b25e9761dc9c950b002 Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Fri, 8 Sep 2023 11:11:14 +0200 Subject: [PATCH 1/7] Fix fetching submodules ignoring update rule set to none Bump version to 1.1.1 --- SS14.MapServer/SS14.MapServer.csproj | 2 +- SS14.MapServer/Services/GitService.cs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/SS14.MapServer/SS14.MapServer.csproj b/SS14.MapServer/SS14.MapServer.csproj index bbc5ae5..d62d91e 100644 --- a/SS14.MapServer/SS14.MapServer.csproj +++ b/SS14.MapServer/SS14.MapServer.csproj @@ -5,7 +5,7 @@ enable enable Linux - 1.1 + 1.1.1 diff --git a/SS14.MapServer/Services/GitService.cs b/SS14.MapServer/Services/GitService.cs index 0537dc5..c16d099 100644 --- a/SS14.MapServer/Services/GitService.cs +++ b/SS14.MapServer/Services/GitService.cs @@ -108,6 +108,9 @@ private void Pull(string repoDirectory, string gitRef) _log.Debug("Updating submodules"); foreach (var submodule in repository.Submodules) { + if (submodule.UpdateRule == SubmoduleUpdate.None) + continue; + repository.Submodules.Update(submodule.Name, new SubmoduleUpdateOptions { OnProgress = LogProgress From 75e1ab2ad49219ed46621da57b49b855214226ef Mon Sep 17 00:00:00 2001 From: Julian Giebel Date: Wed, 20 Sep 2023 23:23:58 +0200 Subject: [PATCH 2/7] Replace libgit2sharp (#3) * Work on ssh support for repositories * Add git check to preflight checks Replace libgit2sharp with just running git commands using process --- .../BuildRunners/LocalBuildService.cs | 5 +- .../Configuration/GitConfiguration.cs | 26 +++-- SS14.MapServer/SS14.MapServer.csproj | 1 - SS14.MapServer/Services/GitService.cs | 95 ++++++++++--------- .../Services/StartupCheckService.cs | 26 ++++- 5 files changed, 97 insertions(+), 56 deletions(-) diff --git a/SS14.MapServer/BuildRunners/LocalBuildService.cs b/SS14.MapServer/BuildRunners/LocalBuildService.cs index 1715bc3..c025254 100644 --- a/SS14.MapServer/BuildRunners/LocalBuildService.cs +++ b/SS14.MapServer/BuildRunners/LocalBuildService.cs @@ -72,7 +72,7 @@ public async Task Build(string directory, CancellationToken cancellationToken = _log.Information("Build finished"); } - public async Task Run(string directory, string command, List arguments, bool joinExecutablePath = false, CancellationToken cancellationToken = default) + public async Task Run(string directory, string command, List arguments, bool joinExecutablePath = false, CancellationToken cancellationToken = default) { var executablePath = joinExecutablePath ? Path.Join(directory, command) : command; @@ -114,6 +114,7 @@ public async Task Run(string directory, string command, List arguments, } _log.Information("Run finished"); + return log; } private void SetUpProcess(Process process, string? executable = "dotnet") @@ -147,6 +148,6 @@ private void ProcessLocalBuildException(string log, string fileName, Exception e }); } - [GeneratedRegex("error?|exception")] + [GeneratedRegex("error?|exception|fatal")] private static partial Regex LogErrorRegex(); } diff --git a/SS14.MapServer/Configuration/GitConfiguration.cs b/SS14.MapServer/Configuration/GitConfiguration.cs index 68664fe..2a08cd5 100644 --- a/SS14.MapServer/Configuration/GitConfiguration.cs +++ b/SS14.MapServer/Configuration/GitConfiguration.cs @@ -5,9 +5,9 @@ public sealed class GitConfiguration public const string Name = "Git"; public string RepositoryUrl { get; set; } = string.Empty; - + public string Branch { get; set; } = "master"; - + /// /// If true the map server will retrieve the list of changed maps from the github diff api. /// If this is false all maps will get updated on every push. @@ -24,13 +24,13 @@ public sealed class GitConfiguration { "Resources/Maps/*.yml" }; - + /// /// Glob patterns for excluding specific map files /// public List MapFileExcludePatterns { get; set; } = new(); - + /// /// Prevent updating maps when there where any c# files changed. /// @@ -48,9 +48,23 @@ public sealed class GitConfiguration { "**/*.cs" }; - + /// /// Setting this to true enables listening to the PullRequest event for putting the rendered map as a comment into the PR /// public bool RunOnPullRequests { get; set; } = true; -} \ No newline at end of file + + /// + /// The identity git will use to pull changes with. This doesn't have an effect on anything but is required + /// for pulling changes in some situations + /// + public GitIdentity Identity { get; set; } = new GitIdentity("ss14.mapserver", "git@mapserver.localhost"); + + /// + /// The ssh command used by git if set. Used for providing an ssh key to use. + /// "ssh -i [path to ssh key]" + /// + public string? SshCommand { get; set; } + + public record GitIdentity(string Name, string Email); +} diff --git a/SS14.MapServer/SS14.MapServer.csproj b/SS14.MapServer/SS14.MapServer.csproj index e271f11..b879fef 100644 --- a/SS14.MapServer/SS14.MapServer.csproj +++ b/SS14.MapServer/SS14.MapServer.csproj @@ -12,7 +12,6 @@ - runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/SS14.MapServer/Services/GitService.cs b/SS14.MapServer/Services/GitService.cs index 55cb7fa..a1a134e 100644 --- a/SS14.MapServer/Services/GitService.cs +++ b/SS14.MapServer/Services/GitService.cs @@ -1,5 +1,4 @@ -using LibGit2Sharp; -using LibGit2Sharp.Handlers; +using System.Diagnostics; using Serilog; using SS14.MapServer.BuildRunners; using SS14.MapServer.Configuration; @@ -9,6 +8,8 @@ namespace SS14.MapServer.Services; public sealed class GitService { + private const string GitCommand = "git"; + private readonly LocalBuildService _buildService; private readonly GitConfiguration _configuration = new(); @@ -22,12 +23,28 @@ public GitService(IConfiguration configuration, LocalBuildService buildService) } /// - /// + /// Gets the git version. Used for determining it git is installed + /// + public async Task GetGitVersion(CancellationToken cancellationToken = default) + { + using var process = new Process(); + process.StartInfo.UseShellExecute = false; + process.StartInfo.RedirectStandardOutput = true; + process.StartInfo.RedirectStandardError = true; + process.StartInfo.FileName = GitCommand; + process.StartInfo.Arguments = "--version"; + process.Start(); + await process.WaitForExitAsync(cancellationToken); + return await process.StandardOutput.ReadToEndAsync(cancellationToken); + } + + /// + /// Clones the repo if it hasn't been cloned yet and pulls the provided git ref or the current branch /// /// /// [Optional] The Ref to pull /// - /// + /// The directory the repository is in public string Sync(string workingDirectory, string? gitRef = null, string? repoUrl = null) { gitRef ??= _configuration.Branch; @@ -55,8 +72,9 @@ public string Sync(string workingDirectory, string? gitRef = null, string? repoU /// public string GetRepoCommitHash(string directory) { - using var repository = new Repository(directory); - return repository.Head.Tip.Sha; + var task = _buildService.Run(directory, GitCommand, new List { "rev-parse HEAD" }); + task.Wait(); + return task.Result; } public static string StripRef(string gitRef) @@ -67,25 +85,15 @@ public static string StripRef(string gitRef) private void Clone(string repoUrl, string directory, string gitRef) { _log.Information("Cloning branch/commit {Ref}...", gitRef); - var repoDirectory = Repository.Clone(repoUrl, directory, new CloneOptions - { - RecurseSubmodules = true, - OnProgress = LogProgress - }); - - using var repository = new Repository(repoDirectory); - - Commands.Fetch( - repository, - "origin", - new []{gitRef}, - new FetchOptions - { - OnProgress = LogProgress - }, - null); - - Commands.Checkout(repository, StripRef(gitRef)); + + var sshConfig = _configuration.SshCommand != null + ? $"--config core.sshcommand=\"{_configuration.SshCommand}\"" + : ""; + + RunCommand(Path.GetFullPath("./..", directory), "clone --recurse-submodules", sshConfig, repoUrl); + RunCommand(directory, "fetch -fu origin", gitRef); + RunCommand(directory, "checkout", StripRef(gitRef)); + _log.Information("Done cloning"); } @@ -94,39 +102,36 @@ private void Pull(string repoDirectory, string gitRef) _log.Information( "Pulling branch/commit {Ref}...", gitRef); _log.Debug("Opening repository in: {RepositoryPath}", repoDirectory); - using var repository = new Repository(repoDirectory); - //Set a dummy identity + //Set an identity _log.Debug("Setting identity"); - repository.Config.Set("user.name", "ss14.mapserver"); - repository.Config.Set("user.email", "git@mapserver.localhost"); + RunCommand(repoDirectory, "config user.name", $"\"{_configuration.Identity.Name}\""); + RunCommand(repoDirectory, "config user.email", $"\"{_configuration.Identity.Email}\""); + + if (_configuration.SshCommand != null) + { + _log.Debug("Setting ssh command"); + RunCommand(repoDirectory, "config core.sshcommand", $"\"{_configuration.SshCommand}\""); + } _log.Debug("Fetching ref"); - _buildService.Run(repoDirectory, "git", new List { "fetch -fu origin", gitRef }).Wait(); + RunCommand(repoDirectory, "fetch -fu origin", gitRef); _log.Debug("Checking out {Ref}", StripRef(gitRef)); - Commands.Checkout(repository, StripRef(gitRef)); + RunCommand(repoDirectory, "checkout", StripRef(gitRef)); _log.Debug("Pulling latest changes"); - _buildService.Run(repoDirectory, "git", new List { "pull origin HEAD --ff-only" }).Wait(); + RunCommand(repoDirectory, "pull origin HEAD --ff-only --force"); _log.Debug("Updating submodules"); - foreach (var submodule in repository.Submodules) - { - if (submodule.UpdateRule == SubmoduleUpdate.None) - continue; - - repository.Submodules.Update(submodule.Name, new SubmoduleUpdateOptions - { - OnProgress = LogProgress - }); - } + RunCommand(repoDirectory, "submodule update --init --recursive"); _log.Information("Done pulling"); } - private bool LogProgress(string? progress) + private string RunCommand(string directory, params string[] arguments) { - _log.Verbose("Progress: {Progress}", progress); - return true; + var task = _buildService.Run(directory, GitCommand, new List(arguments)); + task.Wait(); + return task.Result; } } diff --git a/SS14.MapServer/Services/StartupCheckService.cs b/SS14.MapServer/Services/StartupCheckService.cs index f1536d2..1a7d1e8 100644 --- a/SS14.MapServer/Services/StartupCheckService.cs +++ b/SS14.MapServer/Services/StartupCheckService.cs @@ -11,14 +11,16 @@ public sealed class StartupCheckService private readonly BuildConfiguration _buildConfiguration = new(); private readonly GitConfiguration _gitConfiguration = new(); private readonly ProcessingConfiguration _processingConfiguration = new(); + private readonly GitService _gitService; private readonly ContainerService _containerService; private readonly LocalBuildService _localBuildService; - public StartupCheckService(ContainerService containerService, IConfiguration configuration, LocalBuildService localBuildService) + public StartupCheckService(ContainerService containerService, IConfiguration configuration, LocalBuildService localBuildService, GitService gitService) { _containerService = containerService; _localBuildService = localBuildService; + _gitService = gitService; configuration.Bind(FilePathsConfiguration.Name, _filePathsConfiguration); configuration.Bind(BuildConfiguration.Name, _buildConfiguration); configuration.Bind(GitConfiguration.Name, _gitConfiguration); @@ -66,7 +68,9 @@ private async Task CheckBuildServices() if (!_buildConfiguration.Enabled) return true; - return _buildConfiguration.Runner switch + var result = await CheckGitPrerequisites(); + + return result && _buildConfiguration.Runner switch { BuildRunnerName.Local => await CheckLocalRunnerPrerequisites(), BuildRunnerName.Container => await CheckContainerPrerequisites(), @@ -74,6 +78,24 @@ private async Task CheckBuildServices() }; } + private async Task CheckGitPrerequisites() + { + Log.Information("Checking Git:"); + + try + { + var version = await _gitService.GetGitVersion(); + Log.Information(" - Git version: {GitVersion}", version.Replace("\n", "")); + } + catch (Exception e) + { + Log.Fatal(e, "Couldn't determine Git version"); + return false; + } + + return true; + } + private async Task CheckContainerPrerequisites() { Log.Information("Checking container manager:"); From 4cfd5c1aeb3ec4b1547fa339b9db218df6b871dd Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Thu, 21 Sep 2023 10:34:22 +0200 Subject: [PATCH 3/7] Implement reverse proxy header and path base configuration options Bump version number --- .../Configuration/ServerConfiguration.cs | 14 ++++++++++++++ SS14.MapServer/Program.cs | 19 +++++++++++++++++-- SS14.MapServer/SS14.MapServer.csproj | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/SS14.MapServer/Configuration/ServerConfiguration.cs b/SS14.MapServer/Configuration/ServerConfiguration.cs index 8e6e793..5390d22 100644 --- a/SS14.MapServer/Configuration/ServerConfiguration.cs +++ b/SS14.MapServer/Configuration/ServerConfiguration.cs @@ -15,6 +15,20 @@ public class ServerConfiguration /// public bool UseHttps { get; set; } = false; + /// + /// Enables support for reverse proxy headers like "X-Forwarded-Host" if true. Set this to true if run behind a reverse proxy + /// + public bool UseForwardedHeaders { get; set; } = true; + + /// + /// Sets the request base path used before any routes apply i.e. "/base/api/Maps" with "/base" being the PathBase.
+ /// Set this if run behind a reverse proxy on a sub path and the proxy doesn't strip the path the server is hosted on. + /// + /// + /// Add a slash before the path: "/path" + /// + public string? PathBase { get; set; } + public int RateLimitCount { get; set; } = 20; public long RateLimitWindowMinutes { get; set; } = 1; public bool EnableSentry { get; set; } = false; diff --git a/SS14.MapServer/Program.cs b/SS14.MapServer/Program.cs index 73c0f34..9e83eae 100644 --- a/SS14.MapServer/Program.cs +++ b/SS14.MapServer/Program.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.OpenApi.Models; @@ -6,7 +7,6 @@ using Serilog; using SS14.GithubApiHelper.Extensions; using SS14.GithubApiHelper.Services; -using SS14.MapServer; using SS14.MapServer.BuildRunners; using SS14.MapServer.Configuration; using SS14.MapServer.Extensions; @@ -17,7 +17,6 @@ using SS14.MapServer.Services; using SS14.MapServer.Services.Github; using SS14.MapServer.Services.Interfaces; -using Swashbuckle.AspNetCore.SwaggerGen; var builder = WebApplication.CreateBuilder(args); @@ -49,6 +48,12 @@ }); }); +//Forwarded headers +builder.Services.Configure(options => +{ + options.ForwardedHeaders = ForwardedHeaders.All; +}); + //Rate limiting builder.Services.AddApiRateLimiting(serverConfiguration); @@ -137,6 +142,16 @@ Log.Information("Preflight checks passed"); // Configure the HTTP request pipeline. +if (serverConfiguration.PathBase != null) +{ + app.UsePathBase(serverConfiguration.PathBase); +} + +if (serverConfiguration.UseForwardedHeaders) +{ + app.UseForwardedHeaders(); +} + if (app.Environment.IsDevelopment()) { app.UseSwagger(); diff --git a/SS14.MapServer/SS14.MapServer.csproj b/SS14.MapServer/SS14.MapServer.csproj index b879fef..83dbfd6 100644 --- a/SS14.MapServer/SS14.MapServer.csproj +++ b/SS14.MapServer/SS14.MapServer.csproj @@ -5,7 +5,7 @@ enable enable Linux - 1.1.1 + 1.1.2 From b2198245331a2cb6bcf8dea9375f27b18ccfd2b3 Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Thu, 21 Sep 2023 14:29:26 +0200 Subject: [PATCH 4/7] Fix generated urls for maps not taking PathBase into account --- SS14.MapServer/Controllers/GitHubWebhookController.cs | 2 +- SS14.MapServer/Controllers/MapController.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SS14.MapServer/Controllers/GitHubWebhookController.cs b/SS14.MapServer/Controllers/GitHubWebhookController.cs index 02aa9c3..3d5bea4 100644 --- a/SS14.MapServer/Controllers/GitHubWebhookController.cs +++ b/SS14.MapServer/Controllers/GitHubWebhookController.cs @@ -286,7 +286,7 @@ private string GetGridImageUrl(Guid mapGuid, int gridId) { //Can't access Url.Action here return - $"{_serverConfiguration.Host.Scheme}://{_serverConfiguration.Host.Host}:{_serverConfiguration.Host.Port}/api/Image/grid/{mapGuid}/{gridId}"; + $"{_serverConfiguration.Host.Scheme}://{_serverConfiguration.Host.Host}:{_serverConfiguration.Host.Port}{_serverConfiguration.PathBase}/api/Image/grid/{mapGuid}/{gridId}"; } } diff --git a/SS14.MapServer/Controllers/MapController.cs b/SS14.MapServer/Controllers/MapController.cs index bf49382..9ae3108 100644 --- a/SS14.MapServer/Controllers/MapController.cs +++ b/SS14.MapServer/Controllers/MapController.cs @@ -288,7 +288,7 @@ private Map SetMapGridUrls(Map map) "Image", new { id = map.MapGuid, gridId = grid.GridId }, _serverConfiguration.Host.Scheme, - $"{_serverConfiguration.Host.Host}:{_serverConfiguration.Host.Port}" + $"{_serverConfiguration.Host.Host}:{_serverConfiguration.Host.Port}{_serverConfiguration.PathBase}" ); } map.Grids.Sort((grid, grid1) => grid1.Extent.CompareTo(grid.Extent)); From 8c16809e1dec5c78608363d3816c03b0b170a97a Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Sun, 24 Sep 2023 11:40:42 +0200 Subject: [PATCH 5/7] Fix github webook only setup Implement syncing all maps at once --- .../Configuration/BuildConfiguration.cs | 4 +-- .../Controllers/GitHubWebhookController.cs | 17 +++++----- SS14.MapServer/Controllers/MapController.cs | 6 ++-- SS14.MapServer/Jobs/SyncMaps.cs | 16 ++++++++-- SS14.MapServer/MapProcessing/ProcessItem.cs | 7 +++- .../Services/MapUpdateService.cs | 32 ++++++++++++++++--- .../Services/ProcessQueueHostedService.cs | 1 + SS14.MapServer/appsettings.yaml | 1 + 8 files changed, 63 insertions(+), 21 deletions(-) diff --git a/SS14.MapServer/Configuration/BuildConfiguration.cs b/SS14.MapServer/Configuration/BuildConfiguration.cs index 1e880dc..575f945 100644 --- a/SS14.MapServer/Configuration/BuildConfiguration.cs +++ b/SS14.MapServer/Configuration/BuildConfiguration.cs @@ -3,7 +3,7 @@ public sealed class BuildConfiguration { public const string Name = "Build"; - + public bool Enabled { get; set; } = true; public BuildRunnerName Runner { get; set; } = BuildRunnerName.Local; public string RelativeOutputPath { get; set; } = "bin"; @@ -20,4 +20,4 @@ public enum BuildRunnerName { Local, Container -} \ No newline at end of file +} diff --git a/SS14.MapServer/Controllers/GitHubWebhookController.cs b/SS14.MapServer/Controllers/GitHubWebhookController.cs index 3d5bea4..37e950f 100644 --- a/SS14.MapServer/Controllers/GitHubWebhookController.cs +++ b/SS14.MapServer/Controllers/GitHubWebhookController.cs @@ -1,18 +1,15 @@ -using Microsoft.AspNetCore.Authorization; +using System.Collections.Immutable; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; -using Microsoft.IdentityModel.Tokens; using Octokit; using SS14.GithubApiHelper.Helpers; using SS14.MapServer.Configuration; -using SS14.MapServer.Helpers; using SS14.MapServer.MapProcessing; using SS14.MapServer.MapProcessing.Services; using SS14.MapServer.Models; using SS14.MapServer.Models.Entities; -using SS14.MapServer.Services; using SS14.MapServer.Services.Github; namespace SS14.MapServer.Controllers; @@ -123,10 +120,12 @@ private async Task HandlePushEvent(PatchedPushEventPayload payload) if (!payload.Ref.Equals(GitBranchRefPrefix + _gitConfiguration.Branch)) return; - //if (!_gitConfiguration.RetrieveMapFilesFromDiff) - //{ - //TODO: Add a way to just render all maps so the instance doesn'T have to be registered as a github app. - //} + if (!_gitConfiguration.RetrieveMapFilesFromDiff) + { + var processAllItem = new ProcessItem(Path.GetFileName(payload.Ref), ImmutableList.Empty, (_, _) => { }, SyncAll: true); + await _processQueue.TryQueueProcessItem(processAllItem); + return; + } var enumerable = await CheckFiles( payload.Installation.Id, diff --git a/SS14.MapServer/Controllers/MapController.cs b/SS14.MapServer/Controllers/MapController.cs index 9ae3108..838136c 100644 --- a/SS14.MapServer/Controllers/MapController.cs +++ b/SS14.MapServer/Controllers/MapController.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Web; using BrunoZell.ModelBinding; @@ -196,11 +197,12 @@ public async Task DeleteMap(string id, string gitRef) [HttpPost("sync")] [Consumes("application/json")] - public async Task SyncMaps(List mapFileNames) + public async Task SyncMaps(List? mapFileNames, bool syncAll) { var data = new JobDataMap { - {Jobs.SyncMaps.MapListKey, mapFileNames} + {Jobs.SyncMaps.MapListKey, mapFileNames ?? new List() }, + {Jobs.SyncMaps.SyncAllKey, syncAll} }; await _schedulingService.RunJob(nameof(Jobs.SyncMaps), "Sync", data); diff --git a/SS14.MapServer/Jobs/SyncMaps.cs b/SS14.MapServer/Jobs/SyncMaps.cs index 8e0f033..a2ed97b 100644 --- a/SS14.MapServer/Jobs/SyncMaps.cs +++ b/SS14.MapServer/Jobs/SyncMaps.cs @@ -11,12 +11,14 @@ namespace SS14.MapServer.Jobs; public class SyncMaps : IJob { public const string MapListKey = "Maps"; + public const string SyncAllKey = "SyncAll"; public const string GitRefKey = "GitRef"; private readonly ProcessQueue _processQueue; public List? Maps { get; set; } public string GitRef { get; set; } = "master"; + public bool SyncAll { get; set; } = false; public SyncMaps(ProcessQueue processQueue) { @@ -25,13 +27,21 @@ public SyncMaps(ProcessQueue processQueue) public async Task Execute(IJobExecutionContext context) { - if (Maps.IsNullOrEmpty()) + if (Maps.IsNullOrEmpty() && !SyncAll) throw new JobExecutionException($"Job data value with key ${MapListKey} and type List is missing"); - var processItem = new ProcessItem(GitRef, Maps!, (_, value) => - { Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds); }); + var processItem = new ProcessItem( + GitRef, + Maps!, + LogCompletion, + SyncAll: SyncAll); if (!await _processQueue.TryQueueProcessItem(processItem)) throw new JobExecutionException("Failed to start map sync process. Process queue is full."); } + + private void LogCompletion(IServiceProvider _, MapProcessResult value) + { + Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds); + } } diff --git a/SS14.MapServer/MapProcessing/ProcessItem.cs b/SS14.MapServer/MapProcessing/ProcessItem.cs index 9970d55..2aa5f4c 100644 --- a/SS14.MapServer/MapProcessing/ProcessItem.cs +++ b/SS14.MapServer/MapProcessing/ProcessItem.cs @@ -1,3 +1,8 @@ namespace SS14.MapServer.MapProcessing; -public record ProcessItem(string GitRef, IList Maps, Action OnCompletion, string? RepositoryUrl = null); +public record ProcessItem( + string GitRef, + IList Maps, + Action OnCompletion, + string? RepositoryUrl = null, + bool SyncAll = false); diff --git a/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs b/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs index 31d0cfc..b698884 100644 --- a/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs +++ b/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs @@ -1,4 +1,6 @@ -using SS14.MapServer.BuildRunners; +using System.Collections.Immutable; +using Microsoft.Extensions.FileSystemGlobbing; +using SS14.MapServer.BuildRunners; using SS14.MapServer.Configuration; using SS14.MapServer.Services; using SS14.MapServer.Services.Interfaces; @@ -8,6 +10,7 @@ namespace SS14.MapServer.MapProcessing.Services; public sealed class MapUpdateService { private readonly BuildConfiguration _buildConfiguration = new(); + private readonly GitConfiguration _gitConfiguration = new(); private readonly GitService _gitService; private readonly LocalBuildService _localBuildService; private readonly ContainerService _containerService; @@ -25,6 +28,7 @@ public MapUpdateService( _containerService = containerService; _mapReaderService = mapReaderService; configuration.Bind(BuildConfiguration.Name, _buildConfiguration); + configuration.Bind(GitConfiguration.Name, _gitConfiguration); } /// @@ -34,20 +38,41 @@ public MapUpdateService( /// The git ref to pull (branch/commit) /// A list of map file names to be generated /// The clone url of the repository to clone from + /// Ignore the maps parameter and update all maps /// /// /// The commit that was checked out for building and running the map renderer /// /// Syncing the maps doesn't create a new working directory so running this in parallel on the same directory would cause errors.
/// - public async Task UpdateMapsFromGit( - string directory, + public async Task UpdateMapsFromGit(string directory, string gitRef, IEnumerable maps, string? repositoryUrl = null, + bool syncAll = false, CancellationToken cancellationToken = default) { var workingDirectory = _gitService.Sync(directory, gitRef, repositoryUrl); + var strippedGitRef = GitService.StripRef(gitRef); + + if (syncAll) + { + var files = _gitConfiguration.MapFilePatterns + .Select(pattern => Directory.GetFiles(workingDirectory, pattern)) + .SelectMany(x => x) + .Select(file => Path.GetRelativePath(workingDirectory, file)); + + //Check for map files + var mapFileMatcher = new Matcher(); + mapFileMatcher.AddIncludePatterns(_gitConfiguration.MapFilePatterns); + mapFileMatcher.AddExcludePatterns(_gitConfiguration.MapFileExcludePatterns); + var mapFileMatchResult = mapFileMatcher.Match(files); + + if (!mapFileMatchResult.HasMatches) + return new MapProcessResult(strippedGitRef, ImmutableList.Empty); + + maps = mapFileMatchResult.Files.Select(file => Path.GetFileName(file.Path)); + } var command = Path.Join( _buildConfiguration.RelativeOutputPath, @@ -69,7 +94,6 @@ public async Task UpdateMapsFromGit( _ => throw new ArgumentOutOfRangeException() }; - var strippedGitRef = GitService.StripRef(gitRef); var mapIds = await _mapReaderService.UpdateMapsFromFs(path, strippedGitRef, cancellationToken); return new MapProcessResult(strippedGitRef, mapIds); } diff --git a/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs b/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs index f98563e..7957460 100644 --- a/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs +++ b/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs @@ -61,6 +61,7 @@ await mapUpdateService.UpdateMapsFromGit( processItem.GitRef, processItem.Maps, processItem.RepositoryUrl, + processItem.SyncAll, cancellationToken) .ContinueWith( task => diff --git a/SS14.MapServer/appsettings.yaml b/SS14.MapServer/appsettings.yaml index 1f18164..f7c33db 100644 --- a/SS14.MapServer/appsettings.yaml +++ b/SS14.MapServer/appsettings.yaml @@ -8,6 +8,7 @@ Serilog: Microsoft.Hosting.Lifetime: "Information" Microsoft.AspNetCore: "Warning" Microsoft.AspNetCore.DataProtection: "Error" #This service doesn't use data protection + SS14.MapServer.Security.ApiKeyHandler: "Warning" #Ignore "... was not authenticated." spam WriteTo: - Name: Sentry From 629e8e3aaf8c1d8f281f707e5567fdac5fae934f Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Sun, 24 Sep 2023 11:47:42 +0200 Subject: [PATCH 6/7] Bumb version number --- SS14.MapServer/SS14.MapServer.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SS14.MapServer/SS14.MapServer.csproj b/SS14.MapServer/SS14.MapServer.csproj index 83dbfd6..6256682 100644 --- a/SS14.MapServer/SS14.MapServer.csproj +++ b/SS14.MapServer/SS14.MapServer.csproj @@ -5,7 +5,7 @@ enable enable Linux - 1.1.2 + 1.1.3 From 62ce7f47f59a60bb697d0e58f782635f3e3bdad0 Mon Sep 17 00:00:00 2001 From: juliangiebel Date: Mon, 25 Sep 2023 19:07:16 +0200 Subject: [PATCH 7/7] Change git checkout to be forced to discard local changes --- SS14.MapServer/Services/GitService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SS14.MapServer/Services/GitService.cs b/SS14.MapServer/Services/GitService.cs index a1a134e..b893acb 100644 --- a/SS14.MapServer/Services/GitService.cs +++ b/SS14.MapServer/Services/GitService.cs @@ -92,7 +92,7 @@ private void Clone(string repoUrl, string directory, string gitRef) RunCommand(Path.GetFullPath("./..", directory), "clone --recurse-submodules", sshConfig, repoUrl); RunCommand(directory, "fetch -fu origin", gitRef); - RunCommand(directory, "checkout", StripRef(gitRef)); + RunCommand(directory, "checkout --force", StripRef(gitRef)); _log.Information("Done cloning"); } @@ -117,7 +117,7 @@ private void Pull(string repoDirectory, string gitRef) RunCommand(repoDirectory, "fetch -fu origin", gitRef); _log.Debug("Checking out {Ref}", StripRef(gitRef)); - RunCommand(repoDirectory, "checkout", StripRef(gitRef)); + RunCommand(repoDirectory, "checkout --force", StripRef(gitRef)); _log.Debug("Pulling latest changes"); RunCommand(repoDirectory, "pull origin HEAD --ff-only --force");