From d8bcda3cecdc01ab50e2adafe25d14bf04757943 Mon Sep 17 00:00:00 2001 From: Julian Giebel Date: Sun, 24 Sep 2023 11:51:46 +0200 Subject: [PATCH] Implement syncing all maps at once (#7) * Fix github webook only setup Implement syncing all maps at once * Bumb version number to 1.1.3 --- .../Configuration/BuildConfiguration.cs | 4 +-- .../Controllers/GitHubWebhookController.cs | 17 +++++----- SS14.MapServer/Controllers/MapController.cs | 6 ++-- SS14.MapServer/Jobs/SyncMaps.cs | 16 ++++++++-- SS14.MapServer/MapProcessing/ProcessItem.cs | 7 +++- .../Services/MapUpdateService.cs | 32 ++++++++++++++++--- .../Services/ProcessQueueHostedService.cs | 1 + SS14.MapServer/SS14.MapServer.csproj | 2 +- SS14.MapServer/appsettings.yaml | 1 + 9 files changed, 64 insertions(+), 22 deletions(-) diff --git a/SS14.MapServer/Configuration/BuildConfiguration.cs b/SS14.MapServer/Configuration/BuildConfiguration.cs index 1e880dc..575f945 100644 --- a/SS14.MapServer/Configuration/BuildConfiguration.cs +++ b/SS14.MapServer/Configuration/BuildConfiguration.cs @@ -3,7 +3,7 @@ public sealed class BuildConfiguration { public const string Name = "Build"; - + public bool Enabled { get; set; } = true; public BuildRunnerName Runner { get; set; } = BuildRunnerName.Local; public string RelativeOutputPath { get; set; } = "bin"; @@ -20,4 +20,4 @@ public enum BuildRunnerName { Local, Container -} \ No newline at end of file +} diff --git a/SS14.MapServer/Controllers/GitHubWebhookController.cs b/SS14.MapServer/Controllers/GitHubWebhookController.cs index 3d5bea4..37e950f 100644 --- a/SS14.MapServer/Controllers/GitHubWebhookController.cs +++ b/SS14.MapServer/Controllers/GitHubWebhookController.cs @@ -1,18 +1,15 @@ -using Microsoft.AspNetCore.Authorization; +using System.Collections.Immutable; +using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.FileSystemGlobbing; -using Microsoft.Extensions.FileSystemGlobbing.Abstractions; -using Microsoft.IdentityModel.Tokens; using Octokit; using SS14.GithubApiHelper.Helpers; using SS14.MapServer.Configuration; -using SS14.MapServer.Helpers; using SS14.MapServer.MapProcessing; using SS14.MapServer.MapProcessing.Services; using SS14.MapServer.Models; using SS14.MapServer.Models.Entities; -using SS14.MapServer.Services; using SS14.MapServer.Services.Github; namespace SS14.MapServer.Controllers; @@ -123,10 +120,12 @@ private async Task HandlePushEvent(PatchedPushEventPayload payload) if (!payload.Ref.Equals(GitBranchRefPrefix + _gitConfiguration.Branch)) return; - //if (!_gitConfiguration.RetrieveMapFilesFromDiff) - //{ - //TODO: Add a way to just render all maps so the instance doesn'T have to be registered as a github app. - //} + if (!_gitConfiguration.RetrieveMapFilesFromDiff) + { + var processAllItem = new ProcessItem(Path.GetFileName(payload.Ref), ImmutableList.Empty, (_, _) => { }, SyncAll: true); + await _processQueue.TryQueueProcessItem(processAllItem); + return; + } var enumerable = await CheckFiles( payload.Installation.Id, diff --git a/SS14.MapServer/Controllers/MapController.cs b/SS14.MapServer/Controllers/MapController.cs index 9ae3108..838136c 100644 --- a/SS14.MapServer/Controllers/MapController.cs +++ b/SS14.MapServer/Controllers/MapController.cs @@ -1,3 +1,4 @@ +using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Web; using BrunoZell.ModelBinding; @@ -196,11 +197,12 @@ public async Task DeleteMap(string id, string gitRef) [HttpPost("sync")] [Consumes("application/json")] - public async Task SyncMaps(List mapFileNames) + public async Task SyncMaps(List? mapFileNames, bool syncAll) { var data = new JobDataMap { - {Jobs.SyncMaps.MapListKey, mapFileNames} + {Jobs.SyncMaps.MapListKey, mapFileNames ?? new List() }, + {Jobs.SyncMaps.SyncAllKey, syncAll} }; await _schedulingService.RunJob(nameof(Jobs.SyncMaps), "Sync", data); diff --git a/SS14.MapServer/Jobs/SyncMaps.cs b/SS14.MapServer/Jobs/SyncMaps.cs index 8e0f033..a2ed97b 100644 --- a/SS14.MapServer/Jobs/SyncMaps.cs +++ b/SS14.MapServer/Jobs/SyncMaps.cs @@ -11,12 +11,14 @@ namespace SS14.MapServer.Jobs; public class SyncMaps : IJob { public const string MapListKey = "Maps"; + public const string SyncAllKey = "SyncAll"; public const string GitRefKey = "GitRef"; private readonly ProcessQueue _processQueue; public List? Maps { get; set; } public string GitRef { get; set; } = "master"; + public bool SyncAll { get; set; } = false; public SyncMaps(ProcessQueue processQueue) { @@ -25,13 +27,21 @@ public SyncMaps(ProcessQueue processQueue) public async Task Execute(IJobExecutionContext context) { - if (Maps.IsNullOrEmpty()) + if (Maps.IsNullOrEmpty() && !SyncAll) throw new JobExecutionException($"Job data value with key ${MapListKey} and type List is missing"); - var processItem = new ProcessItem(GitRef, Maps!, (_, value) => - { Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds); }); + var processItem = new ProcessItem( + GitRef, + Maps!, + LogCompletion, + SyncAll: SyncAll); if (!await _processQueue.TryQueueProcessItem(processItem)) throw new JobExecutionException("Failed to start map sync process. Process queue is full."); } + + private void LogCompletion(IServiceProvider _, MapProcessResult value) + { + Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds); + } } diff --git a/SS14.MapServer/MapProcessing/ProcessItem.cs b/SS14.MapServer/MapProcessing/ProcessItem.cs index 9970d55..2aa5f4c 100644 --- a/SS14.MapServer/MapProcessing/ProcessItem.cs +++ b/SS14.MapServer/MapProcessing/ProcessItem.cs @@ -1,3 +1,8 @@ namespace SS14.MapServer.MapProcessing; -public record ProcessItem(string GitRef, IList Maps, Action OnCompletion, string? RepositoryUrl = null); +public record ProcessItem( + string GitRef, + IList Maps, + Action OnCompletion, + string? RepositoryUrl = null, + bool SyncAll = false); diff --git a/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs b/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs index 31d0cfc..b698884 100644 --- a/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs +++ b/SS14.MapServer/MapProcessing/Services/MapUpdateService.cs @@ -1,4 +1,6 @@ -using SS14.MapServer.BuildRunners; +using System.Collections.Immutable; +using Microsoft.Extensions.FileSystemGlobbing; +using SS14.MapServer.BuildRunners; using SS14.MapServer.Configuration; using SS14.MapServer.Services; using SS14.MapServer.Services.Interfaces; @@ -8,6 +10,7 @@ namespace SS14.MapServer.MapProcessing.Services; public sealed class MapUpdateService { private readonly BuildConfiguration _buildConfiguration = new(); + private readonly GitConfiguration _gitConfiguration = new(); private readonly GitService _gitService; private readonly LocalBuildService _localBuildService; private readonly ContainerService _containerService; @@ -25,6 +28,7 @@ public MapUpdateService( _containerService = containerService; _mapReaderService = mapReaderService; configuration.Bind(BuildConfiguration.Name, _buildConfiguration); + configuration.Bind(GitConfiguration.Name, _gitConfiguration); } /// @@ -34,20 +38,41 @@ public MapUpdateService( /// The git ref to pull (branch/commit) /// A list of map file names to be generated /// The clone url of the repository to clone from + /// Ignore the maps parameter and update all maps /// /// /// The commit that was checked out for building and running the map renderer /// /// Syncing the maps doesn't create a new working directory so running this in parallel on the same directory would cause errors.
/// - public async Task UpdateMapsFromGit( - string directory, + public async Task UpdateMapsFromGit(string directory, string gitRef, IEnumerable maps, string? repositoryUrl = null, + bool syncAll = false, CancellationToken cancellationToken = default) { var workingDirectory = _gitService.Sync(directory, gitRef, repositoryUrl); + var strippedGitRef = GitService.StripRef(gitRef); + + if (syncAll) + { + var files = _gitConfiguration.MapFilePatterns + .Select(pattern => Directory.GetFiles(workingDirectory, pattern)) + .SelectMany(x => x) + .Select(file => Path.GetRelativePath(workingDirectory, file)); + + //Check for map files + var mapFileMatcher = new Matcher(); + mapFileMatcher.AddIncludePatterns(_gitConfiguration.MapFilePatterns); + mapFileMatcher.AddExcludePatterns(_gitConfiguration.MapFileExcludePatterns); + var mapFileMatchResult = mapFileMatcher.Match(files); + + if (!mapFileMatchResult.HasMatches) + return new MapProcessResult(strippedGitRef, ImmutableList.Empty); + + maps = mapFileMatchResult.Files.Select(file => Path.GetFileName(file.Path)); + } var command = Path.Join( _buildConfiguration.RelativeOutputPath, @@ -69,7 +94,6 @@ public async Task UpdateMapsFromGit( _ => throw new ArgumentOutOfRangeException() }; - var strippedGitRef = GitService.StripRef(gitRef); var mapIds = await _mapReaderService.UpdateMapsFromFs(path, strippedGitRef, cancellationToken); return new MapProcessResult(strippedGitRef, mapIds); } diff --git a/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs b/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs index f98563e..7957460 100644 --- a/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs +++ b/SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs @@ -61,6 +61,7 @@ await mapUpdateService.UpdateMapsFromGit( processItem.GitRef, processItem.Maps, processItem.RepositoryUrl, + processItem.SyncAll, cancellationToken) .ContinueWith( task => diff --git a/SS14.MapServer/SS14.MapServer.csproj b/SS14.MapServer/SS14.MapServer.csproj index 83dbfd6..6256682 100644 --- a/SS14.MapServer/SS14.MapServer.csproj +++ b/SS14.MapServer/SS14.MapServer.csproj @@ -5,7 +5,7 @@ enable enable Linux - 1.1.2 + 1.1.3 diff --git a/SS14.MapServer/appsettings.yaml b/SS14.MapServer/appsettings.yaml index 1f18164..f7c33db 100644 --- a/SS14.MapServer/appsettings.yaml +++ b/SS14.MapServer/appsettings.yaml @@ -8,6 +8,7 @@ Serilog: Microsoft.Hosting.Lifetime: "Information" Microsoft.AspNetCore: "Warning" Microsoft.AspNetCore.DataProtection: "Error" #This service doesn't use data protection + SS14.MapServer.Security.ApiKeyHandler: "Warning" #Ignore "... was not authenticated." spam WriteTo: - Name: Sentry