Implement syncing all maps at once (#7)

* Fix github webook only setup
Implement syncing all maps at once

* Bumb version number to 1.1.3

SS14.MapServer/Configuration/BuildConfiguration.cs

@@ -3,7 +3,7 @@
 public sealed class BuildConfiguration
 {
     public const string Name = "Build";
-    
+
     public bool Enabled { get; set; } = true;
     public BuildRunnerName Runner { get; set; } = BuildRunnerName.Local;
     public string RelativeOutputPath { get; set; } = "bin";
@@ -20,4 +20,4 @@ public enum BuildRunnerName
 {
     Local,
     Container
-}
+}

SS14.MapServer/Controllers/GitHubWebhookController.cs

@@ -1,18 +1,15 @@
-using Microsoft.AspNetCore.Authorization;
+using System.Collections.Immutable;
+using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.FileSystemGlobbing;
-using Microsoft.Extensions.FileSystemGlobbing.Abstractions;
-using Microsoft.IdentityModel.Tokens;
 using Octokit;
 using SS14.GithubApiHelper.Helpers;
 using SS14.MapServer.Configuration;
-using SS14.MapServer.Helpers;
 using SS14.MapServer.MapProcessing;
 using SS14.MapServer.MapProcessing.Services;
 using SS14.MapServer.Models;
 using SS14.MapServer.Models.Entities;
-using SS14.MapServer.Services;
 using SS14.MapServer.Services.Github;
 
 namespace SS14.MapServer.Controllers;
@@ -123,10 +120,12 @@ private async Task HandlePushEvent(PatchedPushEventPayload payload)
         if (!payload.Ref.Equals(GitBranchRefPrefix + _gitConfiguration.Branch))
             return;
 
-        //if (!_gitConfiguration.RetrieveMapFilesFromDiff)
-        //{
-            //TODO: Add a way to just render all maps so the instance doesn'T have to be registered as a github app.
-        //}
+        if (!_gitConfiguration.RetrieveMapFilesFromDiff)
+        {
+            var processAllItem = new ProcessItem(Path.GetFileName(payload.Ref), ImmutableList<string>.Empty, (_, _) => { }, SyncAll: true);
+            await _processQueue.TryQueueProcessItem(processAllItem);
+            return;
+        }
 
         var enumerable = await CheckFiles(
             payload.Installation.Id,

SS14.MapServer/Controllers/MapController.cs

@@ -1,3 +1,4 @@
+using System.Collections.Immutable;
 using System.Diagnostics.CodeAnalysis;
 using System.Web;
 using BrunoZell.ModelBinding;
@@ -196,11 +197,12 @@ public async Task<IActionResult> DeleteMap(string id, string gitRef)
 
     [HttpPost("sync")]
     [Consumes("application/json")]
-    public async Task<IActionResult> SyncMaps(List<string> mapFileNames)
+    public async Task<IActionResult> SyncMaps(List<string>? mapFileNames, bool syncAll)
     {
         var data = new JobDataMap
         {
-            {Jobs.SyncMaps.MapListKey, mapFileNames}
+            {Jobs.SyncMaps.MapListKey, mapFileNames ?? new List<string>() },
+            {Jobs.SyncMaps.SyncAllKey, syncAll}
         };
 
         await _schedulingService.RunJob<Jobs.SyncMaps>(nameof(Jobs.SyncMaps), "Sync", data);

SS14.MapServer/Jobs/SyncMaps.cs

@@ -11,12 +11,14 @@ namespace SS14.MapServer.Jobs;
 public class SyncMaps : IJob
 {
     public const string MapListKey = "Maps";
+    public const string SyncAllKey = "SyncAll";
     public const string GitRefKey = "GitRef";
 
     private readonly ProcessQueue _processQueue;
 
     public List<string>? Maps { get; set; }
     public string GitRef { get; set; } = "master";
+    public bool SyncAll { get; set; } = false;
 
     public SyncMaps(ProcessQueue processQueue)
     {
@@ -25,13 +27,21 @@ public SyncMaps(ProcessQueue processQueue)
 
     public async Task Execute(IJobExecutionContext context)
     {
-        if (Maps.IsNullOrEmpty())
+        if (Maps.IsNullOrEmpty() && !SyncAll)
             throw new JobExecutionException($"Job data value with key ${MapListKey} and type List<string> is missing");
 
-        var processItem = new ProcessItem(GitRef, Maps!,  (_, value) =>
-            { Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds); });
+        var processItem = new ProcessItem(
+            GitRef,
+            Maps!,
+            LogCompletion,
+            SyncAll: SyncAll);
 
         if (!await _processQueue.TryQueueProcessItem(processItem))
             throw new JobExecutionException("Failed to start map sync process. Process queue is full.");
     }
+
+    private void LogCompletion(IServiceProvider _, MapProcessResult value)
+    {
+        Log.Debug("Finished processing maps for branch/commit {GitRef}. {MapIds}", value.GitRef, value.MapIds);
+    }
 }

SS14.MapServer/MapProcessing/ProcessItem.cs

@@ -1,3 +1,8 @@
 namespace SS14.MapServer.MapProcessing;
 
-public record ProcessItem(string GitRef, IList<string> Maps, Action<IServiceProvider, MapProcessResult> OnCompletion, string? RepositoryUrl = null);
+public record ProcessItem(
+    string GitRef,
+    IList<string> Maps,
+    Action<IServiceProvider, MapProcessResult> OnCompletion,
+    string? RepositoryUrl = null,
+    bool SyncAll = false);

SS14.MapServer/MapProcessing/Services/MapUpdateService.cs

@@ -1,4 +1,6 @@
-using SS14.MapServer.BuildRunners;
+using System.Collections.Immutable;
+using Microsoft.Extensions.FileSystemGlobbing;
+using SS14.MapServer.BuildRunners;
 using SS14.MapServer.Configuration;
 using SS14.MapServer.Services;
 using SS14.MapServer.Services.Interfaces;
@@ -8,6 +10,7 @@ namespace SS14.MapServer.MapProcessing.Services;
 public sealed class MapUpdateService
 {
     private readonly BuildConfiguration _buildConfiguration = new();
+    private readonly GitConfiguration _gitConfiguration = new();
     private readonly GitService _gitService;
     private readonly LocalBuildService _localBuildService;
     private readonly ContainerService _containerService;
@@ -25,6 +28,7 @@ public MapUpdateService(
         _containerService = containerService;
         _mapReaderService = mapReaderService;
         configuration.Bind(BuildConfiguration.Name, _buildConfiguration);
+        configuration.Bind(GitConfiguration.Name, _gitConfiguration);
     }
 
     /// <summary>
@@ -34,20 +38,41 @@ public MapUpdateService(
     /// <param name="gitRef">The git ref to pull (branch/commit)</param>
     /// <param name="maps">A list of map file names to be generated</param>
     /// <param name="repositoryUrl">The clone url of the repository to clone from</param>
+    /// <param name="syncAll">Ignore the maps parameter and update all maps</param>
     /// <param name="cancellationToken"></param>
     /// <exception cref="ArgumentOutOfRangeException"></exception>
     /// <returns>The commit that was checked out for building and running the map renderer</returns>
     /// <remarks>
     /// Syncing the maps doesn't create a new working directory so running this in parallel on the same directory would cause errors.<br/>
     /// </remarks>
-    public async Task<MapProcessResult> UpdateMapsFromGit(
-        string directory,
+    public async Task<MapProcessResult> UpdateMapsFromGit(string directory,
         string gitRef,
         IEnumerable<string> maps,
         string? repositoryUrl = null,
+        bool syncAll = false,
         CancellationToken cancellationToken = default)
     {
         var workingDirectory = _gitService.Sync(directory, gitRef, repositoryUrl);
+        var strippedGitRef = GitService.StripRef(gitRef);
+
+        if (syncAll)
+        {
+            var files = _gitConfiguration.MapFilePatterns
+                .Select(pattern => Directory.GetFiles(workingDirectory, pattern))
+                .SelectMany(x => x)
+                .Select(file => Path.GetRelativePath(workingDirectory, file));
+
+            //Check for map files
+            var mapFileMatcher = new Matcher();
+            mapFileMatcher.AddIncludePatterns(_gitConfiguration.MapFilePatterns);
+            mapFileMatcher.AddExcludePatterns(_gitConfiguration.MapFileExcludePatterns);
+            var mapFileMatchResult = mapFileMatcher.Match(files);
+
+            if (!mapFileMatchResult.HasMatches)
+                return new MapProcessResult(strippedGitRef, ImmutableList<Guid>.Empty);
+
+            maps = mapFileMatchResult.Files.Select(file => Path.GetFileName(file.Path));
+        }
 
         var command = Path.Join(
             _buildConfiguration.RelativeOutputPath,
@@ -69,7 +94,6 @@ public async Task<MapProcessResult> UpdateMapsFromGit(
             _ => throw new ArgumentOutOfRangeException()
         };
 
-        var strippedGitRef = GitService.StripRef(gitRef);
         var mapIds = await _mapReaderService.UpdateMapsFromFs(path,  strippedGitRef, cancellationToken);
         return new MapProcessResult(strippedGitRef, mapIds);
     }

SS14.MapServer/MapProcessing/Services/ProcessQueueHostedService.cs

@@ -61,6 +61,7 @@ await mapUpdateService.UpdateMapsFromGit(
                                 processItem.GitRef,
                                 processItem.Maps,
                                 processItem.RepositoryUrl,
+                                processItem.SyncAll,
                                 cancellationToken)
                             .ContinueWith(
                                 task =>

SS14.MapServer/SS14.MapServer.csproj

@@ -5,7 +5,7 @@
         <Nullable>enable</Nullable>
         <ImplicitUsings>enable</ImplicitUsings>
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
-        <Version>1.1.2</Version>
+        <Version>1.1.3</Version>
     </PropertyGroup>
 
     <ItemGroup>

SS14.MapServer/appsettings.yaml

@@ -8,6 +8,7 @@ Serilog:
       Microsoft.Hosting.Lifetime: "Information"
       Microsoft.AspNetCore: "Warning"
       Microsoft.AspNetCore.DataProtection: "Error" #This service doesn't use data protection
+      SS14.MapServer.Security.ApiKeyHandler: "Warning" #Ignore "... was not authenticated." spam
 
   WriteTo:
     - Name: Sentry