Add Archive source option

src/Microsoft.Crank.Agent/Startup.cs

@@ -2585,6 +2585,39 @@ private static async Task RetrieveSourceAsync(Source source, string destinationF
                     await Git.InitSubModulesAsync(targetDir);
                 }
             }
+            else if (!String.IsNullOrEmpty(source.Archive))
+            {
+                // If archive is remote, download it first
+                var archivePath = source.Archive;
+
+                if (archivePath.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                {
+                    var tempArchive = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".zip");
+
+                    Log.Info($"Downloading archive from '{archivePath}' to '{tempArchive}'");
+
+                    var ok = await DownloadFileAsync(archivePath, tempArchive, maxRetries: 3, timeout: 60, throwOnError: true);
+
+                    if (!ok)
+                    {
+                        throw new InvalidOperationException($"Could not download archive from {archivePath}");
+                    }
+
+                    ZipFile.ExtractToDirectory(tempArchive, targetDir);
+
+                    try { File.Delete(tempArchive); } catch { }
+                }
+                else
+                {
+                    // Local path
+                    if (!File.Exists(archivePath))
+                    {
+                        throw new InvalidOperationException($"Archive not found: {archivePath}");
+                    }
+
+                    ZipFile.ExtractToDirectory(archivePath, targetDir);
+                }
+            }
 
             if (!targetDir.Equals(destinationFolder))
             {

src/Microsoft.Crank.Controller/Documentation.cs

@@ -15,6 +15,7 @@ These options are specific to a job instance. Replace [JOB] by the name of the s
   --[JOB].sources.[SOURCE].branchOrCommit                       A branch name or commit hash, e.g., my/branch, my/branch#commit-hash
   --[JOB].sources.[SOURCE].initSubmodules                       Whether to init submodules when a git repository is used, e.g., true
   --[JOB].sources.[SOURCE].localFolder                          The local path containing the source code to upload to the server. e.g., /code/mybenchmarks
+  --[JOB].sources.[SOURCE].archive                              A URL or local path to a zip archive that contains the source code. If set, the controller will upload the archive to the agent and it will be extracted. e.g., https://example.com/myapp.zip or ./myapp.zip
 
   ## .NET options
 

src/Microsoft.Crank.Controller/JobConnection.cs

@@ -169,6 +169,10 @@ public async Task<string> StartAsync(string jobName)
                         {
                             uploadLocalSourceTasks.Add(UploadLocalSourceAsync(sourceName, source.LocalFolder));
                         }
+                        else if (!String.IsNullOrEmpty(source.Archive) && !source.Archive.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                        {
+                            uploadLocalSourceTasks.Add(UploadLocalArchiveAsync(sourceName, source.Archive));
+                        }
                     }
 
                     await Task.WhenAll(uploadLocalSourceTasks);
@@ -408,6 +412,24 @@ private async Task UploadLocalSourceAsync(string sourceName, string sourceDir)
             }
         }
 
+        private async Task UploadLocalArchiveAsync(string sourceName, string archivePath)
+        {
+            if (!File.Exists(archivePath))
+            {
+                throw new ControllerException($"Archive not found: {archivePath}");
+            }
+
+            Log.Write($"Using local archive: \"{archivePath}\"");
+
+            var uploadUri = $"{Combine(_serverJobUri, "/source")}?sourceName={sourceName}";
+            var result = await UploadFileAsync(archivePath, uploadUri, gzipped: false);
+
+            if (result != 0)
+            {
+                throw new Exception("Error while uploading source archive");
+            }
+        }
+
         private async Task HandleBuildFileAsync(string buildFileValue)
         {
             var buildFileSegments = buildFileValue.Split(';', 2, StringSplitOptions.RemoveEmptyEntries);

src/Microsoft.Crank.Controller/Program.cs

@@ -2083,6 +2083,20 @@ int interval
                     }
                 }
 
+                // Validate that each source has at least one code location defined
+                foreach (var (sourceName, source) in job.Sources)
+                {
+                    var hasRepo = !String.IsNullOrEmpty(source.Repository);
+                    var hasLocal = !String.IsNullOrEmpty(source.LocalFolder);
+                    var hasArchive = !String.IsNullOrEmpty(source.Archive);
+                    var hasSourceCode = source.SourceCode != null;
+
+                    if (!hasRepo && !hasLocal && !hasArchive && !hasSourceCode)
+                    {
+                        throw new ControllerException($"Invalid job '{jobName}': source '{sourceName}' must define at least one of 'repository', 'localFolder', 'archive' or an uploaded 'sourceCode'.");
+                    }
+                }
+
                 if (job.CollectCounters)
                 {
                     Log.WriteWarning($"WARNING: '{jobName}.collectCounters' has been deprecated, in the future please use '{jobName}.options.collectCounters'.");
@@ -2415,13 +2429,15 @@ public static async Task<JObject> LoadConfigurationAsync(string configurationFil
                         if (jobObject.ContainsKey("source"))
                         {
                             PatchLocalFolderInSource(configurationFilenameOrUrl, (JObject)jobObject["source"]);
+                            PatchArchiveInSource(configurationFilenameOrUrl, (JObject)jobObject["source"]);
                         }
 
                         if (jobObject.ContainsKey("sources"))
                         {
                             foreach (JProperty source in jobObject["sources"])
                             {
                                 PatchLocalFolderInSource(configurationFilenameOrUrl, (JObject)source.Value);
+                                PatchArchiveInSource(configurationFilenameOrUrl, (JObject)source.Value);
                             }
                         }
                     }
@@ -2479,6 +2495,22 @@ private static void PatchLocalFolderInSource(string configurationFilenameOrUrl,
             }
         }
 
+        private static void PatchArchiveInSource(string configurationFilenameOrUrl, JObject source)
+        {
+            if (source.ContainsKey("archive"))
+            {
+                var archive = source["archive"].ToString();
+
+                if (!archive.StartsWith("http", StringComparison.OrdinalIgnoreCase))
+                {
+                    var configurationFilename = new FileInfo(configurationFilenameOrUrl).FullName;
+                    var resolvedFilename = new FileInfo(Path.Combine(Path.GetDirectoryName(configurationFilename), archive)).FullName;
+
+                    source["archive"] = resolvedFilename;
+                }
+            }
+        }
+
         /// <summary>
         /// Merges a JObject into another one.
         /// </summary>

src/Microsoft.Crank.Controller/README.md

@@ -64,6 +64,7 @@ Run 'crank [command] -?|-h|--help' for more information about a command.
   --[JOB].sources.[SOURCE].branchOrCommit                       A branch name or commit hash, e.g., my/branch, my/branch#commit-hash
   --[JOB].sources.[SOURCE].initSubmodules                       Whether to init submodules when a git repository is used, e.g., true
   --[JOB].sources.[SOURCE].localFolder                          The local path containing the source code to upload to the server. e.g., /code/mybenchmarks
+  --[JOB].sources.[SOURCE].archive                              A URL or local path to a zip archive that contains the source code. If set, the controller will upload the archive to the agent and it will be extracted. e.g., https://example.com/myapp.zip or ./myapp.zip
 
   ## Execution
 

src/Microsoft.Crank.Models/Job.cs

@@ -105,6 +105,7 @@ public Source Source
                             DockerPull = DockerPull,
                             InitSubmodules = source.InitSubmodules,
                             LocalFolder = source.LocalFolder,
+                            Archive = source.Archive,
                             NoBuild = NoBuild,
                             Project = Project,
                             Repository = source.Repository,

src/Microsoft.Crank.Models/Source.cs

@@ -17,6 +17,7 @@ public class Source
         public string Repository { get; set; }
         public bool InitSubmodules { get; set; }
         public string LocalFolder { get; set; }
+    public string Archive { get; set; }
 
         /// <summary>
         /// When set, will specify where the source data will be copied to on the agent.
@@ -65,7 +66,8 @@ public SourceKeyData GetSourceKeyData()
                 BranchOrCommit = BranchOrCommit,
                 Repository = Repository,
                 InitSubmodules = InitSubmodules,
-                LocalFolder = LocalFolder
+                LocalFolder = LocalFolder,
+                Archive = Archive
             };
         }
     }
@@ -79,5 +81,6 @@ public class SourceKeyData
         public string Repository { get; set; }
         public bool InitSubmodules { get; set; }
         public string LocalFolder { get; set; }
+        public string Archive { get; set; }
     }
 }

test/Microsoft.Crank.IntegrationTests/CommonTests.cs

@@ -81,6 +81,52 @@ public async Task BenchmarkHello()
             Assert.Contains("ASP.NET Core Version", result.StandardOutput);
         }
 
+        [Fact]
+        public async Task BenchmarkHelloArchive()
+        {
+            _output.WriteLine($"[TEST] Starting controller (archive)");
+
+            // Create a zip next to the hello folder called hello.zip and use the permanent config
+            var assetsDir = Path.Combine(_crankTestsDirectory, "assets");
+
+            // The original config references ../hello relative to the assets file
+            var helloFolder = Path.GetFullPath(Path.Combine(assetsDir, "..", "hello"));
+
+            Assert.True(Directory.Exists(helloFolder), $"Expected hello folder at {helloFolder}");
+
+            var helloZip = Path.GetFullPath(Path.Combine(assetsDir, "..", "hello.zip"));
+
+            if (File.Exists(helloZip)) File.Delete(helloZip);
+
+            System.IO.Compression.ZipFile.CreateFromDirectory(helloFolder, helloZip, System.IO.Compression.CompressionLevel.Fastest, includeBaseDirectory: false);
+
+            var configPath = Path.Combine(_crankTestsDirectory, "assets", "hello-archive.benchmarks.yml");
+
+            try
+            {
+                var result = await ProcessUtil.RunAsync(
+                    "dotnet",
+                    $"exec {Path.Combine(_crankDirectory, "crank.dll")} --config {configPath} --scenario hello --profile local",
+                    workingDirectory: _crankTestsDirectory,
+                    captureOutput: true,
+                    timeout: DefaultTimeOut,
+                    throwOnError: false,
+                    outputDataReceived: t => { _output.WriteLine($"[CTL] {t}"); }
+                );
+
+                Assert.Equal(0, result.ExitCode);
+
+                Assert.Contains("Requests/sec", result.StandardOutput);
+                Assert.Contains(".NET Core SDK Version", result.StandardOutput);
+                Assert.Contains(".NET Runtime Version", result.StandardOutput);
+                Assert.Contains("ASP.NET Core Version", result.StandardOutput);
+            }
+            finally
+            {
+                try { File.Delete(helloZip); } catch { }
+            }
+        }
+
         [Fact]
         public async Task ExecutesScripts()
         {

test/Microsoft.Crank.IntegrationTests/assets/hello-archive.benchmarks.yml

@@ -0,0 +1,50 @@
+jobs:
+  server:
+    sources:
+      hello:
+        archive: '../hello.zip'
+    project: hello/hello.csproj
+    readyStateText: Application started.
+  bombardier:
+    sources:
+      local:
+        # uploading the whole source folder since it requires other libraries
+        localFolder: '../src'
+        destinationFolder: ''
+    project: Microsoft.Crank.Jobs.Bombardier/Microsoft.Crank.Jobs.Bombardier.csproj
+    readyStateText: Bombardier Client
+    waitForExit: true
+    variables:
+      connections: 256
+      warmup: 3
+      duration: 3
+      requests: 0
+      rate: 0
+      transport: fasthttp # | http1 | http2
+      serverScheme: http
+      serverAddress: localhost
+      serverPort: 5000
+      customHeaders: [ ] # list of headers with the format: '<name1>: <value1>', e.g. [ 'content-type: application/json' ]
+    arguments: "-c {{connections}} -w {{warmup}} -d {{duration}} -n {{requests}} --insecure -l {% if rate != 0 %} --rate {{ rate }} {% endif %} {% if transport %} --{{ transport}} {% endif %} {{headers[presetHeaders]}} {% for h in customHeaders %}{% assign s = h | split : ':' %}--header \"{{ s[0] }}: {{ s[1] | strip }}\" {% endfor %} {{serverScheme}}://{{serverAddress}}:{{serverPort}}{{path}}"
+
+scenarios:
+  hello:
+    application:
+      job: server
+    load:
+      job: bombardier
+      variables:
+        path: /
+
+profiles:
+  local:
+    variables:
+      serverPort: 5000
+      serverAddress: localhost
+    jobs:
+      application:
+        endpoints:
+          - http://localhost:5010
+      load:
+        endpoints:
+          - http://localhost:5010