From 10d69abb4e0d3d492c0c4613effb7ac13801fd75 Mon Sep 17 00:00:00 2001
From: June Rhodes <jrhodes@redpoint.games>
Date: Sat, 23 Nov 2024 18:32:23 +1100
Subject: [PATCH] Add support for predefining tests and using them in `uet
 test` (#56)

---
 .../Plugin/Custom/CustomPluginTestProvider.cs |   1 -
 .../BuildGraphGeneratorTests.cs               |   1 +
 .../DynamicBuildGraphIncludeTests.cs          |   1 +
 .../BuildGraph/BuildGraph_Plugin.xml          |   7 +-
 .../DefaultDynamicBuildGraphIncludeWriter.cs  |  58 +++-
 .../IDynamicBuildGraphIncludeWriter.cs        |   2 +
 .../BuildConfigSourceGenerationContext.cs     |   1 +
 .../Dynamic/BuildConfigDynamic.cs             |   1 +
 .../Dynamic/BuildConfigDynamicConverter.cs    | 166 ++--------
 .../BuildConfigDynamicConverterHelpers.cs     | 208 ++++++++++++
 .../Dynamic/BuildConfigPredefinedDynamic.cs   |  25 ++
 .../BuildConfigPredefinedDynamicConverter.cs  |  95 ++++++
 .../BuildConfigTestPredefinedConverter.cs     |  13 +
 .../Plugin/BuildConfigPlugin.cs               |   8 +
 ...dConfigPluginPredefinedTestDependencies.cs |  28 ++
 UET/uet/Commands/Build/BuildCommand.cs        |   4 +-
 .../DefaultBuildSpecificationGenerator.cs     |  11 +-
 .../Build/IBuildSpecificationGenerator.cs     |   4 +-
 .../DefaultJsonSchemaGenerator.cs             |  35 ++
 UET/uet/Commands/Test/TestCommand.cs          | 312 ++++++++++++------
 20 files changed, 728 insertions(+), 253 deletions(-)
 create mode 100644 UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverterHelpers.cs
 create mode 100644 UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamic.cs
 create mode 100644 UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamicConverter.cs
 create mode 100644 UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigTestPredefinedConverter.cs
 create mode 100644 UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPluginPredefinedTestDependencies.cs

diff --git a/UET/Redpoint.Uet.BuildPipeline.Providers.Test/Plugin/Custom/CustomPluginTestProvider.cs b/UET/Redpoint.Uet.BuildPipeline.Providers.Test/Plugin/Custom/CustomPluginTestProvider.cs
index dbbb3c5e..9ad10d23 100644
--- a/UET/Redpoint.Uet.BuildPipeline.Providers.Test/Plugin/Custom/CustomPluginTestProvider.cs
+++ b/UET/Redpoint.Uet.BuildPipeline.Providers.Test/Plugin/Custom/CustomPluginTestProvider.cs
@@ -22,7 +22,6 @@ public CustomPluginTestProvider(
 
         public IRuntimeJson DynamicSettings { get; } = new TestProviderRuntimeJson(TestProviderSourceGenerationContext.WithStringEnum).BuildConfigPluginTestCustom;
 
-
         public async Task WriteBuildGraphNodesAsync(
             IBuildGraphEmitContext context,
             XmlWriter writer,
diff --git a/UET/Redpoint.Uet.BuildPipeline.Tests/BuildGraphGeneratorTests.cs b/UET/Redpoint.Uet.BuildPipeline.Tests/BuildGraphGeneratorTests.cs
index 6d919d54..137b45b5 100644
--- a/UET/Redpoint.Uet.BuildPipeline.Tests/BuildGraphGeneratorTests.cs
+++ b/UET/Redpoint.Uet.BuildPipeline.Tests/BuildGraphGeneratorTests.cs
@@ -128,6 +128,7 @@ public async void CanGenerateBuildGraphForPlugin()
                     { $"Allow2019", $"false" },
                     { $"EnginePrefix", $"Unreal" },
                     { $"ExecutePackage", $"false" },
+                    { $"ExecuteZip", $"false" },
                     { $"VersionNumber", $"10000" },
                     { $"VersionName", $"2023.05.17" },
                     { $"PackageName", $"Packaged" },
diff --git a/UET/Redpoint.Uet.BuildPipeline.Tests/DynamicBuildGraphIncludeTests.cs b/UET/Redpoint.Uet.BuildPipeline.Tests/DynamicBuildGraphIncludeTests.cs
index b2594674..6318be65 100644
--- a/UET/Redpoint.Uet.BuildPipeline.Tests/DynamicBuildGraphIncludeTests.cs
+++ b/UET/Redpoint.Uet.BuildPipeline.Tests/DynamicBuildGraphIncludeTests.cs
@@ -52,6 +52,7 @@ public async Task TestPluginTests()
                 await writer.WriteBuildGraphNodeInclude(
                     memory,
                     false,
+                    new BuildConfigPlugin(),
                     new BuildConfigPluginDistribution
                     {
                         Name = "Test",
diff --git a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/BuildGraph_Plugin.xml b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/BuildGraph_Plugin.xml
index 9da5e5f3..f68d99a9 100644
--- a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/BuildGraph_Plugin.xml
+++ b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/BuildGraph_Plugin.xml
@@ -34,6 +34,7 @@
 
   <!-- Package options -->
   <Option Name="ExecutePackage" Restrict="true|false" DefaultValue="true" Description="If false, the default Create Package and Zip Plugin steps are not run" />
+  <Option Name="ExecuteZip" Restrict="true|false" DefaultValue="true" Description="If false, the Create Package step will not generate a .zip file" />
   <Option Name="VersionNumber" Restrict="[0-9]+" DefaultValue="10000" Description="The version number to embed in the packaged .uplugin file" />
   <Option Name="VersionName" Restrict="[^ ]+" DefaultValue="Unversioned" Description="The version name to embed in the packaged .uplugin file" />
   <Option Name="PackageFolder" Restrict="[^ ]+" DefaultValue="Packaged" Description="The folder to place the packaged plugin in" />
@@ -606,7 +607,9 @@ $(PackageExclude);
       OutputTag="#PackagedPlugin"
       If="'$(ExecutePackage)' == 'true'"
     />
-    <Agent Name="Zip Plugin (Create Final Package)" Type="Win64">
+    <Property Name="PackageTasks" Value="$(PackageTasks)#PackagedPlugin;" />
+
+    <Agent Name="Zip Plugin (Create Final Package)" Type="Win64" If="'$(ExecuteZip)' == 'true'">
       <Node Name="Zip Plugin" Requires="#PackagedPlugin" Produces="#PackagedZip">
         <Delete Files="$(ProjectRoot)/$(PluginName)-$(Distribution)-$(VersionName).zip" />
         <Zip
@@ -617,7 +620,7 @@ $(PackageExclude);
       />
       </Node>
     </Agent>
-    <Property Name="PackageTasks" Value="$(PackageTasks)#PackagedZip;#PackagedPlugin;" />
+    <Property Name="PackageTasks" Value="$(PackageTasks)#PackagedZip;" If="'$(ExecuteZip)' == 'true'" />
   </Do>
 
   <!-- 
diff --git a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/DefaultDynamicBuildGraphIncludeWriter.cs b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/DefaultDynamicBuildGraphIncludeWriter.cs
index 4c69e983..59570879 100644
--- a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/DefaultDynamicBuildGraphIncludeWriter.cs
+++ b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/DefaultDynamicBuildGraphIncludeWriter.cs
@@ -80,9 +80,41 @@ private static async Task WriteBuildGraphNodesAsync<TDistribution, TBaseClass>(
             XmlWriter writer,
             TDistribution buildConfigDistribution,
             IDynamicProvider<TDistribution, TBaseClass>[] providers,
-            BuildConfigDynamic<TDistribution, TBaseClass>[] dynamicSettings)
+            BuildConfigDynamic<TDistribution, TBaseClass>[] dynamicSettings,
+            BuildConfigDynamic<TDistribution, TBaseClass>[] predefinedDynamicSettings)
         {
-            foreach (var byType in dynamicSettings.GroupBy(x => x.Type))
+            var predefinedDynamicSettingsLookup = predefinedDynamicSettings == null
+                ? new Dictionary<string, BuildConfigDynamic<TDistribution, TBaseClass>>()
+                : predefinedDynamicSettings.ToDictionary(k => k.Name, v => v);
+
+            var resolvedDynamicSettings = new List<BuildConfigDynamic<TDistribution, TBaseClass>>();
+            foreach (var dynamicSetting in dynamicSettings)
+            {
+                if (dynamicSetting.Type == BuildConfigConstants.Predefined)
+                {
+                    if (dynamicSetting.DynamicSettings is string predefinedName)
+                    {
+                        if (predefinedDynamicSettingsLookup.TryGetValue(predefinedName, out var resolvedDynamicSetting))
+                        {
+                            resolvedDynamicSettings.Add(resolvedDynamicSetting);
+                        }
+                        else
+                        {
+                            throw new InvalidOperationException($"Entry '{dynamicSetting.Name}' refers to predefined entry by name '{predefinedName}', but no such predefined entry is available.");
+                        }
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException($"Entry '{dynamicSetting.Name}' has invalid 'Predefined' value; it should be a string value.");
+                    }
+                }
+                else
+                {
+                    resolvedDynamicSettings.Add(dynamicSetting);
+                }
+            }
+
+            foreach (var byType in resolvedDynamicSettings.GroupBy(x => x.Type))
             {
                 var provider = providers.First(x => x.Type == byType.Key);
                 await provider.WriteBuildGraphNodesAsync(
@@ -96,6 +128,7 @@ await provider.WriteBuildGraphNodesAsync(
         public async Task WriteBuildGraphNodeInclude(
             Stream stream,
             bool filterHostToCurrentPlatformOnly,
+            object buildConfig,
             object buildConfigDistribution,
             bool executeTests,
             bool executeDeployment)
@@ -124,7 +157,10 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             pluginDistribution,
                             _pluginTests,
-                            pluginDistribution.Tests).ConfigureAwait(false);
+                            pluginDistribution.Tests,
+                            buildConfig is BuildConfigPlugin buildConfigPlugin && buildConfigPlugin.Tests != null
+                                ? buildConfigPlugin.Tests
+                                : []).ConfigureAwait(false);
                     }
 
                     if (pluginDistribution.Deployment != null && executeDeployment)
@@ -134,7 +170,8 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             pluginDistribution,
                             _pluginDeployments,
-                            pluginDistribution.Deployment).ConfigureAwait(false);
+                            pluginDistribution.Deployment,
+                            []).ConfigureAwait(false);
                     }
                 }
                 else if (buildConfigDistribution is BuildConfigProjectDistribution projectDistribution)
@@ -146,7 +183,8 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             projectDistribution,
                             _projectTests,
-                            projectDistribution.Tests).ConfigureAwait(false);
+                            projectDistribution.Tests,
+                            []).ConfigureAwait(false);
                     }
 
                     if (projectDistribution.Deployment != null && executeDeployment)
@@ -156,7 +194,8 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             projectDistribution,
                             _projectDeployments,
-                            projectDistribution.Deployment).ConfigureAwait(false);
+                            projectDistribution.Deployment,
+                            []).ConfigureAwait(false);
                     }
                 }
                 else
@@ -172,6 +211,7 @@ await WriteBuildGraphNodesAsync(
         public async Task WriteBuildGraphMacroInclude(
             Stream stream,
             bool filterHostToCurrentPlatformOnly,
+            object buildConfig,
             object buildConfigDistribution)
         {
             var emitContext = new BuildGraphEmitContext(
@@ -198,7 +238,8 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             pluginDistribution,
                             _pluginPrepare,
-                            pluginDistribution.Prepare).ConfigureAwait(false);
+                            pluginDistribution.Prepare,
+                            []).ConfigureAwait(false);
                     }
                 }
                 else if (buildConfigDistribution is BuildConfigProjectDistribution projectDistribution)
@@ -210,7 +251,8 @@ await WriteBuildGraphNodesAsync(
                             writer,
                             projectDistribution,
                             _projectPrepare,
-                            projectDistribution.Prepare).ConfigureAwait(false);
+                            projectDistribution.Prepare,
+                            []).ConfigureAwait(false);
                     }
                 }
                 else
diff --git a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/IDynamicBuildGraphIncludeWriter.cs b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/IDynamicBuildGraphIncludeWriter.cs
index b0d7f68a..89009f05 100644
--- a/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/IDynamicBuildGraphIncludeWriter.cs
+++ b/UET/Redpoint.Uet.BuildPipeline/BuildGraph/Dynamic/IDynamicBuildGraphIncludeWriter.cs
@@ -7,6 +7,7 @@ public interface IDynamicBuildGraphIncludeWriter
         Task WriteBuildGraphNodeInclude(
             Stream stream,
             bool filterHostToCurrentPlatformOnly,
+            object buildConfig,
             object buildConfigDistribution,
             bool executeTests,
             bool executeDeployment);
@@ -14,6 +15,7 @@ Task WriteBuildGraphNodeInclude(
         Task WriteBuildGraphMacroInclude(
             Stream stream,
             bool filterHostToCurrentPlatformOnly,
+            object buildConfig,
             object buildConfigDistribution);
     }
 }
diff --git a/UET/Redpoint.Uet.Configuration/BuildConfigSourceGenerationContext.cs b/UET/Redpoint.Uet.Configuration/BuildConfigSourceGenerationContext.cs
index 66ae10b9..49bd3b01 100644
--- a/UET/Redpoint.Uet.Configuration/BuildConfigSourceGenerationContext.cs
+++ b/UET/Redpoint.Uet.Configuration/BuildConfigSourceGenerationContext.cs
@@ -33,6 +33,7 @@ public static BuildConfigSourceGenerationContext Create(
                     new BuildConfigTestConverter<BuildConfigProjectDistribution>(serviceProvider),
                     new BuildConfigDeploymentConverter<BuildConfigPluginDistribution>(serviceProvider),
                     new BuildConfigDeploymentConverter<BuildConfigProjectDistribution>(serviceProvider),
+                    new BuildConfigTestPredefinedConverter<BuildConfigPluginDistribution>(serviceProvider),
                     new BuildConfigTargetPlatformConverter(),
                 }
             });
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamic.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamic.cs
index 295b958d..cdd0ef35 100644
--- a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamic.cs
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamic.cs
@@ -24,6 +24,7 @@ public class BuildConfigDynamic<TDistribution, TBaseClass>
         /// <summary>
         /// If set, this will be emitted as a manual job on build servers. Only applies to deployments. Defaults to false.
         /// </summary>
+        [JsonPropertyName("Manual")]
         public bool? Manual { get; set; }
 
         /// <summary>
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverter.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverter.cs
index e80b3446..f5daf323 100644
--- a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverter.cs
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverter.cs
@@ -3,7 +3,6 @@
     using Microsoft.Extensions.DependencyInjection;
     using System.Text.Json.Serialization;
     using System.Text.Json;
-    using System.Reflection.PortableExecutable;
 
     public abstract class BuildConfigDynamicConverter<TDistribution, TBaseClass> : JsonConverter<BuildConfigDynamic<TDistribution, TBaseClass>>
     {
@@ -18,141 +17,34 @@ protected BuildConfigDynamicConverter(IServiceProvider serviceProvider)
             _providers = serviceProvider.GetServices<IDynamicProvider<TDistribution, TBaseClass>>().ToArray();
         }
 
-        private IDynamicProvider<TDistribution, TBaseClass> ReadAndGetProvider(ref Utf8JsonReader readerClone, JsonSerializerOptions options)
-        {
-            if (readerClone.TokenType != JsonTokenType.StartObject)
-            {
-                throw new JsonException($"Expected {Noun} entry to be a JSON object.");
-            }
-            while (readerClone.Read())
-            {
-                if (readerClone.TokenType == JsonTokenType.EndObject)
-                {
-                    break;
-                }
-
-                if (readerClone.TokenType == JsonTokenType.PropertyName)
-                {
-                    var propertyName = readerClone.GetString();
-                    readerClone.Read();
-                    if (propertyName == "Type")
-                    {
-                        var propertyValue = readerClone.GetString();
-                        if (_providers.Length > 0)
-                        {
-                            foreach (var provider in _providers)
-                            {
-                                if (provider.Type == propertyValue)
-                                {
-                                    return provider;
-                                }
-                            }
-                            throw new JsonException($"{UpperNoun} of type '{propertyValue}' is not recognised as a {Noun} provider. Supported {Noun} types: {string.Join(", ", _providers.Select(x => $"'{x.Type}'"))}");
-                        }
-                        else
-                        {
-                            throw new JsonException($"{UpperNoun} of type '{propertyValue}' is not recognised as a {Noun} provider. There are no supported {Noun} types for this type of BuildConfig.json.");
-                        }
-                    }
-                    else
-                    {
-                        readerClone.TrySkip();
-                    }
-                }
-            }
-            if (_providers.Length > 0)
-            {
-                throw new JsonException($"{UpperNoun} entry was missing the 'Type' property. It must be set to one of the supported {Noun} types: {string.Join(", ", _providers.Select(x => $"'{x.Type}'"))}");
-            }
-            else
-            {
-                throw new JsonException($"{UpperNoun} entry was missing the 'Type' property. There are no supported {Noun} types for this type of BuildConfig.json.");
-            }
-        }
-
         public override BuildConfigDynamic<TDistribution, TBaseClass>? Read(
             ref Utf8JsonReader reader,
             Type typeToConvert,
             JsonSerializerOptions options)
         {
-            IDynamicProvider<TDistribution, TBaseClass> provider;
+            IDynamicProvider<TDistribution, TBaseClass>? provider;
             {
                 var readerClone = reader;
-                provider = ReadAndGetProvider(ref readerClone, options);
+                provider = BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.ReadAndGetProvider(
+                    _providers,
+                    Noun,
+                    UpperNoun,
+                    ref readerClone,
+                    options);
             }
 
             var result = new BuildConfigDynamic<TDistribution, TBaseClass>();
-            var gotName = false;
-            var gotType = false;
-            var gotDynamicSettings = false;
-
-            if (reader.TokenType != JsonTokenType.StartObject)
-            {
-                throw new JsonException($"Expected {Noun} entry to be a JSON object.");
-            }
-            while (reader.Read())
-            {
-                if (reader.TokenType == JsonTokenType.EndObject)
+            BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.ReadInto(
+                result,
+                provider,
+                Noun,
+                ref reader,
+                typeToConvert,
+                options,
+                (BuildConfigDynamic<TDistribution, TBaseClass> _, ref Utf8JsonReader _, string? _) =>
                 {
-                    break;
-                }
-
-                if (reader.TokenType == JsonTokenType.PropertyName)
-                {
-                    var propertyName = reader.GetString();
-                    reader.Read();
-                    switch (propertyName)
-                    {
-                        case "Name":
-                            var name = reader.GetString();
-                            if (name == null)
-                            {
-                                throw new JsonException($"Expected {Noun} entry to have a non-null name.");
-                            }
-                            result.Name = name;
-                            gotName = true;
-                            break;
-                        case "Type":
-                            var type = reader.GetString();
-                            if (type == null)
-                            {
-                                throw new JsonException($"Expected {Noun} entry to have a non-null type.");
-                            }
-                            result.Type = type;
-                            gotType = true;
-                            break;
-                        case "Manual":
-                            // @todo: Enforce that this can only appear for IDeploymentProvider.
-                            result.Manual = reader.GetBoolean();
-                            break;
-                        default:
-                            if (propertyName == provider.Type)
-                            {
-                                result.DynamicSettings = provider.DynamicSettings.Deserialize(ref reader);
-                                gotDynamicSettings = true;
-                            }
-                            else
-                            {
-                                // @todo: Make this more accurate for whether 'Manual' can be present.
-                                throw new JsonException($"Unexpected property '{propertyName}' found on {Noun} entry. Expected only the properties 'Name', 'Type', '{provider.Type}' and optionally 'Manual' (only for deployments).");
-                            }
-                            break;
-                    }
-                }
-            }
-
-            if (!gotName)
-            {
-                throw new JsonException($"Expected property 'Name' to be found on {Noun} entry.");
-            }
-            if (!gotType)
-            {
-                throw new JsonException($"Expected property 'Type' to be found on {Noun} entry.");
-            }
-            if (!gotDynamicSettings)
-            {
-                throw new JsonException($"Expected property '{provider.Type}' to be found on {Noun} entry.");
-            }
+                    return false;
+                });
 
             return result;
         }
@@ -165,22 +57,14 @@ public override void Write(
             ArgumentNullException.ThrowIfNull(writer);
             ArgumentNullException.ThrowIfNull(value);
 
-            var provider = _providers.First(x => x.Type == value.Type);
-
-            writer.WriteStartObject();
-
-            writer.WriteString("Name", value.Name);
-            writer.WriteString("Type", value.Type);
-            if (value.Manual.HasValue)
-            {
-                writer.WriteBoolean("Manual", value.Manual.Value);
-            }
-
-            writer.WritePropertyName(value.Type);
-
-            provider.DynamicSettings.Serialize(writer, value.DynamicSettings);
-
-            writer.WriteEndObject();
+            BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.WriteInto(
+                _providers,
+                value,
+                writer,
+                options,
+                (BuildConfigDynamic<TDistribution, TBaseClass> value, Utf8JsonWriter writer) =>
+                {
+                });
         }
     }
 }
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverterHelpers.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverterHelpers.cs
new file mode 100644
index 00000000..119301cd
--- /dev/null
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigDynamicConverterHelpers.cs
@@ -0,0 +1,208 @@
+namespace Redpoint.Uet.Configuration.Dynamic
+{
+    using System.Text.Json;
+
+    public static class BuildConfigConstants
+    {
+        public const string Predefined = "Predefined";
+    }
+
+    internal static class BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>
+    {
+        internal static IDynamicProvider<TDistribution, TBaseClass>? ReadAndGetProvider(
+            IDynamicProvider<TDistribution, TBaseClass>[] providers,
+            string noun,
+            string upperNoun,
+            ref Utf8JsonReader readerClone,
+            JsonSerializerOptions options)
+        {
+            if (readerClone.TokenType != JsonTokenType.StartObject)
+            {
+                throw new JsonException($"Expected {noun} entry to be a JSON object.");
+            }
+            while (readerClone.Read())
+            {
+                if (readerClone.TokenType == JsonTokenType.EndObject)
+                {
+                    break;
+                }
+
+                if (readerClone.TokenType == JsonTokenType.PropertyName)
+                {
+                    var propertyName = readerClone.GetString();
+                    readerClone.Read();
+                    if (propertyName == "Type")
+                    {
+                        var propertyValue = readerClone.GetString();
+                        if (propertyValue == BuildConfigConstants.Predefined)
+                        {
+                            return null;
+                        }
+                        if (providers.Length > 0)
+                        {
+                            foreach (var provider in providers)
+                            {
+                                if (provider.Type == propertyValue)
+                                {
+                                    return provider;
+                                }
+                            }
+                            throw new JsonException($"{upperNoun} of type '{propertyValue}' is not recognised as a {noun} provider. Supported {noun} types: {string.Join(", ", providers.Select(x => $"'{x.Type}'"))}");
+                        }
+                        else
+                        {
+                            throw new JsonException($"{upperNoun} of type '{propertyValue}' is not recognised as a {noun} provider. There are no supported {noun} types for this type of BuildConfig.json.");
+                        }
+                    }
+                    else
+                    {
+                        readerClone.TrySkip();
+                    }
+                }
+            }
+            if (providers.Length > 0)
+            {
+                throw new JsonException($"{upperNoun} entry was missing the 'Type' property. It must be set to one of the supported {noun} types: {string.Join(", ", providers.Select(x => $"'{x.Type}'"))}");
+            }
+            else
+            {
+                throw new JsonException($"{upperNoun} entry was missing the 'Type' property. There are no supported {noun} types for this type of BuildConfig.json.");
+            }
+        }
+
+        internal delegate bool ReadIntoDelegate<T>(
+            T result,
+            ref Utf8JsonReader reader,
+            string? propertyName) where T : BuildConfigDynamic<TDistribution, TBaseClass>;
+
+        internal static void ReadInto<T>(
+            T result,
+            IDynamicProvider<TDistribution, TBaseClass>? provider,
+            string noun,
+            ref Utf8JsonReader reader,
+            Type typeToConvert,
+            JsonSerializerOptions options,
+            ReadIntoDelegate<T> tryReadInto) where T : BuildConfigDynamic<TDistribution, TBaseClass>
+        {
+            var gotName = false;
+            var gotType = false;
+            var gotDynamicSettings = false;
+
+            var providerType = provider == null /* Predefined */ ? BuildConfigConstants.Predefined : provider.Type;
+
+            if (reader.TokenType != JsonTokenType.StartObject)
+            {
+                throw new JsonException($"Expected {noun} entry to be a JSON object.");
+            }
+            while (reader.Read())
+            {
+                if (reader.TokenType == JsonTokenType.EndObject)
+                {
+                    break;
+                }
+
+                if (reader.TokenType == JsonTokenType.PropertyName)
+                {
+                    var propertyName = reader.GetString();
+                    reader.Read();
+                    switch (propertyName)
+                    {
+                        case "Name":
+                            var name = reader.GetString();
+                            if (name == null)
+                            {
+                                throw new JsonException($"Expected {noun} entry to have a non-null name.");
+                            }
+                            result.Name = name;
+                            gotName = true;
+                            break;
+                        case "Type":
+                            var type = reader.GetString();
+                            if (type == null)
+                            {
+                                throw new JsonException($"Expected {noun} entry to have a non-null type.");
+                            }
+                            result.Type = type;
+                            gotType = true;
+                            break;
+                        case "Manual":
+                            // @todo: Enforce that this can only appear for IDeploymentProvider.
+                            result.Manual = reader.GetBoolean();
+                            break;
+                        default:
+                            if (propertyName == providerType)
+                            {
+                                if (provider == null)
+                                {
+                                    result.DynamicSettings = reader.GetString() ?? string.Empty;
+                                }
+                                else
+                                {
+                                    result.DynamicSettings = provider.DynamicSettings.Deserialize(ref reader);
+                                }
+                                gotDynamicSettings = true;
+                            }
+                            else if (!tryReadInto(result, ref reader, propertyName))
+                            {
+                                // @todo: Make this more accurate for whether 'Manual' can be present.
+                                throw new JsonException($"Unexpected property '{propertyName}' found on {noun} entry. Expected only the properties 'Name', 'Type', '{providerType}' and optionally 'Manual' (only for deployments).");
+                            }
+                            break;
+                    }
+                }
+            }
+
+            if (!gotName)
+            {
+                throw new JsonException($"Expected property 'Name' to be found on {noun} entry.");
+            }
+            if (!gotType)
+            {
+                throw new JsonException($"Expected property 'Type' to be found on {noun} entry.");
+            }
+            if (!gotDynamicSettings)
+            {
+                throw new JsonException($"Expected property '{providerType}' to be found on {noun} entry.");
+            }
+        }
+
+        internal delegate void WriteIntoDelegate<T>(
+            T value,
+            Utf8JsonWriter writer)
+            where T : BuildConfigDynamic<TDistribution, TBaseClass>;
+
+        internal static void WriteInto<T>(
+            IDynamicProvider<TDistribution, TBaseClass>[] providers,
+            T value,
+            Utf8JsonWriter writer,
+            JsonSerializerOptions options,
+            WriteIntoDelegate<T> writeInto)
+            where T : BuildConfigDynamic<TDistribution, TBaseClass>
+        {
+            writer.WriteStartObject();
+
+            writer.WriteString("Name", value.Name);
+            writer.WriteString("Type", value.Type);
+            if (value.Manual.HasValue)
+            {
+                writer.WriteBoolean("Manual", value.Manual.Value);
+            }
+
+            writer.WritePropertyName(value.Type);
+
+            if (value.Type == BuildConfigConstants.Predefined)
+            {
+                writer.WriteStringValue(value.DynamicSettings as string);
+            }
+            else
+            {
+                var provider = providers.First(x => x.Type == value.Type);
+                provider.DynamicSettings.Serialize(writer, value.DynamicSettings);
+            }
+
+            writeInto(value, writer);
+
+            writer.WriteEndObject();
+        }
+    }
+}
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamic.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamic.cs
new file mode 100644
index 00000000..e496ffce
--- /dev/null
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamic.cs
@@ -0,0 +1,25 @@
+namespace Redpoint.Uet.Configuration.Dynamic
+{
+    using System.Text.Json.Serialization;
+
+    /// <summary>
+    /// A dynamically driven preparation, test or deployment step that is defined outside of a distribution, where dependencies need to be specified.
+    /// </summary>
+    /// <typeparam name="TDistribution">The distribution type, which specifies whether this is for projects or plugins.</typeparam>
+    /// <typeparam name="TBaseClass">The base class for configuration, which specifies whether this is for preparation, tests or deployment.</typeparam>
+    /// <typeparam name="TDependencies">The class that describes the dependencies of this item when not referenced as part of a distribution.</typeparam>
+    public class BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies> : BuildConfigDynamic<TDistribution, TBaseClass>
+    {
+        /// <summary>
+        /// For tests defined outside of distributions, an optional shorter name that can be used with `uet test`.
+        /// </summary>
+        [JsonPropertyName("ShortName")]
+        public string? ShortName { get; set; }
+
+        /// <summary>
+        /// If set, the dependencies that will be processed/required before this entry can be used.
+        /// </summary>
+        [JsonPropertyName("Dependencies")]
+        public TDependencies? Dependencies { get; set; }
+    }
+}
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamicConverter.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamicConverter.cs
new file mode 100644
index 00000000..474be6ee
--- /dev/null
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigPredefinedDynamicConverter.cs
@@ -0,0 +1,95 @@
+namespace Redpoint.Uet.Configuration.Dynamic
+{
+    using Microsoft.Extensions.DependencyInjection;
+    using System.Text.Json.Serialization;
+    using System.Text.Json;
+
+    public abstract class BuildConfigPredefinedDynamicConverter<TDistribution, TBaseClass, TDependencies> : JsonConverter<BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies>>
+    {
+        private readonly IDynamicProvider<TDistribution, TBaseClass>[] _providers;
+
+        protected abstract string Noun { get; }
+
+        private string UpperNoun => Noun[..1].ToUpperInvariant() + Noun[1..];
+
+        protected BuildConfigPredefinedDynamicConverter(IServiceProvider serviceProvider)
+        {
+            _providers = serviceProvider.GetServices<IDynamicProvider<TDistribution, TBaseClass>>().ToArray();
+        }
+
+        public override BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies>? Read(
+            ref Utf8JsonReader reader,
+            Type typeToConvert,
+            JsonSerializerOptions options)
+        {
+            IDynamicProvider<TDistribution, TBaseClass>? provider;
+            {
+                var readerClone = reader;
+                provider = BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.ReadAndGetProvider(
+                    _providers,
+                    Noun,
+                    UpperNoun,
+                    ref readerClone,
+                    options);
+            }
+
+            var result = new BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies>();
+            BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.ReadInto(
+                result,
+                provider,
+                Noun,
+                ref reader,
+                typeToConvert,
+                options,
+                (BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies> result, ref Utf8JsonReader reader, string? propertyName) =>
+                {
+                    switch (propertyName)
+                    {
+                        case "ShortName":
+                            var shortName = reader.GetString();
+                            if (shortName == null)
+                            {
+                                throw new JsonException($"Expected {Noun} entry to have a non-null name.");
+                            }
+                            result.ShortName = shortName;
+                            return true;
+                        case "Dependencies":
+                            result.Dependencies = (TDependencies?)JsonSerializer.Deserialize(ref reader, options.GetTypeInfo(typeof(TDependencies)));
+                            return true;
+                        default:
+                            return false;
+                    }
+                });
+
+            return result;
+        }
+
+        public override void Write(
+            Utf8JsonWriter writer,
+            BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies> value,
+            JsonSerializerOptions options)
+        {
+            ArgumentNullException.ThrowIfNull(writer);
+            ArgumentNullException.ThrowIfNull(value);
+
+            BuildConfigDynamicConverterHelpers<TDistribution, TBaseClass>.WriteInto(
+                _providers,
+                value,
+                writer,
+                options,
+                (BuildConfigPredefinedDynamic<TDistribution, TBaseClass, TDependencies> value, Utf8JsonWriter writer) =>
+                {
+                    if (!string.IsNullOrWhiteSpace(value.ShortName))
+                    {
+                        writer.WriteString("ShortName", value.ShortName);
+                    }
+
+                    if (value.Dependencies != null)
+                    {
+                        writer.WritePropertyName("Dependencies");
+                        JsonSerializer.Serialize(writer, value.Dependencies, options.GetTypeInfo(typeof(TDependencies)));
+                    }
+                });
+        }
+    }
+}
diff --git a/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigTestPredefinedConverter.cs b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigTestPredefinedConverter.cs
new file mode 100644
index 00000000..f8f77caf
--- /dev/null
+++ b/UET/Redpoint.Uet.Configuration/Dynamic/BuildConfigTestPredefinedConverter.cs
@@ -0,0 +1,13 @@
+namespace Redpoint.Uet.Configuration.Dynamic
+{
+    using Redpoint.Uet.Configuration.Plugin;
+
+    internal sealed class BuildConfigTestPredefinedConverter<TDistribution> : BuildConfigPredefinedDynamicConverter<TDistribution, ITestProvider, BuildConfigPluginPredefinedTestDependencies>
+    {
+        public BuildConfigTestPredefinedConverter(IServiceProvider serviceProvider) : base(serviceProvider)
+        {
+        }
+
+        protected override string Noun => "test";
+    }
+}
diff --git a/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPlugin.cs b/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPlugin.cs
index 6522d493..b5902177 100644
--- a/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPlugin.cs
+++ b/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPlugin.cs
@@ -1,5 +1,6 @@
 namespace Redpoint.Uet.Configuration.Plugin
 {
+    using Redpoint.Uet.Configuration.Dynamic;
     using System.Diagnostics.CodeAnalysis;
     using System.Text.Json.Serialization;
 
@@ -17,6 +18,13 @@ public class BuildConfigPlugin : BuildConfig
         [JsonPropertyName("Copyright"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
         public BuildConfigPluginCopyright? Copyright { get; set; }
 
+        /// <summary>
+        /// A list of predefined tests that you can use with `uet test` or in other "Tests" sections for distributions.
+        /// </summary>
+        [JsonPropertyName("Tests"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        [SuppressMessage("Performance", "CA1819:Properties should not return arrays", Justification = "This property is used for JSON serialization.")]
+        public BuildConfigPredefinedDynamic<BuildConfigPluginDistribution, ITestProvider, BuildConfigPluginPredefinedTestDependencies>[]? Tests { get; set; }
+
         /// <summary>
         /// A list of distributions.
         /// </summary>
diff --git a/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPluginPredefinedTestDependencies.cs b/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPluginPredefinedTestDependencies.cs
new file mode 100644
index 00000000..28f57768
--- /dev/null
+++ b/UET/Redpoint.Uet.Configuration/Plugin/BuildConfigPluginPredefinedTestDependencies.cs
@@ -0,0 +1,28 @@
+namespace Redpoint.Uet.Configuration.Plugin
+{
+    using System.Text.Json.Serialization;
+
+    /// <summary>
+    /// Specifies how to how to package and build the plugin so this test can be run.
+    /// </summary>
+    public class BuildConfigPluginPredefinedTestDependencies
+    {
+        /// <summary>
+        /// A list of environment variables to set during the build.
+        /// </summary>
+        [JsonPropertyName("EnvironmentVariables"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+        public Dictionary<string, string>? EnvironmentVariables { get; set; }
+
+        /// <summary>
+        /// Specifies how to build the plugin to test it.
+        /// </summary>
+        [JsonPropertyName("Build"), JsonRequired]
+        public BuildConfigPluginBuild? Build { get; set; }
+
+        /// <summary>
+        /// Specifies how to package the plugin in order to test it.
+        /// </summary>
+        [JsonPropertyName("Package"), JsonRequired]
+        public BuildConfigPluginPackage? Package { get; set; }
+    }
+}
diff --git a/UET/uet/Commands/Build/BuildCommand.cs b/UET/uet/Commands/Build/BuildCommand.cs
index ecc75f78..7edf871d 100644
--- a/UET/uet/Commands/Build/BuildCommand.cs
+++ b/UET/uet/Commands/Build/BuildCommand.cs
@@ -476,6 +476,7 @@ public async Task<int> ExecuteAsync(InvocationContext context)
                                     buildSpec = await _buildSpecificationGenerator.BuildConfigProjectToBuildSpecAsync(
                                         engineSpec,
                                         buildGraphEnvironment,
+                                        (BuildConfigProject)distribution.BuildConfig,
                                         projectDistribution,
                                         repositoryRoot: path.DirectoryPath,
                                         executeBuild: true,
@@ -495,11 +496,12 @@ public async Task<int> ExecuteAsync(InvocationContext context)
                                     buildSpec = await _buildSpecificationGenerator.BuildConfigPluginToBuildSpecAsync(
                                         engineSpec,
                                         buildGraphEnvironment,
-                                        pluginDistribution,
                                         (BuildConfigPlugin)distribution.BuildConfig,
+                                        pluginDistribution,
                                         repositoryRoot: path.DirectoryPath,
                                         executeBuild: true,
                                         executePackage: true,
+                                        executeZip: true,
                                         executeTests: test,
                                         executeDeployment: deploy,
                                         strictIncludes: strictIncludes,
diff --git a/UET/uet/Commands/Build/DefaultBuildSpecificationGenerator.cs b/UET/uet/Commands/Build/DefaultBuildSpecificationGenerator.cs
index 8b45af95..fffcf907 100644
--- a/UET/uet/Commands/Build/DefaultBuildSpecificationGenerator.cs
+++ b/UET/uet/Commands/Build/DefaultBuildSpecificationGenerator.cs
@@ -310,6 +310,7 @@ private static string[] FilterIncompatiblePlatforms(string[] platforms, bool loc
         private async Task<(string nodeInclude, string macroInclude)> WriteDynamicBuildGraphIncludeAsync(
             BuildGraphEnvironment env,
             bool localExecutor,
+            object buildConfig,
             object distribution,
             bool executeTests,
             bool executeDeployment)
@@ -327,6 +328,7 @@ private static string[] FilterIncompatiblePlatforms(string[] platforms, bool loc
                 await _dynamicBuildGraphIncludeWriter.WriteBuildGraphNodeInclude(
                     stream,
                     localExecutor,
+                    buildConfig,
                     distribution,
                     executeTests,
                     executeDeployment).ConfigureAwait(false);
@@ -338,6 +340,7 @@ await _dynamicBuildGraphIncludeWriter.WriteBuildGraphNodeInclude(
                 await _dynamicBuildGraphIncludeWriter.WriteBuildGraphMacroInclude(
                     stream,
                     localExecutor,
+                    buildConfig,
                     distribution).ConfigureAwait(false);
             }
             await _worldPermissionApplier.GrantEveryonePermissionAsync(Path.Combine(sharedStorageAbsolutePath, macroFilename), CancellationToken.None).ConfigureAwait(false);
@@ -348,11 +351,12 @@ await _dynamicBuildGraphIncludeWriter.WriteBuildGraphMacroInclude(
         public async Task<BuildSpecification> BuildConfigPluginToBuildSpecAsync(
             BuildEngineSpecification engineSpec,
             BuildGraphEnvironment buildGraphEnvironment,
-            BuildConfigPluginDistribution distribution,
             BuildConfigPlugin pluginInfo,
+            BuildConfigPluginDistribution distribution,
             string repositoryRoot,
             bool executeBuild,
             bool executePackage,
+            bool executeZip,
             bool executeTests,
             bool executeDeployment,
             bool strictIncludes,
@@ -448,6 +452,7 @@ public async Task<BuildSpecification> BuildConfigPluginToBuildSpecAsync(
             var (scriptNodeIncludes, scriptMacroIncludes) = await WriteDynamicBuildGraphIncludeAsync(
                 buildGraphEnvironment,
                 localExecutor,
+                pluginInfo,
                 distribution,
                 executeTests,
                 executeDeployment).ConfigureAwait(false);
@@ -547,6 +552,7 @@ public async Task<BuildSpecification> BuildConfigPluginToBuildSpecAsync(
 
                     // Package options
                     { $"ExecutePackage", executePackage ? "true" : "false" },
+                    { $"ExecuteZip", executeZip ? "true" : "false" },
                     { "VersionNumber", versionInfo.versionNumber },
                     { "VersionName", versionInfo.versionName },
                     { "PackageFolder", distribution.Package?.OutputFolderName ?? "Packaged" },
@@ -569,6 +575,7 @@ public async Task<BuildSpecification> BuildConfigPluginToBuildSpecAsync(
         public async Task<BuildSpecification> BuildConfigProjectToBuildSpecAsync(
             BuildEngineSpecification engineSpec,
             BuildGraphEnvironment buildGraphEnvironment,
+            BuildConfigProject buildConfig,
             BuildConfigProjectDistribution distribution,
             string repositoryRoot,
             bool executeBuild,
@@ -588,6 +595,7 @@ public async Task<BuildSpecification> BuildConfigProjectToBuildSpecAsync(
             var (scriptNodeIncludes, scriptMacroIncludes) = await WriteDynamicBuildGraphIncludeAsync(
                 buildGraphEnvironment,
                 localExecutor,
+                buildConfig,
                 distribution,
                 executeTests,
                 executeDeployment).ConfigureAwait(false);
@@ -746,6 +754,7 @@ await File.ReadAllTextAsync(pathSpec.UPluginPath!).ConfigureAwait(false),
 
                     // Package options
                     { $"ExecutePackage", package ? "true" : "false" },
+                    { $"ExecuteZip", "true" },
                     { "VersionNumber", versionInfo.versionNumber },
                     { "VersionName", versionInfo.versionName },
                     { "PackageFolder", packageType.ToString() },
diff --git a/UET/uet/Commands/Build/IBuildSpecificationGenerator.cs b/UET/uet/Commands/Build/IBuildSpecificationGenerator.cs
index 1e6291a9..a89e401b 100644
--- a/UET/uet/Commands/Build/IBuildSpecificationGenerator.cs
+++ b/UET/uet/Commands/Build/IBuildSpecificationGenerator.cs
@@ -12,6 +12,7 @@ internal interface IBuildSpecificationGenerator
         Task<BuildSpecification> BuildConfigProjectToBuildSpecAsync(
             BuildEngineSpecification engineSpec,
             BuildGraphEnvironment buildGraphEnvironment,
+            BuildConfigProject buildConfig,
             BuildConfigProjectDistribution distribution,
             string repositoryRoot,
             bool executeBuild,
@@ -24,11 +25,12 @@ Task<BuildSpecification> BuildConfigProjectToBuildSpecAsync(
         Task<BuildSpecification> BuildConfigPluginToBuildSpecAsync(
             BuildEngineSpecification engineSpec,
             BuildGraphEnvironment buildGraphEnvironment,
+            BuildConfigPlugin buildConfig,
             BuildConfigPluginDistribution distribution,
-            BuildConfigPlugin pluginInfo,
             string repositoryRoot,
             bool executeBuild,
             bool executePackage,
+            bool executeZip,
             bool executeTests,
             bool executeDeployment,
             bool strictIncludes,
diff --git a/UET/uet/Commands/Internal/GenerateJsonSchema/DefaultJsonSchemaGenerator.cs b/UET/uet/Commands/Internal/GenerateJsonSchema/DefaultJsonSchemaGenerator.cs
index c6d8fa7b..da5eaaff 100644
--- a/UET/uet/Commands/Internal/GenerateJsonSchema/DefaultJsonSchemaGenerator.cs
+++ b/UET/uet/Commands/Internal/GenerateJsonSchema/DefaultJsonSchemaGenerator.cs
@@ -163,6 +163,7 @@ private void GenerateSchemaForObject(Utf8JsonWriter writer, JsonTypeInfo jsonTyp
                 {
                     writer.WriteStringValue(type);
                 }
+                writer.WriteStringValue(BuildConfigConstants.Predefined);
                 writer.WriteEndArray();
             }
 
@@ -278,6 +279,7 @@ private void GenerateSchemaForObject(Utf8JsonWriter writer, JsonTypeInfo jsonTyp
                     writer.WriteEndArray();
                     writer.WritePropertyName("properties");
                     writer.WriteStartObject();
+                    writer.WriteBoolean(BuildConfigConstants.Predefined, false);
                     foreach (var otherProvider in providers.Where(x => ((IDynamicProviderRegistration)x).Type != ((IDynamicProviderRegistration)type).Type))
                     {
                         writer.WriteBoolean(((IDynamicProviderRegistration)otherProvider).Type, false);
@@ -307,6 +309,39 @@ private void GenerateSchemaForObject(Utf8JsonWriter writer, JsonTypeInfo jsonTyp
                     writer.WriteEndObject();
                     writer.WriteEndObject();
                 }
+                {
+                    writer.WriteStartObject();
+                    writer.WritePropertyName("if");
+                    writer.WriteStartObject();
+                    writer.WritePropertyName("properties");
+                    writer.WriteStartObject();
+                    writer.WritePropertyName("Type");
+                    writer.WriteStartObject();
+                    writer.WriteString("const", BuildConfigConstants.Predefined);
+                    writer.WriteEndObject();
+                    writer.WriteEndObject();
+                    writer.WriteEndObject();
+                    writer.WritePropertyName("then");
+                    writer.WriteStartObject();
+                    writer.WritePropertyName("required");
+                    writer.WriteStartArray();
+                    writer.WriteStringValue(BuildConfigConstants.Predefined);
+                    writer.WriteEndArray();
+                    writer.WritePropertyName("properties");
+                    writer.WriteStartObject();
+                    foreach (var otherProvider in providers)
+                    {
+                        writer.WriteBoolean(((IDynamicProviderRegistration)otherProvider).Type, false);
+                    }
+                    writer.WritePropertyName(BuildConfigConstants.Predefined);
+                    writer.WriteStartObject();
+                    writer.WriteString("type", "string");
+                    writer.WriteString("description", "The predefined name defined earlier in configuration.");
+                    writer.WriteEndObject();
+                    writer.WriteEndObject();
+                    writer.WriteEndObject();
+                    writer.WriteEndObject();
+                }
                 writer.WriteEndArray();
             }
             else
diff --git a/UET/uet/Commands/Test/TestCommand.cs b/UET/uet/Commands/Test/TestCommand.cs
index 87bdd3df..d54cfce8 100644
--- a/UET/uet/Commands/Test/TestCommand.cs
+++ b/UET/uet/Commands/Test/TestCommand.cs
@@ -16,6 +16,7 @@
     using System.CommandLine;
     using System.CommandLine.Invocation;
     using System.Threading.Tasks;
+    using UET.BuildConfig;
     using UET.Commands.Build;
     using UET.Commands.EngineSpec;
     using static Crayon.Output;
@@ -28,13 +29,14 @@ internal sealed class Options
             public Option<PathSpec> Path;
 
             public Option<string> Prefix;
+            public Option<string> Name;
 
             public Options()
             {
                 Path = new Option<PathSpec>(
                     "--path",
-                    description: "The directory path that contains a .uproject file or a .uplugin file. BuildConfig.json files will be ignored for this command; if you need to run tests for a BuildConfig.json, use the 'build' command instead with the --test option.",
-                    parseArgument: result => PathSpec.ParsePathSpec(result, true),
+                    description: "The directory path that contains a .uproject file, .uplugin file or BuildConfig.json file.",
+                    parseArgument: result => PathSpec.ParsePathSpec(result, false),
                     isDefault: true);
                 Path.AddAlias("-p");
                 Path.Arity = ArgumentArity.ExactlyOne;
@@ -50,12 +52,34 @@ public Options()
                 Prefix = new Option<string>(
                     "--prefix",
                     description: "The prefix for tests to run. If not set, defaults to either '<project name>.' or '<plugin name>.'.");
+
+                Name = new Option<string>(
+                    "--name",
+                    description: "When testing against a BuildConfig.json file, the name or short name of the predefined test to run.");
+                Name.AddAlias("-n");
             }
         }
 
         public static Command CreateTestCommand()
         {
-            var command = new Command("test", "Run automation tests in the editor for a project or plugin.");
+            var command = new Command("test", "Run tests in the editor for a project or plugin.")
+            {
+                FullDescription = """
+                This command runs tests for either an Unreal Engine project or plugin. The behaviour of this command depends on the arguments that you've passed to it and the current directory.
+
+                -------------
+
+                If this command is run in a directory with a .uproject or .uplugin file (but no BuildConfig.json file), this command runs automation tests against the Unreal Engine project or plugin using the editor binary for the current platform.
+
+                If you don't specify --prefix, all automation tests are run. This is usually a lot more tests than you want, so you should specify --prefix in most cases.
+
+                -------------
+
+                If this command is run in a directory with a BuildConfig.json file, where that BuildConfig.json file describes a plugin, you must specify --name.
+
+                The specified --name must match a test predefined in the 'Tests' section (outside of 'Distributions').
+                """
+            };
             command.AddServicedOptionsHandler<TestCommandInstance, Options>(
                 services =>
                 {
@@ -71,26 +95,30 @@ private sealed class TestCommandInstance : ICommandInstance
             private readonly IBuildSpecificationGenerator _buildSpecificationGenerator;
             private readonly LocalBuildExecutorFactory _localBuildExecutorFactory;
             private readonly IStringUtilities _stringUtilities;
+            private readonly IServiceProvider _serviceProvider;
 
             public TestCommandInstance(
                 ILogger<TestCommandInstance> logger,
                 Options options,
                 IBuildSpecificationGenerator buildSpecificationGenerator,
                 LocalBuildExecutorFactory localBuildExecutorFactory,
-                IStringUtilities stringUtilities)
+                IStringUtilities stringUtilities,
+                IServiceProvider serviceProvider)
             {
                 _logger = logger;
                 _options = options;
                 _buildSpecificationGenerator = buildSpecificationGenerator;
                 _localBuildExecutorFactory = localBuildExecutorFactory;
                 _stringUtilities = stringUtilities;
+                _serviceProvider = serviceProvider;
             }
 
             public async Task<int> ExecuteAsync(InvocationContext context)
             {
                 var engine = context.ParseResult.GetValueForOption(_options.Engine)!;
                 var path = context.ParseResult.GetValueForOption(_options.Path)!;
-                var prefix = context.ParseResult.GetValueForOption(_options.Prefix)!;
+                var prefix = context.ParseResult.GetValueForOption(_options.Prefix);
+                var name = context.ParseResult.GetValueForOption(_options.Name);
 
                 // @todo: Should we surface these options?
                 var windowsSharedStoragePath = Path.Combine(path.DirectoryPath, ".uet", "shared-storage");
@@ -113,6 +141,7 @@ public async Task<int> ExecuteAsync(InvocationContext context)
                 _logger.LogInformation($"--engine:                      {engine}");
                 _logger.LogInformation($"--path:                        {path}");
                 _logger.LogInformation($"--prefix:                      {prefix}");
+                _logger.LogInformation($"--name:                        {name}");
 
                 BuildEngineSpecification engineSpec;
                 switch (engine.Type)
@@ -166,127 +195,214 @@ public async Task<int> ExecuteAsync(InvocationContext context)
                     switch (path!.Type)
                     {
                         case PathSpecType.UProject:
-                            // Use heuristics to guess the targets for this build.
-                            string editorTarget;
-                            string gameTarget;
-                            if (Directory.Exists(Path.Combine(path.DirectoryPath, "Source")))
-                            {
-                                var files = Directory.GetFiles(Path.Combine(path.DirectoryPath, "Source"), "*.Target.cs");
-                                editorTarget = files.Where(x => x.EndsWith("Editor.Target.cs", StringComparison.Ordinal)).Select(x => Path.GetFileName(x)).First();
-                                editorTarget = editorTarget[..editorTarget.LastIndexOf(".Target.cs", StringComparison.Ordinal)];
-                                gameTarget = editorTarget[..editorTarget.LastIndexOf("Editor", StringComparison.Ordinal)];
-                            }
-                            else
                             {
-                                editorTarget = "UnrealEditor";
-                                gameTarget = "UnrealGame";
-                            }
+                                // Use heuristics to guess the targets for this build.
+                                string editorTarget;
+                                string gameTarget;
+                                if (Directory.Exists(Path.Combine(path.DirectoryPath, "Source")))
+                                {
+                                    var files = Directory.GetFiles(Path.Combine(path.DirectoryPath, "Source"), "*.Target.cs");
+                                    editorTarget = files.Where(x => x.EndsWith("Editor.Target.cs", StringComparison.Ordinal)).Select(x => Path.GetFileName(x)).First();
+                                    editorTarget = editorTarget[..editorTarget.LastIndexOf(".Target.cs", StringComparison.Ordinal)];
+                                    gameTarget = editorTarget[..editorTarget.LastIndexOf("Editor", StringComparison.Ordinal)];
+                                }
+                                else
+                                {
+                                    editorTarget = "UnrealEditor";
+                                    gameTarget = "UnrealGame";
+                                }
 
-                            var buildConfigProject = new BuildConfigProject
-                            {
-                                Type = Redpoint.Uet.Configuration.BuildConfigType.Plugin,
-                                Distributions = new List<BuildConfigProjectDistribution>
+                                var buildConfigProject = new BuildConfigProject
                                 {
-                                    new BuildConfigProjectDistribution
+                                    Type = BuildConfigType.Plugin,
+                                    Distributions = new List<BuildConfigProjectDistribution>
                                     {
-                                        Name = "Test",
-                                        ProjectName = Path.GetFileNameWithoutExtension(path.UProjectPath)!,
-                                        Build = new BuildConfigProjectBuild
+                                        new BuildConfigProjectDistribution
                                         {
-                                            Editor = new BuildConfigProjectBuildEditor
+                                            Name = "Test",
+                                            ProjectName = Path.GetFileNameWithoutExtension(path.UProjectPath)!,
+                                            Build = new BuildConfigProjectBuild
                                             {
-                                                Target = editorTarget,
-                                            }
-                                        },
-                                        Tests = new[]
-                                        {
-                                            new BuildConfigDynamic<BuildConfigProjectDistribution, ITestProvider>
+                                                Editor = new BuildConfigProjectBuildEditor
+                                                {
+                                                    Target = editorTarget,
+                                                }
+                                            },
+                                            Tests = new[]
                                             {
-                                                Type = "Automation",
-                                                Name = "Test",
-                                                Manual = false,
-                                                DynamicSettings = new BuildConfigProjectTestAutomation
+                                                new BuildConfigDynamic<BuildConfigProjectDistribution, ITestProvider>
                                                 {
-                                                    TestPrefix = prefix,
-                                                    TargetName = editorTarget,
+                                                    Type = "Automation",
+                                                    Name = "Test",
+                                                    Manual = false,
+                                                    DynamicSettings = new BuildConfigProjectTestAutomation
+                                                    {
+                                                        TestPrefix = prefix ?? string.Empty,
+                                                        TargetName = editorTarget,
+                                                    }
                                                 }
                                             }
                                         }
                                     }
-                                }
-                            };
-                            buildSpec = await _buildSpecificationGenerator.BuildConfigProjectToBuildSpecAsync(
-                                engineSpec,
-                                buildGraphEnvironment,
-                                buildConfigProject.Distributions[0],
-                                repositoryRoot: path.DirectoryPath,
-                                executeBuild: true,
-                                executeTests: true,
-                                executeDeployment: false,
-                                strictIncludes: false,
-                                localExecutor: true,
-                                alternateStagingDirectory: null).ConfigureAwait(false);
-                            break;
+                                };
+                                buildSpec = await _buildSpecificationGenerator.BuildConfigProjectToBuildSpecAsync(
+                                    engineSpec,
+                                    buildGraphEnvironment,
+                                    buildConfigProject,
+                                    buildConfigProject.Distributions[0],
+                                    repositoryRoot: path.DirectoryPath,
+                                    executeBuild: true,
+                                    executeTests: true,
+                                    executeDeployment: false,
+                                    strictIncludes: false,
+                                    localExecutor: true,
+                                    alternateStagingDirectory: null).ConfigureAwait(false);
+                                break;
+                            }
                         case PathSpecType.UPlugin:
-                            var buildConfigPlugin = new BuildConfigPlugin
                             {
-                                PluginName = Path.GetFileNameWithoutExtension(path.UPluginPath!),
-                                Type = Redpoint.Uet.Configuration.BuildConfigType.Plugin,
-                                Distributions = new List<BuildConfigPluginDistribution>
+                                var buildConfigPlugin = new BuildConfigPlugin
                                 {
-                                    new BuildConfigPluginDistribution
+                                    PluginName = Path.GetFileNameWithoutExtension(path.UPluginPath!),
+                                    Type = BuildConfigType.Plugin,
+                                    Distributions = new List<BuildConfigPluginDistribution>
                                     {
-                                        Name = "Test",
-                                        Build = new BuildConfigPluginBuild
-                                        {
-                                            Editor = new BuildConfigPluginBuildEditor
-                                            {
-                                                Platforms = new[]
-                                                {
-                                                    OperatingSystem.IsWindows()
-                                                        ? BuildConfigPluginBuildEditorPlatform.Win64
-                                                        : BuildConfigPluginBuildEditorPlatform.Mac
-                                                },
-                                            }
-                                        },
-                                        Tests = new[]
+                                        new BuildConfigPluginDistribution
                                         {
-                                            new BuildConfigDynamic<BuildConfigPluginDistribution, ITestProvider>
+                                            Name = "Test",
+                                            Build = new BuildConfigPluginBuild
                                             {
-                                                Type = "Automation",
-                                                Name = "Test",
-                                                Manual = false,
-                                                DynamicSettings = new BuildConfigPluginTestAutomation
+                                                Editor = new BuildConfigPluginBuildEditor
                                                 {
                                                     Platforms = new[]
                                                     {
                                                         OperatingSystem.IsWindows()
-                                                            ? BuildConfigHostPlatform.Win64
-                                                            : BuildConfigHostPlatform.Mac
+                                                            ? BuildConfigPluginBuildEditorPlatform.Win64
+                                                            : BuildConfigPluginBuildEditorPlatform.Mac
                                                     },
-                                                    TestPrefix = prefix,
+                                                }
+                                            },
+                                            Tests = new[]
+                                            {
+                                                new BuildConfigDynamic<BuildConfigPluginDistribution, ITestProvider>
+                                                {
+                                                    Type = "Automation",
+                                                    Name = "Test",
+                                                    Manual = false,
+                                                    DynamicSettings = new BuildConfigPluginTestAutomation
+                                                    {
+                                                        Platforms = new[]
+                                                        {
+                                                            OperatingSystem.IsWindows()
+                                                                ? BuildConfigHostPlatform.Win64
+                                                                : BuildConfigHostPlatform.Mac
+                                                        },
+                                                        TestPrefix = prefix ?? string.Empty,
+                                                    }
                                                 }
                                             }
                                         }
                                     }
+                                };
+                                buildSpec = await _buildSpecificationGenerator.BuildConfigPluginToBuildSpecAsync(
+                                    engineSpec,
+                                    buildGraphEnvironment,
+                                    buildConfigPlugin,
+                                    buildConfigPlugin.Distributions[0],
+                                    repositoryRoot: path.DirectoryPath,
+                                    executeBuild: true,
+                                    executePackage: true,
+                                    executeZip: true,
+                                    executeTests: true,
+                                    executeDeployment: false,
+                                    strictIncludes: false,
+                                    localExecutor: true,
+                                    isPluginRooted: true,
+                                    commandlinePluginVersionName: null,
+                                    commandlinePluginVersionNumber: null).ConfigureAwait(false);
+                                break;
+                            }
+                        case PathSpecType.BuildConfig:
+                            {
+                                var loadResult = BuildConfigLoader.TryLoad(
+                                    _serviceProvider,
+                                    Path.Combine(path.DirectoryPath, "BuildConfig.json"));
+                                if (!loadResult.Success)
+                                {
+                                    _logger.LogError(string.Join("\n", loadResult.ErrorList));
+                                    return 1;
+                                }
+                                if (loadResult.BuildConfig is BuildConfigPlugin buildConfigPlugin &&
+                                    buildConfigPlugin.Tests != null)
+                                {
+                                    var test = buildConfigPlugin.Tests.FirstOrDefault(
+                                        t => t.Name.Equals(name, StringComparison.OrdinalIgnoreCase)
+                                            || (t.ShortName != null && t.ShortName.Equals(name, StringComparison.OrdinalIgnoreCase)));
+                                    if (test == null)
+                                    {
+                                        _logger.LogError($"This BuildConfig.json does not have a predefined test with a name or short name that matches '{name}'.");
+                                        return 1;
+                                    }
+
+                                    // Now we make a synthetic BuildConfig.json file with a synthetic distribution that
+                                    // specifies the dependencies correctly.
+                                    var syntheticBuildConfigPlugin = new BuildConfigPlugin
+                                    {
+                                        PluginName = buildConfigPlugin.PluginName,
+                                        Type = BuildConfigType.Plugin,
+                                        Copyright = buildConfigPlugin.Copyright,
+                                        Distributions = new List<BuildConfigPluginDistribution>
+                                        {
+                                            new BuildConfigPluginDistribution
+                                            {
+                                                Name = "Test",
+                                                EnvironmentVariables = test.Dependencies?.EnvironmentVariables,
+                                                Package = test.Dependencies?.Package ?? new BuildConfigPluginPackage
+                                                {
+                                                },
+                                                Build = test.Dependencies?.Build ?? new BuildConfigPluginBuild
+                                                {
+                                                    Editor = new BuildConfigPluginBuildEditor
+                                                    {
+                                                        Platforms = new[]
+                                                        {
+                                                            OperatingSystem.IsWindows()
+                                                                ? BuildConfigPluginBuildEditorPlatform.Win64
+                                                                : BuildConfigPluginBuildEditorPlatform.Mac
+                                                        },
+                                                    }
+                                                },
+                                                Tests =
+                                                [
+                                                    test
+                                                ]
+                                            }
+                                        }
+                                    };
+                                    buildSpec = await _buildSpecificationGenerator.BuildConfigPluginToBuildSpecAsync(
+                                        engineSpec,
+                                        buildGraphEnvironment,
+                                        syntheticBuildConfigPlugin,
+                                        syntheticBuildConfigPlugin.Distributions[0],
+                                        repositoryRoot: path.DirectoryPath,
+                                        executeBuild: true,
+                                        executePackage: true,
+                                        executeZip: false,
+                                        executeTests: true,
+                                        executeDeployment: false,
+                                        strictIncludes: false,
+                                        localExecutor: true,
+                                        isPluginRooted: false,
+                                        commandlinePluginVersionName: null,
+                                        commandlinePluginVersionNumber: null).ConfigureAwait(false);
                                 }
-                            };
-                            buildSpec = await _buildSpecificationGenerator.BuildConfigPluginToBuildSpecAsync(
-                                engineSpec,
-                                buildGraphEnvironment,
-                                buildConfigPlugin.Distributions[0],
-                                buildConfigPlugin,
-                                repositoryRoot: path.DirectoryPath,
-                                executeBuild: true,
-                                executePackage: true,
-                                executeTests: true,
-                                executeDeployment: false,
-                                strictIncludes: false,
-                                localExecutor: true,
-                                isPluginRooted: true,
-                                commandlinePluginVersionName: null,
-                                commandlinePluginVersionNumber: null).ConfigureAwait(false);
-                            break;
+                                else
+                                {
+                                    _logger.LogError("This type of BuildConfig.json file is not supported.");
+                                    return 1;
+                                }
+                                break;
+                            }
                         default:
                             throw new NotSupportedException();
                     }