From 98e0700c0e64218819d63493315c1ab2a8da48e3 Mon Sep 17 00:00:00 2001 From: Ryan Campbell <89273172+bigtallcampbell@users.noreply.github.com> Date: Tue, 9 Jul 2024 15:57:30 -0500 Subject: [PATCH 1/2] update proto path (#6) --- src_pluginBase/pluginBase.csproj | 4 ++-- test/integrationTests/integrationTests.csproj | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src_pluginBase/pluginBase.csproj b/src_pluginBase/pluginBase.csproj index 717a444..e3d10ac 100644 --- a/src_pluginBase/pluginBase.csproj +++ b/src_pluginBase/pluginBase.csproj @@ -19,7 +19,7 @@ - - + + diff --git a/test/integrationTests/integrationTests.csproj b/test/integrationTests/integrationTests.csproj index 22d908c..e5c71f5 100644 --- a/test/integrationTests/integrationTests.csproj +++ b/test/integrationTests/integrationTests.csproj @@ -24,6 +24,6 @@ - + From 9fbcf33a1f081630f7b0a9fa8949631a85ebe22c Mon Sep 17 00:00:00 2001 From: Ryan Campbell <89273172+bigtallcampbell@users.noreply.github.com> Date: Fri, 19 Jul 2024 17:12:35 -0500 Subject: [PATCH 2/2] Update to handle SMB enable/disable and move templating to helm (#7) * updating paths * adding spacefx chart utils * adding postStart * adding helm template util * adding resourceLimits to pods * adding appSettings and appSettingsMount * Adding appsettings * updating for volume templates * adding persistentvolumes and persistentvolumeclaims * updating for persistentvolumeclaim * updating for integration test * updating to include service account * updating to reuse persistentvolumeclaim when available * updating devcontainer to prod feature --- .devcontainer/devcontainer.json | 21 +- .devcontainer/postStart.sh | 1 + .vscode/tasks.json | 3 +- src/MessageHandlers/MessageHandler.cs | 34 ++ src/Models/AppConfig.cs | 26 +- src/Models/KubernetesObjects.cs | 73 +++ src/Program.cs | 48 +- src/Services/DeployRequestProcessor.cs | 4 +- src/Services/ScheduleProcessor.cs | 6 + src/Utils/K8sClient.cs | 465 +++--------------- src/Utils/TemplateUtil.cs | 453 +++++++++++++++++ src/appsettings.Development.json | 4 +- src/appsettings.IntegrationTest.json | 2 +- src/platform-deployment.csproj | 2 +- test/debugClient/Services/MessageSender.cs | 9 +- test/debugClient/debugClient.csproj | 2 +- test/debugClient/outbox/schedule/busybox.json | 11 + .../schedule}/busybox.yaml | 2 +- test/debugClient/sampleSchedules/busybox.json | 21 - .../integrationTestPlugin.csproj | 2 +- test/integrationTests/Tests/DeploymentTest.cs | 8 +- test/sampleSchedules/ServiceAppBuild.docker | 4 +- test/sampleSchedules/busybox.json | 18 +- test/sampleSchedules/busybox.yaml | 2 +- 24 files changed, 729 insertions(+), 492 deletions(-) create mode 100755 .devcontainer/postStart.sh create mode 100644 src/MessageHandlers/MessageHandler.cs create mode 100644 src/Models/KubernetesObjects.cs create mode 100644 src/Utils/TemplateUtil.cs create mode 100644 test/debugClient/outbox/schedule/busybox.json rename test/debugClient/{sampleSchedules => outbox/schedule}/busybox.yaml (91%) delete mode 100644 test/debugClient/sampleSchedules/busybox.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 188e4eb..49429c0 100755 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,8 +6,8 @@ "runArgs": [ "--name=platform-deployment" ], - "workspaceFolder": "/workspaces/platform-deployment", - "workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/platform-deployment,type=bind,consistency=cached", + "workspaceFolder": "/workspace/platform-deployment", + "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/platform-deployment,type=bind,consistency=cached", "hostRequirements": { "cpus": 8, "memory": "8gb" @@ -17,7 +17,8 @@ "app_name": "platform-deployment", "app_type": "sdk-service", "addl_debug_shim_suffixes": "client", - "debug_shim_post_yaml_file": "/workspaces/platform-deployment/.vscode/debugShim-svcAcct-clusterAdmin.yaml" + "debug_shim_post_yaml_file": "/workspace/platform-deployment/.vscode/debugShim-svcAcct-clusterAdmin.yaml", + "smb_enabled_in_cluster": "true" } }, "customizations": { @@ -112,10 +113,22 @@ "contents": "read", "packages": "read" } + }, + "microsoft/azure-orbital-space-sdk-data-generators": { + "permissions": { + "contents": "read", + "packages": "read" + } } } } }, + "remoteEnv": { + "KUBECONFIG": "/workspace/platform-deployment/.git/spacefx-dev/k3s.devcontainer.yaml" + }, + "containerEnv": { + "KUBECONFIG": "/workspace/platform-deployment/.git/spacefx-dev/k3s.devcontainer.yaml" + }, "remoteUser": "root", - "postStartCommand": "regctl image export ghcr.io/dapr/samples/pubsub-csharp-subscriber:1.9.0 --name pubsub-csharp-subscriber > /workspaces/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar" + "postStartCommand": "/workspace/platform-deployment/.devcontainer/postStart.sh" } \ No newline at end of file diff --git a/.devcontainer/postStart.sh b/.devcontainer/postStart.sh new file mode 100755 index 0000000..374c4cf --- /dev/null +++ b/.devcontainer/postStart.sh @@ -0,0 +1 @@ +regctl image export ghcr.io/dapr/samples/pubsub-csharp-subscriber:1.9.0 --name pubsub-csharp-subscriber > /workspace/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar \ No newline at end of file diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 9b38904..c6a3d56 100755 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -19,7 +19,8 @@ "args": [ "/spacefx-dev/debugShim-deploy.sh", "--debug_shim", - "${DEBUG_SHIM_HOST}" + "${DEBUG_SHIM_HOST}", + "--disable_plugin_configs" ], "presentation": { "echo": true, diff --git a/src/MessageHandlers/MessageHandler.cs b/src/MessageHandlers/MessageHandler.cs new file mode 100644 index 0000000..c694289 --- /dev/null +++ b/src/MessageHandlers/MessageHandler.cs @@ -0,0 +1,34 @@ +namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment; + +public partial class MessageHandler : Microsoft.Azure.SpaceFx.Core.IMessageHandler where T : notnull { + private readonly ILogger> _logger; + public static EventHandler? MessageReceivedEvent; + private readonly Microsoft.Azure.SpaceFx.Core.Services.PluginLoader _pluginLoader; + private readonly IServiceProvider _serviceProvider; + private readonly Core.Client _client; + private readonly Models.APP_CONFIG _appConfig; + private readonly PluginDelegates _pluginDelegates; + + public MessageHandler(ILogger> logger, PluginDelegates pluginDelegates, Microsoft.Azure.SpaceFx.Core.Services.PluginLoader pluginLoader, IServiceProvider serviceProvider, Core.Client client) { + _logger = logger; + _pluginDelegates = pluginDelegates; + _pluginLoader = pluginLoader; + _serviceProvider = serviceProvider; + _client = client; + + _appConfig = new Models.APP_CONFIG(); + } + + public void MessageReceived(T message, MessageFormats.Common.DirectToApp fullMessage) => Task.Run(() => { + using (var scope = _serviceProvider.CreateScope()) { + + if (message == null || EqualityComparer.Default.Equals(message, default)) { + _logger.LogInformation("Received empty message '{messageType}' from '{appId}'. Discarding message.", typeof(T).Name, fullMessage.SourceAppId); + return; + } + + // This function is just a catch all for any messages that come in. They are not processed and no plugins are triggered for security reasons. + // We're catching all messages here to reduce the log warnings for OOTB messages that are flowing + } + }); +} \ No newline at end of file diff --git a/src/Models/AppConfig.cs b/src/Models/AppConfig.cs index a56ed36..1823dc4 100644 --- a/src/Models/AppConfig.cs +++ b/src/Models/AppConfig.cs @@ -1,7 +1,5 @@ -using YamlDotNet.Serialization; - namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment; -public static class Models { +public static partial class Models { public class APP_CONFIG : Core.APP_CONFIG { [Flags] [JsonConverter(typeof(JsonStringEnumConverter))] @@ -48,7 +46,6 @@ public PLUG_IN() { public string CONTAINER_REGISTRY { get; set; } public string CONTAINER_REGISTRY_INTERNAL { get; set; } public string SCHEDULE_IMPORT_DIRECTORY { get; set; } - public string DAPR_ANNOTATIONS { get; set; } public string DEFAULT_LIMIT_MEMORY { get; set; } public string DEFAULT_LIMIT_CPU { get; set; } public string DEFAULT_REQUEST_MEMORY { get; set; } @@ -56,14 +53,7 @@ public PLUG_IN() { public string FILESERVER_APP_CRED_NAME { get; set; } public string FILESERVER_CRED_NAME { get; set; } public string FILESERVER_CRED_NAMESPACE { get; set; } - public string FILESERVER_PERSISTENT_VOLUMES { get; set; } - public string FILESERVER_PERSISTENT_VOLUMECLAIMS { get; set; } - public string FILESERVER_CLIENT_VOLUME_MOUNTS { get; set; } - public string FILESERVER_CLIENT_VOLUMES { get; set; } - public string PAYLOAD_APP_ANNOTATIONS { get; set; } - public string PAYLOAD_APP_CONFIG { get; set; } - public string PAYLOAD_APP_LABELS { get; set; } - public string PAYLOAD_APP_ENVIRONMENTVARIABLES { get; set; } + public bool FILESERVER_SMB_ENABLED { get; set; } public TimeSpan DEFAULT_MAX_DURATION { get; set; } public APP_CONFIG() : base() { @@ -90,19 +80,9 @@ public APP_CONFIG() : base() { FILESERVER_APP_CRED_NAME = Core.GetConfigSetting("fileserverappcredname").Result; FILESERVER_CRED_NAME = Core.GetConfigSetting("fileservercredname").Result; FILESERVER_CRED_NAMESPACE = Core.GetConfigSetting("fileservercrednamespace").Result; + FILESERVER_SMB_ENABLED = bool.Parse(Core.GetConfigSetting("fileserversmb").Result); - FILESERVER_PERSISTENT_VOLUMES = Core.GetConfigSetting("fileserverclientpv").Result; - FILESERVER_PERSISTENT_VOLUMECLAIMS = Core.GetConfigSetting("fileserverclientpvc").Result; - - FILESERVER_CLIENT_VOLUMES = Core.GetConfigSetting("fileServerclientvolumes").Result; - FILESERVER_CLIENT_VOLUME_MOUNTS = Core.GetConfigSetting("fileServerclientvolumemounts").Result; - - DAPR_ANNOTATIONS = Core.GetConfigSetting("daprannotations").Result; - PAYLOAD_APP_ANNOTATIONS = Core.GetConfigSetting("payloadappannotations").Result; - PAYLOAD_APP_CONFIG = Core.GetConfigSetting("payloadappconfig").Result; - PAYLOAD_APP_LABELS = Core.GetConfigSetting("payloadapplabels").Result; - PAYLOAD_APP_ENVIRONMENTVARIABLES = Core.GetConfigSetting("payloadappenvironmentvariables").Result; if (Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") == "Development" || Environment.GetEnvironmentVariable("DOTNET_ENVIRONMENT") == "IntegrationTest") { ENABLE_YAML_DEBUG = true; diff --git a/src/Models/KubernetesObjects.cs b/src/Models/KubernetesObjects.cs new file mode 100644 index 0000000..1f789c9 --- /dev/null +++ b/src/Models/KubernetesObjects.cs @@ -0,0 +1,73 @@ + +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment; +public static partial class Models { + public class KubernetesObjects { + public class ResourceDefinition { + public ResourceSection Resources { get; set; } + + public ResourceDefinition() { + Resources = new ResourceSection(); + } + } + + public class ResourceSection { + public ResourceDetails Limits { get; set; } + public ResourceDetails Requests { get; set; } + + public ResourceSection() { + Limits = new ResourceDetails(); + Requests = new ResourceDetails(); + } + } + + public class ResourceDetails { + public string Cpu { get; set; } + public string Memory { get; set; } + + public ResourceDetails() { + Cpu = ""; + Memory = ""; + } + } + + public class VolumeMountRoot { + public List VolumeMounts { get; set; } + public VolumeMountRoot() { + VolumeMounts = new List(); + } + } + + public class VolumeRoot { + public List Volumes { get; set; } + public VolumeRoot() { + Volumes = new List(); + } + + public class ConfigMapVolumeSource { + public string Name { get; set; } + public ConfigMapVolumeSource() { + Name = ""; + } + } + + public class SecretVolumeSource { + public string SecretName { get; set; } + public SecretVolumeSource() { + SecretName = ""; + } + } + + public class PersistentVolumeClaimVolumeSource { + public string ClaimName { get; set; } + public PersistentVolumeClaimVolumeSource() { + ClaimName = ""; + } + } + } + } + +} diff --git a/src/Program.cs b/src/Program.cs index 9eac623..bfa01ac 100644 --- a/src/Program.cs +++ b/src/Program.cs @@ -1,35 +1,12 @@ namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment; public class Program { - private static void Test() { - MessageFormats.PlatformServices.Deployment.DeployRequest _request = new MessageFormats.PlatformServices.Deployment.DeployRequest(); - _request.StartTime = Google.Protobuf.WellKnownTypes.Timestamp.FromDateTime(DateTime.UtcNow); - _request.MaxDuration = Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(new TimeSpan(0, 0, 0, 0, 0)); - _request.AppContextFile = new MessageFormats.PlatformServices.Deployment.DeployRequest.Types.AppContextFile() { - FileName = "test1.jpg", - Required = false - }; - - _request.GpuRequirement = MessageFormats.PlatformServices.Deployment.DeployRequest.Types.GpuOptions.Nvidia; - _request.DeployAction = MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Create; - - - - - Google.Protobuf.JsonFormatter formatter = new Google.Protobuf.JsonFormatter(Google.Protobuf.JsonFormatter.Settings.Default); - string jsonString = formatter.Format(_request); - - Console.WriteLine(jsonString); - Console.WriteLine("Woohoo!"); - - } public static void Main(string[] args) { - //Test(); var builder = WebApplication.CreateBuilder(args); - builder.Configuration.AddJsonFile("/workspaces/platform-deployment-config/appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile("/workspaces/platform-deployment/src/appsettings.json", optional: true, reloadOnChange: true) - .AddJsonFile("/workspaces/platform-deployment/src/appsettings.{env:DOTNET_ENVIRONMENT}.json", optional: true, reloadOnChange: true).Build(); + builder.Configuration.AddJsonFile("/workspace/platform-deployment-config/appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile("/workspace/platform-deployment/src/appsettings.json", optional: true, reloadOnChange: true) + .AddJsonFile("/workspace/platform-deployment/src/appsettings.{env:DOTNET_ENVIRONMENT}.json", optional: true, reloadOnChange: true).Build(); builder.WebHost.ConfigureKestrel(options => options.ListenAnyIP(50051, o => o.Protocols = HttpProtocols.Http2)) .ConfigureServices((services) => { @@ -42,8 +19,13 @@ public static void Main(string[] args) { services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(p => p.GetRequiredService()); + services.AddSingleton, MessageHandler>(); + services.AddSingleton, MessageHandler>(); + + }).ConfigureLogging((logging) => { logging.AddProvider(new Microsoft.Extensions.Logging.SpaceFX.Logger.HostSvcLoggerProvider()); logging.AddConsole(); @@ -58,6 +40,20 @@ public static void Main(string[] args) { await context.Response.WriteAsync("Communication with gRPC endpoints must be made through a gRPC client. To learn how to create a client, visit: https://go.microsoft.com/fwlink/?linkid=2086909"); }); }); + + // Add a middleware to catch exceptions and stop the host gracefully + app.Use(async (context, next) => { + try { + await next.Invoke(); + } catch (Exception ex) { + Console.Error.WriteLine($"Exception caught in middleware: {ex.Message}"); + + // Stop the host gracefully so it triggers the pod to error + var lifetime = context.RequestServices.GetService(); + lifetime?.StopApplication(); + } + }); + app.Run(); } } diff --git a/src/Services/DeployRequestProcessor.cs b/src/Services/DeployRequestProcessor.cs index eb05e13..4b6002b 100644 --- a/src/Services/DeployRequestProcessor.cs +++ b/src/Services/DeployRequestProcessor.cs @@ -16,6 +16,7 @@ public class DeployRequestProcessor : BackgroundService { private readonly Utils.TimeUtils _timeUtils; private string _scheduleImportDirectory; private string _regctlApp; + private readonly ConcurrentDictionary _deployRequestCache; public DeployRequestProcessor(ILogger logger, IServiceProvider serviceProvider, IOptions appConfig, Core.Services.PluginLoader pluginLoader, Core.Client client, PluginDelegates pluginDelegates, Utils.K8sClient k8sClient, Utils.DownlinkUtil downlinkUtil, Utils.TimeUtils timeUtil) { _logger = logger; @@ -27,12 +28,14 @@ public DeployRequestProcessor(ILogger logger, IServicePr _k8sClient = k8sClient; _downlinkUtil = downlinkUtil; _timeUtils = timeUtil; + _deployRequestCache = new ConcurrentDictionary(); _scheduleImportDirectory = Path.Combine(_client.GetXFerDirectories().Result.inbox_directory, _appConfig.SCHEDULE_IMPORT_DIRECTORY); _regctlApp = Path.Combine(_client.GetXFerDirectories().Result.root_directory, "tmp", "regctl", "regctl"); + if (File.Exists(_regctlApp)) { _logger.LogInformation("regctl found at '{regctlApp}'", _regctlApp); } else { @@ -42,7 +45,6 @@ public DeployRequestProcessor(ILogger logger, IServicePr if (_appConfig.PURGE_SCHEDULE_ON_BOOTUP) { _client.ClearCache(); - if (Directory.Exists(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults"))) Directory.Delete(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults"), true); } PopulateCacheFromDisk(); diff --git a/src/Services/ScheduleProcessor.cs b/src/Services/ScheduleProcessor.cs index 7fdbf4a..fb6e32c 100644 --- a/src/Services/ScheduleProcessor.cs +++ b/src/Services/ScheduleProcessor.cs @@ -70,6 +70,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { File.Move(file, file + ".processed"); downlinkFileName = file + ".processed"; + } catch (FileNotFoundException fileEx) { + _logger.LogWarning("Detected a missing file '{file}'. Likely hasn't finished uploaded. Will retry. ", fileEx.FileName); + return deployResponses; // This'll be empty } catch (Utils.NotAScheduleFileException notAScheduleFileEx) { _logger.LogInformation("Detected a json file that isn't a schedule file. {exceptionMessage}", notAScheduleFileEx.Message); return deployResponses; // This'll be empty @@ -238,6 +241,9 @@ private void WaitForFileToFinishCopying(string filePath) { returnDeployItems.Add(_response); } + } catch (FileNotFoundException fileEx) { + _logger.LogError("File '{file}' found. Likely hasnt finished uploading. Will retry.", fileEx.FileName); + throw; } catch (DataException dataEx) { _logger.LogError("Item #'{itemX}' in '{file}' is invalid. Rejecting entire schedule file. Error: '{error}'. Please re-upload for reprocessing", itemCount, scheduleFilePath, dataEx.Message); throw; diff --git a/src/Utils/K8sClient.cs b/src/Utils/K8sClient.cs index 2d28ea1..75d76f0 100644 --- a/src/Utils/K8sClient.cs +++ b/src/Utils/K8sClient.cs @@ -11,23 +11,27 @@ public class K8sClient { private readonly IServiceProvider _serviceProvider; private readonly Models.APP_CONFIG _appConfig; private readonly string _deploymentOutputDir; - public K8sClient(ILogger logger, IServiceProvider serviceProvider, Core.Client client, IOptions appConfig) { + private Utils.TemplateUtil _templateUtil; + public K8sClient(ILogger logger, IServiceProvider serviceProvider, Core.Client client, IOptions appConfig, Utils.TemplateUtil templateUtil) { _logger = logger; _serviceProvider = serviceProvider; _client = client; _appConfig = appConfig.Value; - KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildDefaultConfig(); - _k8sClient = new Kubernetes(config); - + _templateUtil = templateUtil; _deploymentOutputDir = Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deployments"); - if (_appConfig.PURGE_SCHEDULE_ON_BOOTUP) { - if (Directory.Exists(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults"))) Directory.Delete(Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deploymentResults")); - if (Directory.Exists(_deploymentOutputDir)) Directory.Delete(_deploymentOutputDir, true); + if (_appConfig.PURGE_SCHEDULE_ON_BOOTUP && Directory.Exists(_deploymentOutputDir)) { + Directory.Delete(_deploymentOutputDir, true); } + Directory.CreateDirectory(_deploymentOutputDir); + KubernetesClientConfiguration config = KubernetesClientConfiguration.BuildDefaultConfig(); + _k8sClient = new Kubernetes(config); + + + _logger.LogInformation("Services.{serviceName} Initialized.", nameof(K8sClient)); } @@ -49,7 +53,6 @@ public MessageFormats.PlatformServices.Deployment.DeployResponse DeployItem(Mess private MessageFormats.PlatformServices.Deployment.DeployResponse DeployToKubernetes(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { string tokensizedYamlObject = ""; IKubernetesObject? processed_kubernetesObject; - int itemX = 0; _logger.LogInformation("Deployment Started. (AppName: '{AppName}' / DeployAction: '{DeployAction}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')", deploymentItem.DeployRequest.AppName, deploymentItem.DeployRequest.DeployAction, deploymentItem.DeployRequest.RequestHeader.TrackingId, deploymentItem.DeployRequest.RequestHeader.CorrelationId); // _logger.LogInformation("Passing {requestType} and {responseType} to plugins (trackingId: '{trackingId}' / correlationId: '{correlationId}')", deploymentItem.GetType().Name, returnResponse.GetType().Name, deployRequest.RequestHeader.TrackingId, deployRequest.RequestHeader.CorrelationId); @@ -86,13 +89,6 @@ private MessageFormats.PlatformServices.Deployment.DeployResponse DeployToKubern File.WriteAllText(Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_orig"), deploymentItem.DeployRequest.YamlFileContents); } - // Go and make sure the file server credentials are created - if ((deploymentItem.DeployRequest.DeployAction == MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Apply) || - (deploymentItem.DeployRequest.DeployAction == MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Create)) { - AddFileServerCredentials(deploymentItem.DeployRequest); - AddFileServerVolumesAndClaims(deploymentItem.DeployRequest); - AddConfigurationAsSecrets(deploymentItem.DeployRequest); - } if (deploymentItem.DeployRequest.DeployAction == MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.RestartDeployment) { _logger.LogInformation("Restarting deployment '{AppName}' in Namespace '{NameSpace}' (AppName: '{AppName}' / DeployAction: '{DeployAction}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')", deploymentItem.DeployRequest.AppName, deploymentItem.DeployRequest.NameSpace, deploymentItem.DeployRequest.AppName, deploymentItem.DeployRequest.DeployAction, deploymentItem.DeployRequest.RequestHeader.TrackingId, deploymentItem.DeployRequest.RequestHeader.CorrelationId); @@ -118,67 +114,68 @@ private MessageFormats.PlatformServices.Deployment.DeployResponse DeployToKubern } - // Loop through the yaml objects from the string - foreach (var input_kubernetesObject in (List) KubernetesYaml.LoadAllFromString(deploymentItem.DeployRequest.YamlFileContents)) { - itemX++; - if (((IMetadata) input_kubernetesObject).Metadata == null) { - throw new NullReferenceException("Metadata is null or empty"); - } + List kubernetesObjects = _templateUtil.GenerateKubernetesObjectsFromDeployment(deploymentItem); - if (string.IsNullOrEmpty(((IMetadata) input_kubernetesObject).Metadata.Name)) { - throw new NullReferenceException("Metadata.Name is null or empty"); - } - - processed_kubernetesObject = (IKubernetesObject) input_kubernetesObject; + V1PersistentVolumeList allVolumes = _k8sClient.ListPersistentVolumeAsync().Result; + V1PersistentVolumeClaimList allVolumeClaims = _k8sClient.ListPersistentVolumeClaimForAllNamespacesAsync().Result; + V1ServiceAccountList allServiceAccounts = _k8sClient.ListServiceAccountForAllNamespacesAsync().Result; - if (input_kubernetesObject.GetType() == typeof(V1Deployment)) { - processed_kubernetesObject = AddUpdateMetaDataToDeployment((V1Deployment) input_kubernetesObject, deploymentItem.DeployRequest.ContainerInjectionTarget, deploymentItem.DeployRequest); - processed_kubernetesObject = AddContainerAndVolumeInjections((V1Deployment) processed_kubernetesObject, deploymentItem.DeployRequest.ContainerInjectionTarget, deploymentItem.DeployRequest); - processed_kubernetesObject = AddServiceAccountName((V1Deployment) processed_kubernetesObject); + // Loop through the yaml objects and start deploying them + for (int itemX = 0; itemX < kubernetesObjects.Count; itemX++) { + IKubernetesObject kubernetesObject = kubernetesObjects[itemX]; - // Update the deployment object's tokens - tokensizedYamlObject = replaceTemplateTokens(KubernetesYaml.Serialize(processed_kubernetesObject), deploymentItem.DeployRequest); - // Reload the yaml object with the next tokenized stuff - processed_kubernetesObject = (IKubernetesObject) KubernetesYaml.LoadAllFromString(tokensizedYamlObject).First(); - if (_appConfig.ENABLE_YAML_DEBUG) { - _logger.LogDebug("ENABLE_YAML_DEBUG = 'true'. Outputting item #{itemCount} 'pre' file to '{yamlDestination}'. (AppName: '{AppName}' / DeployAction: '{DeployAction}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')", itemX.ToString(), Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_pre"), deploymentItem.DeployRequest.AppName, deploymentItem.DeployRequest.DeployAction, deploymentItem.DeployRequest.RequestHeader.TrackingId, deploymentItem.DeployRequest.RequestHeader.CorrelationId); - File.WriteAllText(Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_pre"), KubernetesYaml.Serialize(processed_kubernetesObject)); + if (_appConfig.ENABLE_YAML_DEBUG) { + _logger.LogDebug("ENABLE_YAML_DEBUG = 'true'. Outputting generated file to '{yamlDestination}'. (AppName: '{AppName}' / DeployAction: '{DeployAction}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')", + Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_post_" + itemX.ToString() + "_post"), + deploymentItem.DeployRequest.AppName, + deploymentItem.DeployRequest.DeployAction, + deploymentItem.DeployRequest.RequestHeader.TrackingId, + deploymentItem.DeployRequest.RequestHeader.CorrelationId); - _logger.LogDebug("ENABLE_YAML_DEBUG = 'true'. Outputting item #{itemCount} 'post' file to '{yamlDestination}'. (AppName: '{AppName}' / DeployAction: '{DeployAction}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')", itemX.ToString(), Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_post"), deploymentItem.DeployRequest.AppName, deploymentItem.DeployRequest.DeployAction, deploymentItem.DeployRequest.RequestHeader.TrackingId, deploymentItem.DeployRequest.RequestHeader.CorrelationId); - File.WriteAllText(Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_post"), tokensizedYamlObject); - } + File.WriteAllText(Path.Combine(_deploymentOutputDir, deploymentItem.DeployRequest.RequestHeader.TrackingId + "_post_" + itemX.ToString() + "_post"), KubernetesYaml.Serialize(kubernetesObject)); } + switch (deploymentItem.DeployRequest.DeployAction) { + case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Apply: + if ((kubernetesObject is V1PersistentVolumeClaim volumeClaim) && allVolumeClaims.Items.Any(pvc => pvc.Name().Equals(volumeClaim.Name(), StringComparison.InvariantCultureIgnoreCase))) { + if (volumeClaim.Namespace().Equals(volumeClaim.Namespace(), StringComparison.InvariantCultureIgnoreCase)) { + _logger.LogDebug("Found pre-existing PersistentVolumeClaim '{volumeName}'. Nothing to do", volumeClaim.Name()); + break; + } + _logger.LogDebug("Found pre-existing PersistentVolumeClaim '{volumeName}' in wrong namespace '{volumeNameSpace}'. Deleting old claim to allow new provision", volumeClaim.Name(), volumeClaim.Namespace()); + _k8sClient.DeleteNamespacedPersistentVolumeClaim(name: volumeClaim.Name(), namespaceParameter: volumeClaim.Namespace()); + } - try { - switch (deploymentItem.DeployRequest.DeployAction) { - case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Apply: - PatchViaYamlObject(processed_kubernetesObject); - break; - case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Delete: - DeleteViaYamlObject(processed_kubernetesObject); + if ((kubernetesObject is V1PersistentVolume volume) && allVolumes.Items.Any(pvv => pvv.Name().Equals(volume.Name(), StringComparison.InvariantCultureIgnoreCase))) { + _logger.LogDebug("Found pre-existing PersistentVolume '{volumeName}'. Nothing to do", volume.Name()); break; - case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Create: - CreateViaYamlObject(processed_kubernetesObject); - break; - default: - throw new Exception(string.Format($"Unknown DeployAction: {deploymentItem.DeployRequest.DeployAction}")); - } - } catch (Exception ex) { - _logger.LogError("Failed to '{action}' item #{itemX} '{objectName}' in yaml. Writing failures to '{failurePath}'. Error: {ex} (trackingId: '{trackingId}' / correlationId: '{correlationId}')", - deploymentItem.DeployRequest.DeployAction, itemX, input_kubernetesObject.GetType().Name, Path.Combine(_deploymentOutputDir, "failures"), ex.Message, deploymentItem.DeployRequest.RequestHeader.TrackingId, deploymentItem.DeployRequest.RequestHeader.CorrelationId); + } - File.WriteAllText(Path.Combine(_deploymentOutputDir, "failures", deploymentItem.DeployRequest.RequestHeader.TrackingId + "_orig"), deploymentItem.DeployRequest.YamlFileContents); - if (processed_kubernetesObject != null) File.WriteAllText(Path.Combine(_deploymentOutputDir, "failures", deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_pre"), KubernetesYaml.Serialize(processed_kubernetesObject)); - if (!string.IsNullOrWhiteSpace(tokensizedYamlObject)) File.WriteAllText(Path.Combine(_deploymentOutputDir, "failures", deploymentItem.DeployRequest.RequestHeader.TrackingId + "_item" + itemX.ToString() + "_post"), tokensizedYamlObject); + if ((kubernetesObject is V1ServiceAccount serviceAccount) && allServiceAccounts.Items.Any(svc => svc.Name().Equals(serviceAccount.Name(), StringComparison.InvariantCultureIgnoreCase) && svc.Namespace().Equals(serviceAccount.Namespace(), StringComparison.InvariantCultureIgnoreCase))) { + _logger.LogDebug("Found pre-existing ServiceAccount '{serviceAccount}'. Nothing to do", serviceAccount.Name()); + break; + } - deploymentItem.ResponseHeader.Status = MessageFormats.Common.StatusCodes.GeneralFailure; - deploymentItem.ResponseHeader.Message = string.Format($"Failed '{deploymentItem.DeployRequest.DeployAction}' action. Error: {ex.Message}"); - throw; + PatchViaYamlObject(kubernetesObject); + break; + case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Delete: + DeleteViaYamlObject(kubernetesObject); + break; + case MessageFormats.PlatformServices.Deployment.DeployRequest.Types.DeployActions.Create: + CreateViaYamlObject(kubernetesObject); + break; + default: + throw new Exception(string.Format($"Unknown DeployAction: {deploymentItem.DeployRequest.DeployAction}")); } } + + if (_appConfig.FILESERVER_SMB_ENABLED) { + AddFileServerCredentials(deploymentItem.DeployRequest); + } + + deploymentItem.ResponseHeader.Status = MessageFormats.Common.StatusCodes.Successful; } catch (Exception ex) { _logger.LogError("Failed to deploy action '{action}'. Error: {ex} (trackingId: '{trackingId}' / correlationId: '{correlationId}')", @@ -187,12 +184,20 @@ private MessageFormats.PlatformServices.Deployment.DeployResponse DeployToKubern deploymentItem.ResponseHeader.Message = string.Format($"Failed '{deploymentItem.DeployRequest.DeployAction}' action. Error: {ex.Message}"); } + + return deploymentItem; } private void PatchViaYamlObject(IKubernetesObject yamlObject) { try { switch (yamlObject) { + case V1PersistentVolumeClaim pvc: + _k8sClient.PatchNamespacedPersistentVolumeClaim(new V1Patch(pvc, V1Patch.PatchType.MergePatch), name: pvc.Metadata.Name, namespaceParameter: pvc.Metadata.NamespaceProperty); + break; + case V1PersistentVolume pv: + _k8sClient.PatchPersistentVolume(new V1Patch(pv, V1Patch.PatchType.MergePatch), name: pv.Metadata.Name); + break; case V1ConfigMap cm: _k8sClient.PatchNamespacedConfigMap(new V1Patch(cm, V1Patch.PatchType.MergePatch), name: cm.Metadata.Name, namespaceParameter: cm.Metadata.NamespaceProperty); break; @@ -248,6 +253,12 @@ private void PatchViaYamlObject(IKubernetesObject yamlObject) { private void DeleteViaYamlObject(IKubernetesObject yamlObject) { try { switch (yamlObject) { + case V1PersistentVolumeClaim pvc: + _k8sClient.DeleteNamespacedPersistentVolumeClaim(name: pvc.Metadata.Name, namespaceParameter: pvc.Metadata.NamespaceProperty); + break; + case V1PersistentVolume pv: + _k8sClient.DeletePersistentVolume(name: pv.Metadata.Name); + break; case V1ConfigMap cm: if (cm.Metadata == null || string.IsNullOrEmpty(cm.Metadata.NamespaceProperty)) { throw new NullReferenceException("Metadata.NamespaceProperty is null or empty"); } _k8sClient.DeleteNamespacedConfigMap(name: cm.Metadata.Name, namespaceParameter: cm.Metadata.NamespaceProperty); @@ -289,10 +300,15 @@ private void DeleteViaYamlObject(IKubernetesObject yamlObject) { throw; } } - private void CreateViaYamlObject(IKubernetesObject yamlObject) { try { switch (yamlObject) { + case V1PersistentVolume pv: + _k8sClient.CreatePersistentVolume(body: pv); + break; + case V1PersistentVolumeClaim pvc: + _k8sClient.CreateNamespacedPersistentVolumeClaim(body: pvc, namespaceParameter: pvc.Metadata.NamespaceProperty); + break; case V1Namespace ns: _k8sClient.CreateNamespace(body: ns); break; @@ -482,330 +498,7 @@ private void AddFileServerCredentials(MessageFormats.PlatformServices.Deployment File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_fileServerSecrets"), KubernetesYaml.Serialize(allFileServerSecrets)); File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_appFileServerCreds"), KubernetesYaml.Serialize(appFileServerCreds)); } - - - } - - /// - /// Check if we need to add the Persistent Volumes and Claims - /// - private void AddFileServerVolumesAndClaims(MessageFormats.PlatformServices.Deployment.DeployRequest deployRequest) { - _logger.LogDebug("Checking for Persistent Volume Claims for '{appId}'...", deployRequest.AppName.ToLower()); - bool hasClaim = false; - bool hasVolume = false; - - _logger.LogDebug("Adding '{appId}' file server claims...", deployRequest.AppName.ToLower()); - - string input_yaml = replaceTemplateTokens($"{_appConfig.FILESERVER_PERSISTENT_VOLUMES}\n{_appConfig.FILESERVER_PERSISTENT_VOLUMECLAIMS}", deployRequest); - - if (_appConfig.ENABLE_YAML_DEBUG) { - File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_fileServerVolumeClaims_pre"), input_yaml); - } - - foreach (var obj in KubernetesYaml.LoadAllFromString(input_yaml)) { - hasClaim = false; - hasVolume = false; - IKubernetesObject k8sObject = (IKubernetesObject) obj; - - if (k8sObject.GetType() == typeof(V1PersistentVolume)) { - V1PersistentVolume k8sVolume = (V1PersistentVolume) obj; - - V1PersistentVolumeList allVolumes = _k8sClient.ListPersistentVolumeAsync().Result; - - foreach (V1PersistentVolume volume in allVolumes) { - if (volume.Name().Equals(k8sVolume.Name(), comparisonType: StringComparison.InvariantCultureIgnoreCase)) hasVolume = true; - } - - if (hasVolume) { - _logger.LogDebug("Found pre-existing volume '{volumeName}'.", k8sVolume.Name()); - } else { - _logger.LogDebug("Adding volume '{volumeName}'....", k8sVolume.Name()); - _ = _k8sClient.CreatePersistentVolumeAsync(k8sVolume).Result; - } - } - - if (k8sObject.GetType() == typeof(V1PersistentVolumeClaim)) { - V1PersistentVolumeClaimList allClaims = _k8sClient.ListNamespacedPersistentVolumeClaimAsync(namespaceParameter: deployRequest.NameSpace).Result; - V1PersistentVolumeClaim k8sVolumeClaim = (V1PersistentVolumeClaim) obj; - - foreach (V1PersistentVolumeClaim volumeClaim in allClaims) { - if (volumeClaim.Name().Equals(k8sVolumeClaim.Name(), comparisonType: StringComparison.InvariantCultureIgnoreCase)) hasClaim = true; - } - - if (hasClaim) { - _logger.LogDebug("Found pre-existing volume claim '{claimName}'.", k8sVolumeClaim.Name()); - } else { - _logger.LogDebug("Adding volume claim '{claimName}'....", k8sVolumeClaim.Name()); - _ = _k8sClient.CreateNamespacedPersistentVolumeClaimAsync(k8sVolumeClaim, namespaceParameter: deployRequest.NameSpace).Result; - } - } - - } - - if (_appConfig.ENABLE_YAML_DEBUG) { - File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_fileServerVolumeClaims_post"), input_yaml); - } - - } - - /// - /// Check if we need to add the Persistent Volumes and Claims - /// - private void AddConfigurationAsSecrets(MessageFormats.PlatformServices.Deployment.DeployRequest deployRequest) { - _logger.LogDebug("Adding configuration secret for '{appId}'...", deployRequest.AppName.ToLower()); - - V1SecretList currentSecrets = _k8sClient.ListNamespacedSecretAsync(namespaceParameter: deployRequest.NameSpace).Result; - - string input_yaml = replaceTemplateTokens($"{_appConfig.PAYLOAD_APP_CONFIG}", deployRequest); - - foreach (var obj in KubernetesYaml.LoadAllFromString(input_yaml)) { - IKubernetesObject k8sObject = (IKubernetesObject) obj; - - if (k8sObject.GetType() == typeof(V1Secret)) { - V1Secret k8sSecret = (V1Secret) obj; - - // The payload_app_config value is base64 encoded. We need to decode it and then re-encode it - k8sSecret.Data = k8sSecret.Data.ToDictionary( - data => data.Key, - data => Encoding.UTF8.GetBytes(replaceTemplateTokens(Encoding.UTF8.GetString(Convert.FromBase64String(Encoding.UTF8.GetString(data.Value))), deployRequest)) - ); - - if (currentSecrets.Items.Any(secret => secret.EnsureMetadata().Name == k8sSecret.Metadata.Name)) { - _ = _k8sClient.PatchNamespacedSecretAsync(new V1Patch(k8sSecret, V1Patch.PatchType.MergePatch), name: k8sSecret.Metadata.Name, namespaceParameter: deployRequest.NameSpace).Result; - } else { - _ = _k8sClient.CreateNamespacedSecretAsync(k8sSecret, namespaceParameter: deployRequest.NameSpace).Result; - } - } - } - - _logger.LogDebug("Configuration secret successfully added for '{appId}'...", deployRequest.AppName.ToLower()); - - } - - /// - /// Add any missing labels and anootations to a deployment - /// - private V1Deployment AddUpdateMetaDataToDeployment(V1Deployment yamlDeployment, string containerInjectionTarget, MessageFormats.PlatformServices.Deployment.DeployRequest deployRequest) { - yamlDeployment.EnsureMetadata(); - yamlDeployment.Metadata.EnsureAnnotations(); - yamlDeployment.Metadata.EnsureLabels(); - yamlDeployment.Spec.Template.EnsureMetadata(); - yamlDeployment.Spec.Template.Metadata.EnsureAnnotations(); - yamlDeployment.Spec.Template.Metadata.EnsureLabels(); - - string input_yaml = replaceTemplateTokens($"{_appConfig.PAYLOAD_APP_ANNOTATIONS}", deployRequest); - - if (_appConfig.ENABLE_YAML_DEBUG) { - File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_payloadAppNotations_pre"), input_yaml); - } - - // Loop through and add annotations - foreach (string annotation in input_yaml.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { - string[] parts = annotation.Split(':'); - string key = parts[0].Trim(); - string value = parts[1].Trim(); - - // Remove the quotes that are automatically added by chart - value = value.Replace("\"", ""); - - if (!yamlDeployment.Metadata.Annotations.ContainsKey(key)) yamlDeployment.Metadata.Annotations.Add(key, value); - if (!yamlDeployment.Spec.Template.Metadata.Annotations.ContainsKey(key)) yamlDeployment.Spec.Template.Metadata.Annotations.Add(key, value); - } - - - input_yaml = replaceTemplateTokens($"{_appConfig.DAPR_ANNOTATIONS}", deployRequest); - - // Loop through and add annotations - foreach (string annotation in input_yaml.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { - string[] parts = annotation.Split(':'); - string key = parts[0].Trim(); - string value = parts[1].Trim(); - - // Remove the quotes that are automatically added by chart - value = value.Replace("\"", ""); - - if (!yamlDeployment.Metadata.Annotations.ContainsKey(key)) yamlDeployment.Metadata.Annotations.Add(key, value); - if (!yamlDeployment.Spec.Template.Metadata.Annotations.ContainsKey(key)) yamlDeployment.Spec.Template.Metadata.Annotations.Add(key, value); - } - - - input_yaml = replaceTemplateTokens($"{_appConfig.PAYLOAD_APP_LABELS}", deployRequest); - - if (_appConfig.ENABLE_YAML_DEBUG) { - File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_payloadAppLabels_pre"), input_yaml); - } - - // Loop through and add labels - foreach (string label in input_yaml.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) { - string[] parts = label.Split(':'); - string key = parts[0].Trim(); - string value = parts[1].Trim(); - - // Remove the quotes that are automatically added by chart - value = value.Replace("\"", ""); - - if (!yamlDeployment.Metadata.Labels.ContainsKey(key)) yamlDeployment.Metadata.Labels.Add(key, value); - if (!yamlDeployment.Spec.Template.Metadata.Labels.ContainsKey(key)) yamlDeployment.Spec.Template.Metadata.Labels.Add(key, value); - } - - int containerInjectionTargetX = yamlDeployment.Spec.Template.Spec.Containers.IndexOf(yamlDeployment.Spec.Template.Spec.Containers.FirstOrDefault(_container => _container.Name == containerInjectionTarget)); - if (containerInjectionTargetX == -1) containerInjectionTargetX = 0; - - // Loop through the environment variables and add them to the target container - input_yaml = replaceTemplateTokens($"{_appConfig.PAYLOAD_APP_ENVIRONMENTVARIABLES}", deployRequest); - - var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() - .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention.Instance) - .Build(); - - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env ??= new List(); - - foreach (var environmentvariable in deserializer.Deserialize>>(input_yaml)) { - if (!yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Any(yamlEnvVar => yamlEnvVar.Name == environmentvariable["name"])) { - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Add(new V1EnvVar() { Name = environmentvariable["name"], Value = environmentvariable["value"] }); - } - } - - // Add the SPACEFX_DIR and SPACEFX_SECRET_DIR since the path may be dynamically changed - if (yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.FirstOrDefault(yamlEnvVar => yamlEnvVar.Name == "SPACEFX_DIR") == null) { - // Value doesn't exist and needs to be created - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Add(new V1EnvVar() { Name = "SPACEFX_DIR", Value = Environment.GetEnvironmentVariable("SPACEFX_DIR") }); - } else { - // Value is already specified. Overwrite it with the value from Platform-Deployment - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.First(yamlEnvVar => yamlEnvVar.Name == "SPACEFX_DIR").Value = Environment.GetEnvironmentVariable("SPACEFX_DIR"); - } - - if (yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.FirstOrDefault(yamlEnvVar => yamlEnvVar.Name == "SPACEFX_SECRET_DIR") == null) { - // Value doesn't exist and needs to be created - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Add(new V1EnvVar() { Name = "SPACEFX_SECRET_DIR", Value = Environment.GetEnvironmentVariable("SPACEFX_SECRET_DIR") }); - } else { - // Value is already specified. Overwrite it with the value from Platform-Deployment - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.First(yamlEnvVar => yamlEnvVar.Name == "SPACEFX_SECRET_DIR").Value = Environment.GetEnvironmentVariable("SPACEFX_SECRET_DIR"); - } - - if (deployRequest.AppContextString != null && !string.IsNullOrWhiteSpace(deployRequest.AppContextString.AppContext)) { - if (yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.FirstOrDefault(yamlEnvVar => yamlEnvVar.Name == "APP_CONTEXT") == null) { - // Value doesn't exist and needs to be created - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Add(new V1EnvVar() { Name = "APP_CONTEXT", Value = deployRequest.AppContextString.AppContext }); - } else { - // Value is already specified. Overwrite it with the value from Platform-Deployment - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.First(yamlEnvVar => yamlEnvVar.Name == "APP_CONTEXT").Value = deployRequest.AppContextString.AppContext; - } - } - - // Add default limits and requests to all containers - for (int i = 0; i < yamlDeployment.Spec.Template.Spec.Containers.Count; i++) { - // Only update the target container with the metadata info. Otherwise all the containers get the injections - if (!string.IsNullOrWhiteSpace(containerInjectionTarget) && yamlDeployment.Spec.Template.Spec.Containers[i].Name != containerInjectionTarget) continue; - - yamlDeployment.Spec.Template.Spec.Containers[i].Resources ??= new V1ResourceRequirements(); - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits ??= new Dictionary(); - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Requests ??= new Dictionary(); - - // Add the default value if it's missing - if (!yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Any(limit => string.Equals(limit.Key, "memory", StringComparison.CurrentCultureIgnoreCase))) { - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Add("memory", new ResourceQuantity(_appConfig.DEFAULT_LIMIT_MEMORY)); - } - - if (!yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Any(limit => string.Equals(limit.Key, "cpu", StringComparison.CurrentCultureIgnoreCase))) { - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Add("cpu", new ResourceQuantity(_appConfig.DEFAULT_LIMIT_CPU)); - } - - // Add the default value if it's missing - if (!yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Requests.Any(request => string.Equals(request.Key, "memory", StringComparison.CurrentCultureIgnoreCase))) { - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Requests.Add("memory", new ResourceQuantity(_appConfig.DEFAULT_REQUEST_MEMORY)); - } - - if (!yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Requests.Any(request => string.Equals(request.Key, "cpu", StringComparison.CurrentCultureIgnoreCase))) { - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Requests.Add("cpu", new ResourceQuantity(_appConfig.DEFAULT_REQUEST_CPU)); - } - - - if (deployRequest.GpuRequirement == MessageFormats.PlatformServices.Deployment.DeployRequest.Types.GpuOptions.Nvidia) { - if (!yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Any(limit => string.Equals(limit.Key, "nvidia.com/gpu", StringComparison.CurrentCultureIgnoreCase))) { - yamlDeployment.Spec.Template.Spec.Containers[i].Resources.Limits.Add("nvidia.com/gpu", new ResourceQuantity("1")); - } - } - - } - - return yamlDeployment; } - /// - /// Adds any missing container and volume injections - /// - private V1Deployment AddContainerAndVolumeInjections(V1Deployment yamlDeployment, string containerInjectionTarget, MessageFormats.PlatformServices.Deployment.DeployRequest deployRequest) { - - var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() - .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention.Instance) - .Build(); - - // Calculate our target container - int containerInjectionTargetX = yamlDeployment.Spec.Template.Spec.Containers.IndexOf(yamlDeployment.Spec.Template.Spec.Containers.FirstOrDefault(_container => _container.Name == containerInjectionTarget)); - if (containerInjectionTargetX == -1) containerInjectionTargetX = 0; - - // Loop through the volumemounts and add it to the target container - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts ??= new List(); - string input_yaml = replaceTemplateTokens($"{_appConfig.FILESERVER_CLIENT_VOLUME_MOUNTS}", deployRequest); - - if (_appConfig.ENABLE_YAML_DEBUG) File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_fileServerVolumeMounts_pre"), input_yaml); - - - foreach (V1VolumeMount volumeMount in deserializer.Deserialize>(input_yaml)) { - if (!yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts.Any(yamlVolumeMount => yamlVolumeMount.Name == volumeMount.Name)) { - yamlDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts.Add(new V1VolumeMount() { Name = volumeMount.Name, MountPath = volumeMount.MountPath }); - } - } - - - yamlDeployment.Spec.Template.Spec.Volumes ??= new List(); - - input_yaml = replaceTemplateTokens($"{_appConfig.FILESERVER_CLIENT_VOLUMES}", deployRequest); - - if (_appConfig.ENABLE_YAML_DEBUG) File.WriteAllText(Path.Combine(_deploymentOutputDir, deployRequest.RequestHeader.TrackingId + "_fileServerVolumes_pre"), input_yaml); - - foreach (V1Volume volume in deserializer.Deserialize>(input_yaml)) { - if (!yamlDeployment.Spec.Template.Spec.Volumes.Any(yamlVolume => yamlVolume.Name == volume.Name)) { - yamlDeployment.Spec.Template.Spec.Volumes.Add(volume); - } - } - - return yamlDeployment; - } - - /// - /// Add service account to deployment - /// - private V1Deployment AddServiceAccountName(V1Deployment yamlDeployment) { - //yamlDeployment.Spec.Template.Spec.ServiceAccountName = _appConfig.DEFAULT_SERVICE_ACCOUNT_NAME; - return yamlDeployment; - } - - private static string replaceTemplateTokens(string yamlContents, MessageFormats.PlatformServices.Deployment.DeployRequest deployRequest) { - - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_NAMESPACE", deployRequest.NameSpace); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_NAME", deployRequest.AppName.ToLower()); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_GROUP", deployRequest.AppGroupLabel); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_TRACKING_ID", deployRequest.RequestHeader.TrackingId); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_CUSTOMER_TRACKING_ID", deployRequest.CustomerTrackingId); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_CORRELATION_ID", deployRequest.RequestHeader.CorrelationId); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_START_TIME", deployRequest.StartTime.ToDateTime().ToUniversalTime().ToString("o")); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_MAX_DURATION", (deployRequest.MaxDuration ??= Google.Protobuf.WellKnownTypes.Duration.FromTimeSpan(TimeSpan.FromHours(5))).Seconds.ToString()); - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_SCHEDULE", deployRequest.Schedule); - - if (deployRequest.AppContextCase != MessageFormats.PlatformServices.Deployment.DeployRequest.AppContextOneofCase.None) { - switch (deployRequest.AppContextCase) { - case MessageFormats.PlatformServices.Deployment.DeployRequest.AppContextOneofCase.AppContextString: - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_CONTEXT", Convert.ToBase64String(System.Text.Encoding.UTF8.GetBytes(deployRequest.AppContextString.AppContext))); - break; - case MessageFormats.PlatformServices.Deployment.DeployRequest.AppContextOneofCase.AppContextFile: - yamlContents = yamlContents.Replace("SPACEFX-TEMPLATE_APP_CONTEXT", "\"\""); - break; - } - } - - return yamlContents; - } } } diff --git a/src/Utils/TemplateUtil.cs b/src/Utils/TemplateUtil.cs new file mode 100644 index 0000000..bbc901d --- /dev/null +++ b/src/Utils/TemplateUtil.cs @@ -0,0 +1,453 @@ +namespace Microsoft.Azure.SpaceFx.PlatformServices.Deployment; +public partial class Utils { + /// + /// Utility class to create template from the SpaceFx Chart + /// + public class TemplateUtil { + private readonly ILogger _logger; + private readonly IServiceProvider _serviceProvider; + private readonly Core.Client _client; + private readonly Models.APP_CONFIG _appConfig; + private string _helmApp; + private string _spacefxChart; + private readonly string _deploymentOutputDir; + + public TemplateUtil(ILogger logger, IServiceProvider serviceProvider, Core.Client client, IOptions appConfig) { + _logger = logger; + _serviceProvider = serviceProvider; + _client = client; + _appConfig = appConfig.Value; + _helmApp = Path.Combine(_client.GetXFerDirectories().Result.root_directory, "tmp", "helm", "helm"); + _spacefxChart = Path.Combine(_client.GetXFerDirectories().Result.root_directory, "tmp", "chart", Core.GetConfigSetting("spacefx_version").Result); + + if (!Directory.Exists(_spacefxChart)) { + _logger.LogWarning("SpaceFx chart not found at '{spacefxChart}' and is required for Platform-Deployment. Please check chart is in the right place and restart Platform-Deployment", _spacefxChart); + throw new DirectoryNotFoundException($"SpaceFx chart not found at '{_spacefxChart}' and is required for Platform-Deployment. Please check chart is in the right place and restart Platform-Deployment"); + } + + if (!File.Exists(_helmApp)) { + _logger.LogWarning("helm not found at '{helmApp}' and is required for Platform-Deployment. Please check helm is in the right place and restart Platform-Deployment", _helmApp); + throw new FileNotFoundException("helm not found at '{helmApp}' and is required for Platform-Deployment. Please check helm is in the right place and restart Platform-Deployment", _helmApp); + } + + _deploymentOutputDir = Path.Combine(_client.GetXFerDirectories().Result.outbox_directory, "deployments"); + + Directory.CreateDirectory(_deploymentOutputDir); + + } + + internal List GenerateKubernetesObjectsFromDeployment(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + List returnList = new() { + GenerateAppSettings(deploymentItem), + GenerateServiceAccount(deploymentItem) + }; + + GeneratePersistentVolumes(deploymentItem).ForEach(pv => returnList.Add(pv)); + GeneratePersistentVolumeClaims(deploymentItem).ForEach(pvc => returnList.Add(pvc)); + + + // Update for any deployment objects + KubernetesYaml.LoadAllFromString(deploymentItem.DeployRequest.YamlFileContents) + .OfType() // Filter for V1Deployment objects using LINQ + .ToList() // Convert to a list + .ForEach(k8sDeployment => returnList.Add(UpdateDeployment(deploymentItem: deploymentItem, k8sDeployment: k8sDeployment))); + + return returnList; + } + + internal V1Deployment UpdateDeployment(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem, V1Deployment k8sDeployment) { + k8sDeployment.EnsureMetadata(); + k8sDeployment.Metadata.EnsureAnnotations(); + k8sDeployment.Metadata.EnsureLabels(); + k8sDeployment.Spec.Template.EnsureMetadata(); + k8sDeployment.Spec.Template.Metadata.EnsureAnnotations(); + k8sDeployment.Spec.Template.Metadata.EnsureLabels(); + + if (string.IsNullOrWhiteSpace(k8sDeployment.Metadata.Name)) { + throw new NullReferenceException("Metadata.Name is null or empty"); + } + + k8sDeployment.Metadata.Name = deploymentItem.DeployRequest.AppName; + k8sDeployment.Metadata.SetNamespace(deploymentItem.DeployRequest.NameSpace); + + k8sDeployment.Spec.Template.Metadata.Name = $"{deploymentItem.DeployRequest.AppName}"; + k8sDeployment.Spec.Template.Metadata.SetNamespace(deploymentItem.DeployRequest.NameSpace); + k8sDeployment.Spec.Template.Spec.ServiceAccountName = $"{deploymentItem.DeployRequest.AppName}"; + + Dictionary template_annotations = GenerateAnnotations(deploymentItem); + Dictionary template_annotationsWithDapr = GenerateAnnotations(deploymentItem, enableDapr: true); + Dictionary template_labels = GenerateLabels(deploymentItem); + Dictionary template_environmentVariables = GenerateEnvironmentVariables(deploymentItem); + Models.KubernetesObjects.ResourceDefinition template_resourceLimits = GenerateResourceLimits(deploymentItem); + List template_volumes = GenerateVolumes(deploymentItem); + List template_volumeMounts = GenerateVolumeMounts(deploymentItem); + + + + // Loop through and add annotations to the deployment and the Deployment Spec + foreach (KeyValuePair kvp in template_annotationsWithDapr) { + if (!k8sDeployment.Metadata.Annotations.ContainsKey(kvp.Key)) k8sDeployment.Metadata.Annotations.Add(kvp.Key, kvp.Value); + if (!k8sDeployment.Spec.Template.Metadata.Annotations.ContainsKey(kvp.Key)) k8sDeployment.Spec.Template.Metadata.Annotations.Add(kvp.Key, kvp.Value); + } + + foreach (KeyValuePair kvp in template_labels) { + if (!k8sDeployment.Metadata.Labels.ContainsKey(kvp.Key)) k8sDeployment.Metadata.Labels.Add(kvp.Key, kvp.Value); + if (!k8sDeployment.Spec.Template.Metadata.Labels.ContainsKey(kvp.Key)) k8sDeployment.Spec.Template.Metadata.Labels.Add(kvp.Key, kvp.Value); + } + + if (k8sDeployment.Spec.Template.Spec.Containers.Count == 0) { + throw new NullReferenceException("Spec.Template.Spec.Containers.Count is 0"); + } + + // Update the requested container with the environment variables + int containerInjectionTargetX = k8sDeployment.Spec.Template.Spec.Containers.IndexOf(k8sDeployment.Spec.Template.Spec.Containers.FirstOrDefault(_container => _container.Name == deploymentItem.DeployRequest.ContainerInjectionTarget)); + if (containerInjectionTargetX == -1) containerInjectionTargetX = 0; + + // Add the volumes from the template to the deployment + k8sDeployment.Spec.Template.Spec.Volumes ??= new List(); + foreach (V1Volume volume in template_volumes) { + if (!k8sDeployment.Spec.Template.Spec.Volumes.Any(yamlVolume => yamlVolume.Name == volume.Name)) { + k8sDeployment.Spec.Template.Spec.Volumes.Add(volume); + } + } + + // Add the volume mounts from the template to the target container + k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts ??= new List(); + foreach (V1VolumeMount volumeMount in template_volumeMounts) { + if (!k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts.Any(yamlVolumeMount => yamlVolumeMount.Name == volumeMount.Name)) { + try { + _logger.LogDebug("Adding volume mount to container: {volumeMount}", volumeMount.Name); + k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].VolumeMounts.Add(volumeMount); + } catch (Exception ex) { + _logger.LogError(ex, "Error adding volume mount to container"); + throw; + } + } + } + + k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env ??= new List(); + + foreach (KeyValuePair kvp in template_environmentVariables) { + if (!k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Any(yamlEnvVar => yamlEnvVar.Name == kvp.Key)) { + k8sDeployment.Spec.Template.Spec.Containers[containerInjectionTargetX].Env.Add(new V1EnvVar() { Name = kvp.Key, Value = kvp.Value }); + } + } + + // Loop through and update the containers with the annotations, labels, and specs + for (int x = 0; x < k8sDeployment.Spec.Template.Spec.Containers.Count; x++) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources ??= new V1ResourceRequirements(); + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits ??= new Dictionary(); + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Requests ??= new Dictionary(); + + // Add the default value if it's missing + if (!k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Any(limit => string.Equals(limit.Key, "memory", StringComparison.CurrentCultureIgnoreCase))) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Add("memory", new ResourceQuantity(template_resourceLimits.Resources.Limits.Memory)); + } + + if (!k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Any(limit => string.Equals(limit.Key, "cpu", StringComparison.CurrentCultureIgnoreCase))) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Add("cpu", new ResourceQuantity(template_resourceLimits.Resources.Limits.Cpu)); + } + + // Add the default value if it's missing + if (!k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Requests.Any(request => string.Equals(request.Key, "memory", StringComparison.CurrentCultureIgnoreCase))) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Requests.Add("memory", new ResourceQuantity(template_resourceLimits.Resources.Requests.Memory)); + } + + if (!k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Requests.Any(request => string.Equals(request.Key, "cpu", StringComparison.CurrentCultureIgnoreCase))) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Requests.Add("cpu", new ResourceQuantity(template_resourceLimits.Resources.Requests.Cpu)); + } + + + if (deploymentItem.DeployRequest.GpuRequirement == MessageFormats.PlatformServices.Deployment.DeployRequest.Types.GpuOptions.Nvidia) { + if (!k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Any(limit => string.Equals(limit.Key, "nvidia.com/gpu", StringComparison.CurrentCultureIgnoreCase))) { + k8sDeployment.Spec.Template.Spec.Containers[x].Resources.Limits.Add("nvidia.com/gpu", new ResourceQuantity("1")); + } + } + + } + + return k8sDeployment; + } + + public string GenerateTemplate(Dictionary helmValuesToSet) { + _logger.LogDebug("Generating template for SpaceFx Chart '{spacefxChart}' with values '{helmValuesToSet}'", _spacefxChart, helmValuesToSet); + string output = "", error = ""; + int returnCode = 0; + + StringBuilder helmValues = new(); + + foreach (string key in helmValuesToSet.Keys) { + helmValues.Append($" --set {key}={helmValuesToSet[key]}"); + } + + ProcessStartInfo startInfo = new ProcessStartInfo { + FileName = _helmApp, + Arguments = $"template {_spacefxChart} {helmValues}", + RedirectStandardOutput = true, + RedirectStandardError = true + }; + + _logger.LogDebug("Helm Template Command: '{app} {arguments}'", startInfo.FileName, startInfo.Arguments); + + using (Process process = new Process { StartInfo = startInfo }) { + process.Start(); + output = process.StandardOutput.ReadToEnd(); + error = process.StandardError.ReadToEnd(); + process.WaitForExit(); + returnCode = process.ExitCode; + } + + + if (_appConfig.ENABLE_YAML_DEBUG) { + string outputFile = Path.Combine(_deploymentOutputDir, $"templateOutput_{DateTime.Now.ToString("yyyyMMdd_HHmmss_fff")}{(returnCode == 0 ? "" : ".error")}.yaml"); + _logger.LogDebug($"ENABLE_YAML_DEBUG = 'true'. Outputting helm generated values to '{outputFile}'"); + File.WriteAllText(outputFile, output); + } + + + if (returnCode > 0) { + throw new ApplicationException($"Helm failed to generate requested template. Output: {output}. Error: {error} Return Code: {returnCode}"); + } + + + _logger.LogDebug("Successfully generated template"); + return output; + } + + public Dictionary StandardTemplateRequestItems(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + return new Dictionary { + { "services.payloadapp.payloadappTemplate.enabled", "true" }, + { "services.payloadapp.payloadappTemplate.schedule.startTime", deploymentItem.DeployRequest.StartTime.ToDateTime().ToString("yyyy-MM-ddTHH:mm:ssZ") }, + { "services.payloadapp.payloadappTemplate.schedule.endTime", deploymentItem.DeployRequest.StartTime.ToDateTime().AddSeconds(deploymentItem.DeployRequest.MaxDuration.ToTimeSpan().TotalSeconds).ToString("yyyy-MM-ddTHH:mm:ssZ") }, + { "services.payloadapp.payloadappTemplate.schedule.recurringSchedule", deploymentItem.DeployRequest.Schedule }, + { "services.payloadapp.payloadappTemplate.schedule.maxDuration", deploymentItem.DeployRequest.MaxDuration.ToTimeSpan().TotalSeconds.ToString() }, + { "services.payloadapp.payloadappTemplate.appContext", deploymentItem.DeployRequest.AppContextCase.ToString() }, + { "services.payloadapp.payloadappTemplate.appName", deploymentItem.DeployRequest.AppName }, + { "services.payloadapp.payloadappTemplate.appGroup", deploymentItem.DeployRequest.AppGroupLabel }, + { "services.payloadapp.payloadappTemplate.correlationId", deploymentItem.DeployRequest.RequestHeader.CorrelationId }, + { "services.payloadapp.payloadappTemplate.customerTrackingId", deploymentItem.DeployRequest.CustomerTrackingId }, + { "services.payloadapp.payloadappTemplate.serviceNamespace", deploymentItem.DeployRequest.NameSpace }, + { "services.payloadapp.payloadappTemplate.trackingId", deploymentItem.DeployRequest.RequestHeader.TrackingId }, + }; + } + + public Dictionary GenerateAnnotations(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem, bool enableDapr = false) { + _logger.LogDebug("Generating annotations template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.annotations.enabled", "true"); + + + if (enableDapr) templateRequest.Add("services.payloadapp.payloadappTemplate.annotations.daprEnabled", "true"); + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.NullNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + string templateYaml = GenerateTemplate(templateRequest); + Dictionary returnDictionary = deserializer.Deserialize>(templateYaml); + + returnDictionary = returnDictionary.ToDictionary( + pair => pair.Key, + pair => pair.Value?.ToString() ?? "" + ); + + return returnDictionary; + } + + public Dictionary GenerateLabels(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating labels template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.labels.enabled", "true"); + + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.NullNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + string templateYaml = GenerateTemplate(templateRequest); + Dictionary returnDictionary = deserializer.Deserialize>(templateYaml); + + returnDictionary = returnDictionary.ToDictionary( + pair => pair.Key, + pair => pair.Value?.ToString() ?? "" + ); + + return returnDictionary; + } + + public Dictionary GenerateEnvironmentVariables(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating environment variables template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.environmentVariables.enabled", "true"); + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.NullNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + string templateYaml = GenerateTemplate(templateRequest); + Dictionary returnDictionary = deserializer.Deserialize>(templateYaml); + + returnDictionary = returnDictionary.ToDictionary( + pair => pair.Key, + pair => pair.Value?.ToString() ?? "" + ); + + return returnDictionary; + } + + public Models.KubernetesObjects.ResourceDefinition GenerateResourceLimits(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating resource limits template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.resources.enabled", "true"); + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.LowerCaseNamingConvention.Instance) // Kubernetes YAML typically uses camelCase + .Build(); + + + string templateYaml = GenerateTemplate(templateRequest); + Models.KubernetesObjects.ResourceDefinition returnValue = deserializer.Deserialize(templateYaml); + + return returnValue; + } + + public V1ConfigMap GenerateAppSettings(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating appsettings template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.appsettings.enabled", "true"); + + string templateYaml = GenerateTemplate(templateRequest); + + V1ConfigMap? returnValue = KubernetesYaml.LoadAllFromString(templateYaml) + .OfType() + .FirstOrDefault(); + + if (returnValue == null) + throw new ApplicationException("Failed to generate AppSettings ConfigMap"); + + return returnValue; + } + + public List GenerateVolumes(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating volumes template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.fileServer.volumesEnabled", "true"); + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention.Instance) // Adjust the naming convention as needed + .Build(); + + + string templateYaml = GenerateTemplate(templateRequest); + + Models.KubernetesObjects.VolumeRoot volumeRoot = deserializer.Deserialize(templateYaml); + + return volumeRoot.Volumes; + } + + public List GenerateVolumeMounts(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating volume mounts template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.fileServer.volumeMountsEnabled", "true"); + + var deserializer = new YamlDotNet.Serialization.DeserializerBuilder() + .WithNamingConvention(YamlDotNet.Serialization.NamingConventions.CamelCaseNamingConvention.Instance) // Adjust the naming convention as needed + .Build(); + + string templateYaml = GenerateTemplate(templateRequest); + + Models.KubernetesObjects.VolumeMountRoot volumeMountRoot = deserializer.Deserialize(templateYaml); + return volumeMountRoot.VolumeMounts; + } + + public List GeneratePersistentVolumeClaims(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating PersistentVolumeClaim template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.persistentVolumes.claimsEnabled", "true"); + + string templateYaml = GenerateTemplate(templateRequest); + + List returnValue = KubernetesYaml.LoadAllFromString(templateYaml).OfType().ToList(); + + if (returnValue == null) + throw new ApplicationException("Failed to generate AppSettings ConfigMap"); + + return returnValue; + } + + public List GeneratePersistentVolumes(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating PersistentVolumeClaim template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.persistentVolumes.volumesEnabled", "true"); + + string templateYaml = GenerateTemplate(templateRequest); + + List returnValue = KubernetesYaml.LoadAllFromString(templateYaml).OfType().ToList(); + + if (returnValue == null) + throw new ApplicationException("Failed to generate AppSettings ConfigMap"); + + return returnValue; + } + + public V1ServiceAccount GenerateServiceAccount(MessageFormats.PlatformServices.Deployment.DeployResponse deploymentItem) { + _logger.LogDebug("Generating ServiceAccount template. (AppName: '{AppName}' / trackingId: '{trackingId}' / correlationId: '{correlationId}')'", + deploymentItem.DeployRequest.AppName, + deploymentItem.ResponseHeader.TrackingId, + deploymentItem.ResponseHeader.CorrelationId); + + Dictionary templateRequest = StandardTemplateRequestItems(deploymentItem); + templateRequest.Add("services.payloadapp.payloadappTemplate.serviceAccount.enabled", "true"); + + string templateYaml = GenerateTemplate(templateRequest); + + V1ServiceAccount? returnValue = KubernetesYaml.LoadAllFromString(templateYaml).OfType().FirstOrDefault(); + + if (returnValue == null) + throw new ApplicationException("Failed to generate AppSettings ConfigMap"); + + return returnValue; + } + } +} diff --git a/src/appsettings.Development.json b/src/appsettings.Development.json index d562635..c8f5535 100644 --- a/src/appsettings.Development.json +++ b/src/appsettings.Development.json @@ -6,8 +6,8 @@ "Microsoft.AspNetCore.Routing.EndpointMiddleware": "None", "Microsoft.AspNetCore.Hosting.Diagnostics": "None", "System.Net.Http": "Error", - "Microsoft.Azure.SpaceFx": "Information", - "Microsoft.Azure.SpaceFx.Core.Services.MessageReceiver": "Information", // Set to Debug to see heartbeats + "Microsoft.Azure.SpaceFx": "Debug", + "Microsoft.Azure.SpaceFx.Core.Services.MessageReceiver": "Information", "Microsoft.Azure.SpaceFx.Core.Services.HeartbeatService": "Information", "Microsoft.Azure.SpaceFx.Core.Services.ResourceUtilizationMonitor": "Information" } diff --git a/src/appsettings.IntegrationTest.json b/src/appsettings.IntegrationTest.json index 7089eb5..2c1c405 100644 --- a/src/appsettings.IntegrationTest.json +++ b/src/appsettings.IntegrationTest.json @@ -16,7 +16,7 @@ "ENABLE_ROUTING_TO_HOSTSVC": false, "PLUG_INS": [ { - "PLUGIN_PATH": "/workspaces/platform-deployment/test/integrationTestPlugin/bin/Debug/net6.0/integrationTestPlugin.dll", + "PLUGIN_PATH": "/workspace/platform-deployment/test/integrationTestPlugin/bin/Debug/net6.0/integrationTestPlugin.dll", "CORE_PERMISSIONS": "ALL", "ENABLED": true } diff --git a/src/platform-deployment.csproj b/src/platform-deployment.csproj index b9122d1..1e90ad8 100644 --- a/src/platform-deployment.csproj +++ b/src/platform-deployment.csproj @@ -20,7 +20,7 @@ - + diff --git a/test/debugClient/Services/MessageSender.cs b/test/debugClient/Services/MessageSender.cs index 627ffa8..16872fd 100644 --- a/test/debugClient/Services/MessageSender.cs +++ b/test/debugClient/Services/MessageSender.cs @@ -26,8 +26,13 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { System.IO.Directory.CreateDirectory(OUTBOX); - System.IO.File.Copy("/workspaces/platform-deployment/test/debugClient/sampleSchedules/busybox.json", string.Format($"{OUTBOX}/busybox.json"), overwrite: true); - System.IO.File.Copy("/workspaces/platform-deployment/test/debugClient/sampleSchedules/busybox.yaml", string.Format($"{OUTBOX}/busybox.yaml"), overwrite: true); + // System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/busybox.json", string.Format($"{OUTBOX}/busybox.json"), overwrite: true); + // System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/busybox.yaml", string.Format($"{OUTBOX}/busybox.yaml"), overwrite: true); + + System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/integration-test-deployment.yaml", string.Format($"{OUTBOX}/integration-test-deployment.yaml"), overwrite: true); + System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/integration-test-schedule.json", string.Format($"{OUTBOX}/integration-test-schedule.json"), overwrite: true); + System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar", string.Format($"{OUTBOX}/pubsub-csharp-subscriber.tar"), overwrite: true); + System.IO.File.Copy("/workspace/platform-deployment/test/sampleSchedules/astronaut.jpg", string.Format($"{OUTBOX}/astronaut.jpg"), overwrite: true); await MoveScheduleArtifacts(); } diff --git a/test/debugClient/debugClient.csproj b/test/debugClient/debugClient.csproj index e0a881e..0cb4274 100644 --- a/test/debugClient/debugClient.csproj +++ b/test/debugClient/debugClient.csproj @@ -10,6 +10,6 @@ - + diff --git a/test/debugClient/outbox/schedule/busybox.json b/test/debugClient/outbox/schedule/busybox.json new file mode 100644 index 0000000..0b3f09e --- /dev/null +++ b/test/debugClient/outbox/schedule/busybox.json @@ -0,0 +1,11 @@ +[ + { + "AppName": "busybox", + "NameSpace": "payload-app", + "AppGroupLabel": "ACME", + "CustomerTrackingId": "abc123-456-789", + "MaxDuration": "120s", + "YamlFileContents": "busybox.yaml", + "DeployAction": "Apply" + } +] \ No newline at end of file diff --git a/test/debugClient/sampleSchedules/busybox.yaml b/test/debugClient/outbox/schedule/busybox.yaml similarity index 91% rename from test/debugClient/sampleSchedules/busybox.yaml rename to test/debugClient/outbox/schedule/busybox.yaml index 0cda0c3..bf983c9 100644 --- a/test/debugClient/sampleSchedules/busybox.yaml +++ b/test/debugClient/outbox/schedule/busybox.yaml @@ -20,4 +20,4 @@ spec: image: busybox:1.28 imagePullPolicy: IfNotPresent command: ["/bin/sh"] - args: ["-c", "sleep 10000"] \ No newline at end of file + args: ["-c", "sleep 10000"] diff --git a/test/debugClient/sampleSchedules/busybox.json b/test/debugClient/sampleSchedules/busybox.json deleted file mode 100644 index 00e487c..0000000 --- a/test/debugClient/sampleSchedules/busybox.json +++ /dev/null @@ -1,21 +0,0 @@ -[ - { - "RequestHeader": { - "CorrelationId": "e88f213e-3573-43fd-9661-ea63251682fc" - }, - "AppName": "busybox", - "NameSpace": "hostsvc", - "AppGroupLabel": "ACME", - "CustomerTrackingId": "abc123-456-789", - "MaxDuration": { - "Seconds": 120, - "Nanos": 0 - }, - "YamlFileContents": "busybox.yaml", - "DeployAction": 0, - "Priority": 1, - "AppContextString": null, - "AppContextFile": null, - "AppContextCase": 0 - } -] \ No newline at end of file diff --git a/test/integrationTestPlugin/integrationTestPlugin.csproj b/test/integrationTestPlugin/integrationTestPlugin.csproj index f3c7734..3d4b949 100644 --- a/test/integrationTestPlugin/integrationTestPlugin.csproj +++ b/test/integrationTestPlugin/integrationTestPlugin.csproj @@ -7,7 +7,7 @@ - + diff --git a/test/integrationTests/Tests/DeploymentTest.cs b/test/integrationTests/Tests/DeploymentTest.cs index ecacf1a..2ca3f4c 100644 --- a/test/integrationTests/Tests/DeploymentTest.cs +++ b/test/integrationTests/Tests/DeploymentTest.cs @@ -94,10 +94,10 @@ void LinkResponseEventHandler(object? _, MessageFormats.HostServices.Link.LinkRe string scheduleDir = Path.Combine((await TestSharedContext.SPACEFX_CLIENT.GetXFerDirectories()).outbox_directory, "schedule"); Directory.CreateDirectory(scheduleDir); - File.Copy("/workspaces/platform-deployment/test/sampleSchedules/integration-test-schedule.json", Path.Combine(scheduleDir, "integration-test-deployment.json"), overwrite: true); - File.Copy("/workspaces/platform-deployment/test/sampleSchedules/integration-test-deployment.yaml", Path.Combine(scheduleDir, "integration-test-deployment.yaml"), overwrite: true); - File.Copy("/workspaces/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar", Path.Combine(scheduleDir, "pubsub-csharp-subscriber.tar"), overwrite: true); - File.Copy("/workspaces/platform-deployment/test/sampleSchedules/astronaut.jpg", Path.Combine(scheduleDir, "astronaut.jpg"), overwrite: true); + File.Copy("/workspace/platform-deployment/test/sampleSchedules/integration-test-schedule.json", Path.Combine(scheduleDir, "integration-test-deployment.json"), overwrite: true); + File.Copy("/workspace/platform-deployment/test/sampleSchedules/integration-test-deployment.yaml", Path.Combine(scheduleDir, "integration-test-deployment.yaml"), overwrite: true); + File.Copy("/workspace/platform-deployment/test/sampleSchedules/pubsub-csharp-subscriber.tar", Path.Combine(scheduleDir, "pubsub-csharp-subscriber.tar"), overwrite: true); + File.Copy("/workspace/platform-deployment/test/sampleSchedules/astronaut.jpg", Path.Combine(scheduleDir, "astronaut.jpg"), overwrite: true); Console.WriteLine($"Sending '{tarFile.GetType().Name}' (TrackingId: '{tarFile.RequestHeader.TrackingId}')"); await TestSharedContext.SPACEFX_CLIENT.DirectToApp("hostsvc-link", tarFile); diff --git a/test/sampleSchedules/ServiceAppBuild.docker b/test/sampleSchedules/ServiceAppBuild.docker index 5c7795d..4ec1606 100644 --- a/test/sampleSchedules/ServiceAppBuild.docker +++ b/test/sampleSchedules/ServiceAppBuild.docker @@ -17,6 +17,6 @@ ENV SERVICE_NAME=${SERVICE_NAME} ENV DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=1 # Copy the minimum files needed to run the service -COPY --from=svc-base --chmod=0755 /workspaces/${SERVICE_NAME} /workspaces/${SERVICE_NAME} +COPY --from=svc-base --chmod=0755 /workspace/${SERVICE_NAME} /workspace/${SERVICE_NAME} USER root -WORKDIR /workspaces/${SERVICE_NAME} +WORKDIR /workspace/${SERVICE_NAME} diff --git a/test/sampleSchedules/busybox.json b/test/sampleSchedules/busybox.json index c54e9fd..0b3f09e 100644 --- a/test/sampleSchedules/busybox.json +++ b/test/sampleSchedules/busybox.json @@ -1,21 +1,11 @@ [ { - "RequestHeader": { - "CorrelationId": "e88f213e-3573-43fd-9661-ea63251682fc" - }, "AppName": "busybox", - "NameSpace": "hostsvc", + "NameSpace": "payload-app", "AppGroupLabel": "ACME", "CustomerTrackingId": "abc123-456-789", - "MaxDuration": { - "Seconds": 120, - "Nanos": 0 - }, + "MaxDuration": "120s", "YamlFileContents": "busybox.yaml", - "DeployAction": 0, - "Priority": 1, - "AppContextString": null, - "AppContextFile": null, - "AppContextCase": 0 + "DeployAction": "Apply" } -] +] \ No newline at end of file diff --git a/test/sampleSchedules/busybox.yaml b/test/sampleSchedules/busybox.yaml index bf983c9..532b5b9 100644 --- a/test/sampleSchedules/busybox.yaml +++ b/test/sampleSchedules/busybox.yaml @@ -2,7 +2,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: busybox - namespace: hostsvc + namespace: payload-app labels: app: busybox spec: