From dfc5cbb5eddcf092eae54598c90df10493e728e4 Mon Sep 17 00:00:00 2001 From: Adam Grare Date: Fri, 20 Sep 2024 12:25:17 -0400 Subject: [PATCH] Unify Worker Environment Variables systemd/k8s Handling of worker environment variables was spread across a number of different locations and in a number of cases had to be duplicated for systemd/kubernetes. This adds a common `MiqWorker#environment_variables` method that applies to all runtime platforms, as well as specific systemd/container_environment_variables which are merged in depending on the runtime environment. --- app/models/automation_worker.rb | 5 ++++- app/models/miq_worker.rb | 8 +++++++- app/models/miq_worker/container_common.rb | 19 ++++++++++--------- .../miq_worker/deployment_per_worker.rb | 1 - app/models/miq_worker/service_worker.rb | 9 +++++++-- app/models/miq_worker/systemd_common.rb | 10 +++++++--- app/models/mixins/per_ems_worker_mixin.rb | 2 +- systemd/manageiq-automation@.service | 1 - .../manageiq-ems_metrics_processor@.service | 1 - systemd/manageiq-event_handler@.service | 1 - systemd/manageiq-generic@.service | 1 - systemd/manageiq-priority@.service | 1 - systemd/manageiq-remote_console@.service | 1 - systemd/manageiq-reporting@.service | 1 - systemd/manageiq-schedule@.service | 1 - systemd/manageiq-smart_proxy@.service | 1 - systemd/manageiq-ui@.service | 1 - systemd/manageiq-web_service@.service | 1 - 18 files changed, 36 insertions(+), 29 deletions(-) diff --git a/app/models/automation_worker.rb b/app/models/automation_worker.rb index 19d94828fd0..198e9f6e517 100644 --- a/app/models/automation_worker.rb +++ b/app/models/automation_worker.rb @@ -9,10 +9,13 @@ def self.kill_priority MiqWorkerType::KILL_PRIORITY_GENERIC_WORKERS end + def container_environment_variables + super.merge("AUTOMATION_JOB_SERVICE_ACCOUNT" => ENV.fetch("WORKER_SERVICE_ACCOUNT")) + end + def configure_worker_deployment(definition, replicas = 0) super definition[:spec][:template][:spec][:serviceAccountName] = "manageiq-automation" - definition[:spec][:template][:spec][:containers][0][:env] << {:name => "AUTOMATION_JOB_SERVICE_ACCOUNT", :value => ENV.fetch("WORKER_SERVICE_ACCOUNT")} end end diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 79e7da503e3..55dc140a7ae 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -379,9 +379,15 @@ def command_line self.class.build_command_line(*worker_options.values_at(:guid, :ems_id)) end + def environment_variables + { + "BUNDLER_GROUPS" => self.class.bundler_groups.join(",") + } + end + def start_runner_via_spawn pid = Kernel.spawn( - {"BUNDLER_GROUPS" => self.class.bundler_groups.join(",")}, + environment_variables, command_line, [:out, :err] => [Rails.root.join("log/evm.log"), "a"] ) diff --git a/app/models/miq_worker/container_common.rb b/app/models/miq_worker/container_common.rb index 27809f345d6..4994ae88319 100644 --- a/app/models/miq_worker/container_common.rb +++ b/app/models/miq_worker/container_common.rb @@ -10,18 +10,19 @@ def configure_worker_deployment(definition, replicas = 0) definition[:spec][:template][:spec][:nodeSelector] = zone_selector end + env = container_environment_variables.merge(environment_variables) container = definition[:spec][:template][:spec][:containers].first - if container_image_tag.include?("latest") - container[:imagePullPolicy] = "Always" - else - container[:imagePullPolicy] = "IfNotPresent" - end + container[:imagePullPolicy] = container_image_tag.include?("latest") ? "Always" : "IfNotPresent" + container[:image] = container_image + container[:resources] = resource_constraints + container[:env] = env.map { |name, value| {:name => name, :value => value} } + end - container[:image] = container_image - container[:env] << {:name => "WORKER_CLASS_NAME", :value => self.class.name} - container[:env] << {:name => "BUNDLER_GROUPS", :value => self.class.bundler_groups.join(",")} - container[:resources] = resource_constraints + def container_environment_variables + { + "WORKER_CLASS_NAME" => self.class.name + } end def scale_deployment diff --git a/app/models/miq_worker/deployment_per_worker.rb b/app/models/miq_worker/deployment_per_worker.rb index 612ec90e0da..4b638f36ec0 100644 --- a/app/models/miq_worker/deployment_per_worker.rb +++ b/app/models/miq_worker/deployment_per_worker.rb @@ -5,7 +5,6 @@ module DeploymentPerWorker def create_container_objects ContainerOrchestrator.new.create_deployment(worker_deployment_name) do |definition| configure_worker_deployment(definition, 1) - definition[:spec][:template][:spec][:containers].first[:env] << {:name => "EMS_ID", :value => self.class.ems_id_from_queue_name(queue_name)} end end diff --git a/app/models/miq_worker/service_worker.rb b/app/models/miq_worker/service_worker.rb index baa4be4b328..03faa465ede 100644 --- a/app/models/miq_worker/service_worker.rb +++ b/app/models/miq_worker/service_worker.rb @@ -45,8 +45,6 @@ def configure_service_worker_deployment(definition) container = definition[:spec][:template][:spec][:containers].first container[:ports] = [{:containerPort => SERVICE_PORT}, {:containerPort => HEALTH_PORT}] - container[:env] << {:name => "PORT", :value => container_port.to_s} - container[:env] << {:name => "BINDING_ADDRESS", :value => "0.0.0.0"} container[:volumeMounts] ||= [] definition[:spec][:template][:spec][:volumes] ||= [] end @@ -68,5 +66,12 @@ def container_image_name def container_image ENV["WEBSERVER_WORKER_IMAGE"] || default_image end + + def container_environment_variables + { + "PORT" => container_port.to_s, + "BINDING_ADDRESS" => "0.0.0.0" + } + end end end diff --git a/app/models/miq_worker/systemd_common.rb b/app/models/miq_worker/systemd_common.rb index 316a23b8c61..be125b4404e 100644 --- a/app/models/miq_worker/systemd_common.rb +++ b/app/models/miq_worker/systemd_common.rb @@ -116,10 +116,14 @@ def unit_config_file_path end def unit_config_file + # Merge systemd specific environment variables with the generic worker + # environment variables + environment = systemd_environment_variables.merge(environment_variables) + <<~UNIT_CONFIG_FILE [Service] - #{unit_settings.compact.map { |key, value| "#{key}=#{value}" }.join("\n")} - #{unit_environment_variables.compact.map { |key, value| "Environment=#{key}=#{value}" }.join("\n")} + #{unit_settings.compact.map { |key, value| "#{key}=#{value}" }.join("\n")} + #{environment.compact.map { |key, value| "Environment=#{key}=#{value}" }.join("\n")} UNIT_CONFIG_FILE end @@ -135,7 +139,7 @@ def unit_settings end # Override this in a child class to add environment variables - def unit_environment_variables + def systemd_environment_variables {} end end diff --git a/app/models/mixins/per_ems_worker_mixin.rb b/app/models/mixins/per_ems_worker_mixin.rb index c2ddd73c532..701a2e8bef8 100644 --- a/app/models/mixins/per_ems_worker_mixin.rb +++ b/app/models/mixins/per_ems_worker_mixin.rb @@ -126,7 +126,7 @@ def worker_options super.merge(:ems_id => ems_id) end - def unit_environment_variables + def environment_variables super.merge("EMS_ID" => ems_id) end end diff --git a/systemd/manageiq-automation@.service b/systemd/manageiq-automation@.service index 1496ca2d3cc..f8f7f6d8832 100644 --- a/systemd/manageiq-automation@.service +++ b/systemd/manageiq-automation@.service @@ -4,7 +4,6 @@ PartOf=manageiq-automation.target WantedBy=manageiq-automation.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb AutomationWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-ems_metrics_processor@.service b/systemd/manageiq-ems_metrics_processor@.service index 9b099ed6abd..9dd3e09bb57 100644 --- a/systemd/manageiq-ems_metrics_processor@.service +++ b/systemd/manageiq-ems_metrics_processor@.service @@ -4,7 +4,6 @@ PartOf=manageiq-ems_metrics_processor.target WantedBy=manageiq-ems_metrics_processor.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqEmsMetricsProcessorWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-event_handler@.service b/systemd/manageiq-event_handler@.service index 4aa87245b84..a315c6b2adb 100644 --- a/systemd/manageiq-event_handler@.service +++ b/systemd/manageiq-event_handler@.service @@ -4,7 +4,6 @@ PartOf=manageiq-event_handler.target WantedBy=manageiq-event_handler.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqEventHandler --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-generic@.service b/systemd/manageiq-generic@.service index 4308ee12808..cfd7d7fd813 100644 --- a/systemd/manageiq-generic@.service +++ b/systemd/manageiq-generic@.service @@ -4,7 +4,6 @@ PartOf=manageiq-generic.target WantedBy=manageiq-generic.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqGenericWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-priority@.service b/systemd/manageiq-priority@.service index 561b2c4e05e..c9602423207 100644 --- a/systemd/manageiq-priority@.service +++ b/systemd/manageiq-priority@.service @@ -4,7 +4,6 @@ PartOf=manageiq-priority.target WantedBy=manageiq-priority.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqPriorityWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-remote_console@.service b/systemd/manageiq-remote_console@.service index 85459edc807..64cf2a5b471 100644 --- a/systemd/manageiq-remote_console@.service +++ b/systemd/manageiq-remote_console@.service @@ -5,7 +5,6 @@ Wants=httpd.service WantedBy=manageiq-remote_console.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqRemoteConsoleWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-reporting@.service b/systemd/manageiq-reporting@.service index 2c109ae07a7..1b5d1d27081 100644 --- a/systemd/manageiq-reporting@.service +++ b/systemd/manageiq-reporting@.service @@ -4,7 +4,6 @@ PartOf=manageiq-reporting.target WantedBy=manageiq-reporting.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqReportingWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-schedule@.service b/systemd/manageiq-schedule@.service index 700fd33dd4b..3211191e0f0 100644 --- a/systemd/manageiq-schedule@.service +++ b/systemd/manageiq-schedule@.service @@ -4,7 +4,6 @@ PartOf=manageiq-schedule.target WantedBy=manageiq-schedule.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqScheduleWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-smart_proxy@.service b/systemd/manageiq-smart_proxy@.service index 15516a89085..54d8813a26f 100644 --- a/systemd/manageiq-smart_proxy@.service +++ b/systemd/manageiq-smart_proxy@.service @@ -4,7 +4,6 @@ PartOf=manageiq-smart_proxy.target WantedBy=manageiq-smart_proxy.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqSmartProxyWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-ui@.service b/systemd/manageiq-ui@.service index e3bf2f41240..e8ffca23004 100644 --- a/systemd/manageiq-ui@.service +++ b/systemd/manageiq-ui@.service @@ -5,7 +5,6 @@ Wants=httpd.service WantedBy=manageiq-ui.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqUiWorker --heartbeat --guid=%i User=manageiq diff --git a/systemd/manageiq-web_service@.service b/systemd/manageiq-web_service@.service index 12e2f596267..9ebccc6a973 100644 --- a/systemd/manageiq-web_service@.service +++ b/systemd/manageiq-web_service@.service @@ -5,7 +5,6 @@ Wants=httpd.service WantedBy=manageiq-web_service.target [Service] WorkingDirectory=/var/www/miq/vmdb -Environment=BUNDLER_GROUPS=manageiq_default,ui_dependencies EnvironmentFile=/etc/default/manageiq*.properties ExecStart=/usr/bin/ruby lib/workers/bin/run_single_worker.rb MiqWebServiceWorker --heartbeat --guid=%i User=manageiq