Cleanup orphan worker rows

app/models/miq_server/worker_management/kubernetes.rb

@@ -12,10 +12,6 @@ def sync_from_system
     # we only have to sync the list of pods and deployments once
     ensure_kube_monitors_started if my_server_is_primary?
 
-    # Before syncing the workers check for any orphaned worker rows that don't have
-    # a current pod and delete them
-    cleanup_orphaned_worker_rows
-
     # Update worker deployments with updated settings such as cpu/memory limits
     sync_deployment_settings
   end

app/models/miq_server/worker_management/monitor.rb

@@ -18,6 +18,10 @@ def monitor_workers
     # Cache a list of the native objects backing the miq_workers (e.g.: pods, services, or processes)
     sync_from_system
 
+    # Before syncing the workers check for any orphaned worker rows that don't have
+    # a current resource and delete them
+    cleanup_orphaned_worker_rows
+
     sync_monitor
 
     # Sync the workers after sync'ing the child worker settings

app/models/miq_server/worker_management/process.rb

@@ -15,6 +15,15 @@ def monitor_workers
     do_system_limit_exceeded if kill_workers_due_to_resources_exhausted?
   end
 
+  def cleanup_orphaned_worker_rows
+    orphaned_rows = miq_workers.where.not(:pid => miq_processes.map(&:pid))
+    return if orphaned_rows.empty?
+
+    _log.warn("Removing orphaned worker rows without processes: #{orphaned_rows.collect(&:pid).inspect}")
+
+    orphaned_rows.destroy_all
+  end
+
   def monitor_active_workers
     # Monitor all remaining current worker records
     miq_workers.find_current_or_starting.each do |worker|

app/models/miq_server/worker_management/systemd.rb

@@ -20,6 +20,9 @@ def sync_starting_workers
     starting
   end
 
+  def cleanup_orphaned_worker_rows
+  end
+
   def cleanup_failed_workers
     super
 

spec/models/miq_server/worker_management/kubernetes_spec.rb

@@ -120,42 +120,39 @@
 
   context "#sync_from_system" do
     context "#ensure_kube_monitors_started" do
-      it "podified, ensures pod monitor started and orphaned rows are removed" do
+      it "podified, ensures pod monitor started" do
         expect(server.worker_manager).to receive(:ensure_kube_monitors_started)
-        expect(server.worker_manager).to receive(:cleanup_orphaned_worker_rows)
         server.worker_manager.sync_from_system
       end
     end
   end
 
   context "#cleanup_orphaned_worker_rows" do
-    context "podified" do
-      let(:server2) { EvmSpecHelper.remote_miq_server }
-      let(:worker) do
-        FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :last_heartbeat => 5.minutes.ago)
-      end
+    let(:server2) { EvmSpecHelper.remote_miq_server }
+    let(:worker) do
+      FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :last_heartbeat => 5.minutes.ago)
+    end
 
-      before do
-        server.worker_manager.current_pods = {"1-generic-active" => {}}
-      end
+    before do
+      server.worker_manager.current_pods = {"1-generic-active" => {}}
+    end
 
-      after do
-        server.worker_manager.current_pods.clear
-      end
+    after do
+      server.worker_manager.current_pods.clear
+    end
 
-      it "removes this server's orphaned rows" do
-        worker.update(:system_uid => "1-generic-orphan")
-        FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :system_uid => "1-generic-active")
-        server.worker_manager.cleanup_orphaned_worker_rows
-        expect(MiqWorker.count).to eq(1)
-      end
+    it "removes this server's orphaned rows" do
+      worker.update(:system_uid => "1-generic-orphan")
+      FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :system_uid => "1-generic-active")
+      server.worker_manager.cleanup_orphaned_worker_rows
+      expect(MiqWorker.count).to eq(1)
+    end
 
-      it "skips orphaned rows for other servers" do
-        worker.update(:miq_server => server2, :system_uid => "1-generic-orphan")
-        FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server2, :system_uid => "1-generic-active")
-        server.worker_manager.cleanup_orphaned_worker_rows
-        expect(MiqWorker.count).to eq(2)
-      end
+    it "skips orphaned rows for other servers" do
+      worker.update(:miq_server => server2, :system_uid => "1-generic-orphan")
+      FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server2, :system_uid => "1-generic-active")
+      server.worker_manager.cleanup_orphaned_worker_rows
+      expect(MiqWorker.count).to eq(2)
     end
   end
 

spec/models/miq_server/worker_management/process_spec.rb

@@ -52,4 +52,19 @@
       end
     end
   end
+
+  context "#cleanup_orphaned_worker_rows" do
+    let!(:worker) do
+      FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :status => MiqWorker::STATUS_CREATING, :last_heartbeat => 5.minutes.ago)
+    end
+
+    before { server.worker_manager.instance_variable_set(:@miq_processes, []) }
+    before { server.worker_manager.instance_variable_set(:@miq_processes, nil) }
+
+    it "removes this server's orphaned rows" do
+      byebug
+      server.worker_manager.cleanup_orphaned_worker_rows
+      expect(MiqWorker.count).to be_zero
+    end
+  end
 end

spec/models/miq_server/worker_management/systemd_spec.rb

@@ -53,6 +53,20 @@
     end
   end
 
+  context "#cleanup_orphaned_worker_rows" do
+    let!(:worker) do
+      FactoryBot.create(:miq_worker, :type => "MiqGenericWorker", :miq_server => server, :status => MiqWorker::STATUS_CREATING, :last_heartbeat => 5.minutes.ago)
+    end
+
+    before { server.worker_manager.instance_variable_set(:@miq_processes, []) }
+    before { server.worker_manager.instance_variable_set(:@miq_processes, nil) }
+
+    it "removes this server's orphaned rows" do
+      server.worker_manager.cleanup_orphaned_worker_rows
+      expect(MiqWorker.count).to be_zero
+    end
+  end
+
   context "#failed_miq_services (private)" do
     before { server.worker_manager.sync_from_system }