Fix more start/stop issues

- Add `FFA` to Scenario Mode dropdown for Free For All (Deathmatch)
- Fix start button not immediately responding when starting multiple servers for the first time. The daemon init method no longer requires the mutex used by start/stop.
- Fix hang and other issues on SAW exit caused by mutexing, sequential handling, and trying to kill a server after unsetting the PID. Improved exit logging.
- Improve exit speed when server is starting by sending kill signal as soon as exit request is detected
- Fix Server Control page JavaScript request for server status queuing additional requests before the previous request finished. This should improve performance of the page, especially for slow/intermittent devices/network.
- Fix Ruby exception in background requests for server status when server hasn't yet started (e.g. another server starting)

admin-interface/docroot/public/custom/custom.js

@@ -617,7 +617,7 @@ function updateThreads(id, element) {
 }
 
 function reloadControlStatus() {
-  $.ajax({
+  return $.ajax({
     url: `/server-control-status`,
     type: 'GET',
     success: (response) => {
@@ -630,21 +630,24 @@ function reloadControlStatus() {
 }
 
 function updateServerControlStatus() {
-  reloadControlStatus();
-  id = $('#config-id').html();
-  if (id && $('#server-status').html() == 'ON') {
-    if (!server_chat_log_active && $('#chat-log').length) {
-      setTimeout(()=>{startServerChatTail('#chat-log', id, server_log_tail_interval);}, 0);
-    }
-    if (!server_log_active && $('#server-log').length) {
-      setTimeout(()=>{startServerLogTail('#server-log', id, server_log_tail_interval);}, 0);
-    }
-    if (!server_rcon_log_active && $('#rcon-log').length) {
-      setTimeout(()=>{startServerRconTail('#rcon-log', id, server_log_tail_interval);}, 0);
+  reloadControlStatus().then(_ => {
+    id = $('#config-id').html();
+    if (id && $('#server-status').html() == 'ON') {
+      if (!server_chat_log_active && $('#chat-log').length) {
+        setTimeout(()=>{startServerChatTail('#chat-log', id, server_log_tail_interval);}, 0);
+      }
+      if (!server_log_active && $('#server-log').length) {
+        setTimeout(()=>{startServerLogTail('#server-log', id, server_log_tail_interval);}, 0);
+      }
+      if (!server_rcon_log_active && $('#rcon-log').length) {
+        setTimeout(()=>{startServerRconTail('#rcon-log', id, server_log_tail_interval);}, 0);
+      }
     }
-  }
-  setTimeout(()=>{updateServerControlStatus();}, 1000);
-  setTimeout(()=>{updateMonitoringDetails(id);}, 1000);
+    setTimeout(()=>{updateMonitoringDetails(id);}, 1000);
+    setTimeout(()=>{updateServerControlStatus();}, 1000);
+  }).catch(_ => {
+    setTimeout(()=>{updateServerControlStatus();}, 1000);
+  });
 }
 
 function addLogLines(target, lines) {

admin-interface/lib/config-handler.rb

@@ -119,6 +119,7 @@
   'Ambush',
   'Checkpoint',
   'Domination',
+  'FFA', # Free For All (Deathmatch)
   'Firefight_A', # Ministry... https://newworldinteractive.com/isl/uploads/2019/09/Sandstorm-Server-Admin-Guide-1.4.pdf
   'Firefight_East',
   'Firefight_West',

admin-interface/lib/daemon.rb

@@ -201,7 +201,6 @@ def do_stop_server
   def implode
     log "Daemon for server #{@name} (#{@config['id']}) imploding", level: :info
     @exit_requested = true
-    @game_pid = nil
     @buffer.reset
     @buffer = nil
     @rcon_buffer.reset
@@ -210,10 +209,12 @@ def implode
     game_server_thread = @threads.delete :game_server
     kill_server_process
     game_server_thread.join unless game_server_thread.nil?
+    @game_pid = nil
     @threads.keys.each do |thread_name|
       thread = @threads.delete thread_name
       thread.kill if thread.respond_to? :kill
     end
+    log "Daemon for server #{@name} (#{@config['id']}) imploded", level: :info
   end
 
   def kill_server_process(signal: nil)
@@ -292,6 +293,10 @@ def run_game_server
                   create_monitor
                   Thread.new do
                     sleep 0.5 until @exit_requested || (@monitor && @monitor.all_green?)
+                    if @exit_requested
+                      kill_server_process
+                      Thread.exit
+                    end
                     log "Server is ready (RCON and Query connected)", level: :info
                     @server_started = true
                   end

admin-interface/lib/server-monitor.rb

@@ -8,11 +8,12 @@
 Geocoder.configure(ip_lookup: :ipapi_com) # :ipinfo_io (default, 1,000/day) :ipapi_com (150/min) #:geoip2 (no actual lookup; not reliable)
 
 class ServerMonitor
+  # Useful for testing hang recovery via pry
+  attr_accessor :query_port
+  attr_accessor :rcon_port
   attr_reader :info
   attr_reader :name
   attr_reader :ip
-  attr_reader :query_port
-  attr_reader :rcon_port
   attr_reader :rcon_pass
   attr_reader :rcon_buffer
 

admin-interface/lib/webapp.rb

@@ -48,7 +48,14 @@ def self.set_up
       Thread.new do
         if @@daemons.any?
           log "Stopping daemons", level: :info
-          @@daemons.each { |_, daemon| daemon.implode }
+          threads = []
+          @@daemons.each do |_, daemon|
+            threads << Thread.new do
+              daemon.implode
+            end
+          end
+          threads.each(&:join)
+          log "Stopped daemons", level: :info
         end
       end.join
     end
@@ -109,22 +116,20 @@ def self.handle_arguments
 
   def self.init_daemon(config, start: false)
     id = config['id']
-    @@daemons_mutex.synchronize do
-      if @@daemons[id]
-        log "Daemon with config ID #{id} already exists", level: :info
-      else
-        log "Initializing daemon with config ID #{id}", level: :info
-        @@daemons[id] = SandstormServerDaemon.new(
-          config,
-          @@daemons,
-          @@daemons_mutex,
-          @@rcon_client,
-          create_buffer.last, # Server log
-          create_buffer.last, # RCON
-          create_buffer.last, # Chat
-          steam_api_key: @@config['steam_api_key']
-        )
-      end
+    if @@daemons[id]
+      log "Daemon with config ID #{id} already exists", level: :info
+    else
+      log "Initializing daemon with config ID #{id}", level: :info
+      @@daemons[id] = SandstormServerDaemon.new(
+        config,
+        @@daemons,
+        @@daemons_mutex,
+        @@rcon_client,
+        create_buffer.last, # Server log
+        create_buffer.last, # RCON
+        create_buffer.last, # Chat
+        steam_api_key: @@config['steam_api_key']
+      )
     end
     @@daemons[id].do_start_server if start
     @@daemons[id]
@@ -810,15 +815,21 @@ def redirect_to_config(config_id, path)
   end
 
   get '/server-control-status' do
-    @config = get_active_config
-    @id = @config['id']
+    active_config = get_active_config
+    @id = active_config['id']
     daemon = @@daemons[@id]
     if daemon
       @config = daemon.frozen_config
       log "/server-control-status working with daemon ID #{daemon.config['id']}"
     else
+      @config = active_config
       log "/server-control-status Failed to get running daemon for config with id #{@id}"
     end
+    if @config.nil?
+      # E.g. daemon not yet started
+      status 400
+      return "Config is nil for ID: #{@id}"
+    end
     @game_port = daemon.server_running? ? daemon.active_game_port : @config['server_game_port'] rescue @config['server_game_port']
     @rcon_port = daemon.server_running? ? daemon.active_rcon_port : @config['server_rcon_port'] rescue @config['server_rcon_port']
     @query_port = daemon.server_running? ? daemon.active_query_port : @config['server_query_port'] rescue @config['server_query_port']