Improve uWSGI and consul-template interactions on reloading (#2640)

We fixed the `bind(): Address already in use [core/socket.c line 769]` error in LB
using `exit-on-reload = true` in uWSGI configuration. However, this prevents graceful
reloading.

I did some further investigation as noted in metabrainz/artwork-redirect#46
and have a better fix in mind now.

For the command field in `exec` block of consul template configuration, use
array format. This executes the command directly without spawning a shell wrapper
or setting a process group id. Both of which will interfere with forwarding
signals to the uWSGI process.

docker/services/api_compat/consul-template-api-compat.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi-api-compat.ini"
+    command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi-api-compat.ini"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/api_compat/uwsgi-api-compat.ini

@@ -12,5 +12,4 @@ log-x-forwarded-for=true
 disable-logging = true
 ; quit uwsgi if the python app fails to load
 need-app = true
-; when uwsgi gets a sighup, quit completely and let runit restart us
-exit-on-reload = true
+die-on-term = true

docker/services/apple_metadata_cache/consul-template-apple-metadata-cache.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -u -m listenbrainz.metadata_cache.apple.runner"
+    command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.metadata_cache.apple.runner"]
     splay = "60s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/labs_api/consul-template-labs-api.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi-labs-api.ini"
+    command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi-labs-api.ini"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/labs_api/uwsgi-labs-api.ini

@@ -12,7 +12,6 @@ log-x-forwarded-for=true
 disable-logging = true
 ; quit uwsgi if the python app fails to load
 need-app = true
-; when uwsgi gets a sighup, quit completely and let runit restart us
-exit-on-reload = true
 ; increase buffer size for requests that send a lot of mbids in query params
 buffer-size = 8192
+die-on-term = true

docker/services/mbid_mapping_writer/consul-template-mbid-mapping-writer.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -u -m listenbrainz.mbid_mapping_writer.mbid_mapping_writer"
+    command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.mbid_mapping_writer.mbid_mapping_writer"]
     splay = "60s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/spark_reader/consul-template-spark-reader.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -m listenbrainz.spark.spark_reader"
+    command = ["run-lb-command", "python3", "-m", "listenbrainz.spark.spark_reader"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/spotify_metadata_cache/consul-template-spotify-metadata-cache.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -u -m listenbrainz.metadata_cache.spotify.runner"
+    command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.metadata_cache.spotify.runner"]
     splay = "60s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/spotify_reader/consul-template-spotify-reader.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -m listenbrainz.spotify_updater.spotify_read_listens"
+    command = ["run-lb-command", "python3", "-m", "listenbrainz.spotify_updater.spotify_read_listens"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/timescale_writer/consul-template-timescale-writer.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 -u -m listenbrainz.timescale_writer.timescale_writer"
+    command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.timescale_writer.timescale_writer"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/uwsgi/consul-template-uwsgi.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi.ini"
+    command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi.ini"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

docker/services/uwsgi/uwsgi.ini

@@ -13,7 +13,6 @@ log-x-forwarded-for=true
 disable-logging = true
 ; quit uwsgi if the python app fails to load
 need-app = true
-; when uwsgi gets a sighup, quit completely and let runit restart us
-exit-on-reload = true
 ; increase buffer size for requests that send a lot of mbids in query params
 buffer-size = 8192
+die-on-term = true

docker/services/websockets/consul-template-websockets.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = "run-lb-command python3 manage.py run_websockets -h 0.0.0.0 -p 3031"
+    command = ["run-lb-command", "python3", "manage.py", "run_websockets", "-h", "0.0.0.0", "-p", "3031"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"