From 502189ba2b25c0885fb79b0aa29960d2d2cec883 Mon Sep 17 00:00:00 2001 From: Kartik Ohri Date: Mon, 27 Nov 2023 19:52:45 +0530 Subject: [PATCH] Improve uWSGI and consul-template interactions on reloading (#2640) We fixed the `bind(): Address already in use [core/socket.c line 769]` error in LB using `exit-on-reload = true` in uWSGI configuration. However, this prevents graceful reloading. I did some further investigation as noted in https://github.com/metabrainz/artwork-redirect/pull/46 and have a better fix in mind now. For the command field in `exec` block of consul template configuration, use array format. This executes the command directly without spawning a shell wrapper or setting a process group id. Both of which will interfere with forwarding signals to the uWSGI process. --- docker/services/api_compat/consul-template-api-compat.conf | 2 +- docker/services/api_compat/uwsgi-api-compat.ini | 3 +-- .../consul-template-apple-metadata-cache.conf | 2 +- docker/services/labs_api/consul-template-labs-api.conf | 2 +- docker/services/labs_api/uwsgi-labs-api.ini | 3 +-- .../consul-template-mbid-mapping-writer.conf | 2 +- docker/services/spark_reader/consul-template-spark-reader.conf | 2 +- .../consul-template-spotify-metadata-cache.conf | 2 +- .../spotify_reader/consul-template-spotify-reader.conf | 2 +- .../timescale_writer/consul-template-timescale-writer.conf | 2 +- docker/services/uwsgi/consul-template-uwsgi.conf | 2 +- docker/services/uwsgi/uwsgi.ini | 3 +-- docker/services/websockets/consul-template-websockets.conf | 2 +- 13 files changed, 13 insertions(+), 16 deletions(-) diff --git a/docker/services/api_compat/consul-template-api-compat.conf b/docker/services/api_compat/consul-template-api-compat.conf index 40b4594015..5529855a50 100644 --- a/docker/services/api_compat/consul-template-api-compat.conf +++ b/docker/services/api_compat/consul-template-api-compat.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi-api-compat.ini" + command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi-api-compat.ini"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/api_compat/uwsgi-api-compat.ini b/docker/services/api_compat/uwsgi-api-compat.ini index 533ab82b92..c5ba502ad1 100644 --- a/docker/services/api_compat/uwsgi-api-compat.ini +++ b/docker/services/api_compat/uwsgi-api-compat.ini @@ -12,5 +12,4 @@ log-x-forwarded-for=true disable-logging = true ; quit uwsgi if the python app fails to load need-app = true -; when uwsgi gets a sighup, quit completely and let runit restart us -exit-on-reload = true +die-on-term = true diff --git a/docker/services/apple_metadata_cache/consul-template-apple-metadata-cache.conf b/docker/services/apple_metadata_cache/consul-template-apple-metadata-cache.conf index a75423890b..38ae1ce64d 100644 --- a/docker/services/apple_metadata_cache/consul-template-apple-metadata-cache.conf +++ b/docker/services/apple_metadata_cache/consul-template-apple-metadata-cache.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -u -m listenbrainz.metadata_cache.apple.runner" + command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.metadata_cache.apple.runner"] splay = "60s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/labs_api/consul-template-labs-api.conf b/docker/services/labs_api/consul-template-labs-api.conf index 17a0aeed1f..0355a65d37 100644 --- a/docker/services/labs_api/consul-template-labs-api.conf +++ b/docker/services/labs_api/consul-template-labs-api.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi-labs-api.ini" + command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi-labs-api.ini"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/labs_api/uwsgi-labs-api.ini b/docker/services/labs_api/uwsgi-labs-api.ini index 8eb2a5ad03..db5c27ca87 100644 --- a/docker/services/labs_api/uwsgi-labs-api.ini +++ b/docker/services/labs_api/uwsgi-labs-api.ini @@ -12,7 +12,6 @@ log-x-forwarded-for=true disable-logging = true ; quit uwsgi if the python app fails to load need-app = true -; when uwsgi gets a sighup, quit completely and let runit restart us -exit-on-reload = true ; increase buffer size for requests that send a lot of mbids in query params buffer-size = 8192 +die-on-term = true diff --git a/docker/services/mbid_mapping_writer/consul-template-mbid-mapping-writer.conf b/docker/services/mbid_mapping_writer/consul-template-mbid-mapping-writer.conf index a442baefc2..df78cd83ca 100644 --- a/docker/services/mbid_mapping_writer/consul-template-mbid-mapping-writer.conf +++ b/docker/services/mbid_mapping_writer/consul-template-mbid-mapping-writer.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -u -m listenbrainz.mbid_mapping_writer.mbid_mapping_writer" + command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.mbid_mapping_writer.mbid_mapping_writer"] splay = "60s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/spark_reader/consul-template-spark-reader.conf b/docker/services/spark_reader/consul-template-spark-reader.conf index 696929e29a..dabd66fc1b 100644 --- a/docker/services/spark_reader/consul-template-spark-reader.conf +++ b/docker/services/spark_reader/consul-template-spark-reader.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -m listenbrainz.spark.spark_reader" + command = ["run-lb-command", "python3", "-m", "listenbrainz.spark.spark_reader"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/spotify_metadata_cache/consul-template-spotify-metadata-cache.conf b/docker/services/spotify_metadata_cache/consul-template-spotify-metadata-cache.conf index 7001ba125c..8666c210d0 100644 --- a/docker/services/spotify_metadata_cache/consul-template-spotify-metadata-cache.conf +++ b/docker/services/spotify_metadata_cache/consul-template-spotify-metadata-cache.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -u -m listenbrainz.metadata_cache.spotify.runner" + command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.metadata_cache.spotify.runner"] splay = "60s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/spotify_reader/consul-template-spotify-reader.conf b/docker/services/spotify_reader/consul-template-spotify-reader.conf index fa6bfc7cb1..d823467cfa 100644 --- a/docker/services/spotify_reader/consul-template-spotify-reader.conf +++ b/docker/services/spotify_reader/consul-template-spotify-reader.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -m listenbrainz.spotify_updater.spotify_read_listens" + command = ["run-lb-command", "python3", "-m", "listenbrainz.spotify_updater.spotify_read_listens"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/timescale_writer/consul-template-timescale-writer.conf b/docker/services/timescale_writer/consul-template-timescale-writer.conf index 1b3b2b5a52..a666580ceb 100644 --- a/docker/services/timescale_writer/consul-template-timescale-writer.conf +++ b/docker/services/timescale_writer/consul-template-timescale-writer.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 -u -m listenbrainz.timescale_writer.timescale_writer" + command = ["run-lb-command", "python3", "-u", "-m", "listenbrainz.timescale_writer.timescale_writer"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/uwsgi/consul-template-uwsgi.conf b/docker/services/uwsgi/consul-template-uwsgi.conf index 7971248404..a3d632f639 100644 --- a/docker/services/uwsgi/consul-template-uwsgi.conf +++ b/docker/services/uwsgi/consul-template-uwsgi.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command uwsgi --die-on-term /etc/uwsgi/uwsgi.ini" + command = ["run-lb-command", "uwsgi", "/etc/uwsgi/uwsgi.ini"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM" diff --git a/docker/services/uwsgi/uwsgi.ini b/docker/services/uwsgi/uwsgi.ini index 8e8d2db387..ce478a1180 100644 --- a/docker/services/uwsgi/uwsgi.ini +++ b/docker/services/uwsgi/uwsgi.ini @@ -13,7 +13,6 @@ log-x-forwarded-for=true disable-logging = true ; quit uwsgi if the python app fails to load need-app = true -; when uwsgi gets a sighup, quit completely and let runit restart us -exit-on-reload = true ; increase buffer size for requests that send a lot of mbids in query params buffer-size = 8192 +die-on-term = true diff --git a/docker/services/websockets/consul-template-websockets.conf b/docker/services/websockets/consul-template-websockets.conf index ba795bb097..94f19a01f0 100644 --- a/docker/services/websockets/consul-template-websockets.conf +++ b/docker/services/websockets/consul-template-websockets.conf @@ -4,7 +4,7 @@ template { } exec { - command = "run-lb-command python3 manage.py run_websockets -h 0.0.0.0 -p 3031" + command = ["run-lb-command", "python3", "manage.py", "run_websockets", "-h", "0.0.0.0", "-p", "3031"] splay = "5s" reload_signal = "SIGHUP" kill_signal = "SIGTERM"