From c5bab1deec79746b6c9a3eb5c948a3ac983db652 Mon Sep 17 00:00:00 2001
From: Meysam Azad <MeysamAzad81@gmail.com>
Date: Sat, 23 Sep 2023 20:11:09 +0700
Subject: [PATCH] =?UTF-8?q?fix:=20retry=20failed=20ssl=20renewal=20with=20?=
 =?UTF-8?q?jittered=20backoff=20=F0=9F=90=9B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ansible/roles/letsencrypt/defaults/main.yml   |  3 ++
 .../letsencrypt/templates/renew-ssl-certs.sh  | 50 +++++++++++++++----
 2 files changed, 43 insertions(+), 10 deletions(-)

diff --git a/ansible/roles/letsencrypt/defaults/main.yml b/ansible/roles/letsencrypt/defaults/main.yml
index 5467b4ced..dfedc9d2f 100644
--- a/ansible/roles/letsencrypt/defaults/main.yml
+++ b/ansible/roles/letsencrypt/defaults/main.yml
@@ -1,3 +1,6 @@
 ---
 
 dummy_certs: false
+
+vars_ssl_renew_number_of_retries: 10
+vars_ssl_renew_retry_delay: 5
diff --git a/ansible/roles/letsencrypt/templates/renew-ssl-certs.sh b/ansible/roles/letsencrypt/templates/renew-ssl-certs.sh
index cdcfe09bd..1348eaa13 100644
--- a/ansible/roles/letsencrypt/templates/renew-ssl-certs.sh
+++ b/ansible/roles/letsencrypt/templates/renew-ssl-certs.sh
@@ -26,15 +26,45 @@ rm -rf "/etc/ssl/letsencrypt/accounts/localhost_14000/{{ email }}"
 server="https://acme-v02.api.letsencrypt.org/directory"
 {% endif %}
 
-lego --email "{{ email }}" \
-    --server "${server}" \
-    --accept-tos \
-    --path /etc/ssl/letsencrypt \
-    --http \
-    ${renew_kind} \
-    {% for domain in domains -%}
-    -d "{{ domain }}" \
-    {% endfor -%}
-    ${action}
+retries="{{ vars_ssl_renew_number_of_retries }}"
+wait_time="{{ vars_ssl_renew_retry_delay }}"
+
+lego_cmd="lego --email '{{ email }}' \
+        --server '${server}' \
+        --accept-tos \
+        --path /etc/ssl/letsencrypt \
+        --http \
+        ${renew_kind} \
+        {% for domain in domains -%}
+        -d '{{ domain }}' \
+        {% endfor -%}
+        ${action}"
+
+function run_with_retries {
+    command=$1
+    local i=0
+    while true; do
+        ${command}
+        exit_code=$?
+
+        if [ ${exit_code} -eq 0 ]; then
+            break
+        fi
+
+        if [ ${i} -ge ${retries} ]; then
+            exit ${exit_code}
+        fi
+
+        jitter=$(($RANDOM % 10))
+        wait_time_with_jitter=$((${wait_time} + ${jitter}))
+
+        echo "Command failed with exit code ${exit_code}. Retrying in ${wait_time_with_jitter} seconds..."
+        sleep ${wait_time_with_jitter}
+
+        i=$(($i + 1))
+    done
+}
+
+run_with_retries $lego_cmd
 
 sudo /etc/ssl/letsencrypt/after-renew