From cd718a4271dab58975874b96a247c94e40a63e0d Mon Sep 17 00:00:00 2001
From: Sebastian Nagel <sebastian@commoncrawl.org>
Date: Thu, 1 Aug 2024 18:15:48 +0200
Subject: [PATCH] fix: retry downloads of webgraphs, fixes #19

---
 .../webgraph_ranking/graph_explore_download_webgraph.sh   | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/script/webgraph_ranking/graph_explore_download_webgraph.sh b/src/script/webgraph_ranking/graph_explore_download_webgraph.sh
index fdc11f5..992032a 100755
--- a/src/script/webgraph_ranking/graph_explore_download_webgraph.sh
+++ b/src/script/webgraph_ranking/graph_explore_download_webgraph.sh
@@ -66,14 +66,16 @@ function download_file() {
 
     if $USING_CURL; then
 
-        curl --silent --remote-time -o "$FILE" --time-cond "$FILE" --continue-at - "$URL"
+        curl --silent --remote-time -o "$FILE" --time-cond "$FILE" --continue-at - \
+             --retry 1000 --retry-all-errors --retry-delay 1 "$URL"
 
     elif $USING_WGET; then
 
         if [ "$(dirname "$FILE")" == "." ]; then
-            wget --continue --timestamping "$URL"
+            wget --continue --timestamping --tries=0 --retry-on-http-error=503 --waitretry=1 "$URL"
         else
-            wget --continue --timestamping --directory-prefix="$(dirname "$FILE")" "$URL"
+            wget --continue --timestamping --directory-prefix="$(dirname "$FILE")" \
+                 --tries=0 --retry-on-http-error=503 --waitretry=1 "$URL"
         fi
 
     fi