chore(ci): added exponential back-off to gettool (kopia#3267)

tools/gettool/autodownload/autodownload.go

@@ -158,8 +158,66 @@ func unzip(dir string, r io.Reader, stripPathComponents int) error {
 	return nil
 }
 
-// Download downloads the provided.
-func Download(url, dir string, checksum map[string]string, stripPathComponents int) (err error) {
+// Download downloads the provided URL and extracts it to the provided directory, retrying
+// exponentionally until succeeded.
+func Download(url, dir string, checksum map[string]string, stripPathComponents int) error {
+	const (
+		// sleep durations 5, 10, 20, 40, 80, 160, 320
+		// total: 635 seconds, ~10 minutes
+		maxRetries       = 8
+		initialSleepTime = 5 * time.Second
+	)
+
+	nextSleepTime := initialSleepTime
+
+	for i := 0; i < maxRetries; i++ {
+		err := downloadInternal(url, dir, checksum, stripPathComponents)
+		if err == nil {
+			// success
+			return nil
+		}
+
+		// 404 is non-retryable
+		if errors.Is(err, errNotFound) {
+			return errors.Wrap(err, "non-retryable")
+		}
+
+		// invalid checksum is non-retryable
+		var ec InvalidChecksumError
+		if errors.As(err, &ec) {
+			// invalid checksum, do not retry.
+			return errors.Wrap(err, "non-retryable")
+		}
+
+		// all other errors are retryable
+		if i != maxRetries-1 {
+			log.Printf("Attempt #%v failed, sleeping for %v: %v", i, nextSleepTime, err)
+			time.Sleep(nextSleepTime)
+
+			nextSleepTime *= 2
+
+			if err := os.RemoveAll(dir); err != nil {
+				log.Printf("unable to remove %v: %v", dir, err)
+			}
+		}
+	}
+
+	return errors.Errorf("unable to download %v", url)
+}
+
+// InvalidChecksumError is returned by Download when the checksum of the downloaded file does not match the expected checksum.
+type InvalidChecksumError struct {
+	actual   string
+	expected string
+}
+
+func (e InvalidChecksumError) Error() string {
+	return fmt.Sprintf("invalid checksum: %v, wanted %v", e.actual, e.expected)
+}
+
+var errNotFound = errors.New("not found")
+
+func downloadInternal(url, dir string, checksum map[string]string, stripPathComponents int) (err error) {
 	resp, err := http.Get(url) //nolint:gosec,noctx
 	if err != nil {
 		return errors.Wrapf(err, "unable to get %q", url)
@@ -168,6 +226,10 @@ func Download(url, dir string, checksum map[string]string, stripPathComponents i
 	defer resp.Body.Close() //nolint:errcheck
 
 	if resp.StatusCode != http.StatusOK {
+		if resp.StatusCode == http.StatusNotFound {
+			return errNotFound
+		}
+
 		return errors.Errorf("invalid server response for %q: %v", url, resp.Status)
 	}
 
@@ -187,7 +249,7 @@ func Download(url, dir string, checksum map[string]string, stripPathComponents i
 		return errors.Errorf("missing checksum - calculated as %v", actualChecksum)
 
 	case checksum[url] != actualChecksum:
-		return errors.Errorf("invalid checksum: %v, wanted %v", actualChecksum, checksum[url])
+		return InvalidChecksumError{actualChecksum, checksum[url]}
 
 	default:
 		log.Printf("%v checksum ok", url)
@@ -210,7 +272,7 @@ func Download(url, dir string, checksum map[string]string, stripPathComponents i
 	case strings.HasSuffix(url, ".tar.gz"):
 		return errors.Wrap(untar(dir, r, stripPathComponents), "untar error")
 	case strings.HasSuffix(url, ".zip"):
-		return errors.Wrap(unzip(dir, r, stripPathComponents), "untar error")
+		return errors.Wrap(unzip(dir, r, stripPathComponents), "unzip error")
 	default:
 		return errors.Errorf("unsupported archive format")
 	}