Catch & retry more possible errors in the insights proxy (#1876)

No-Issue

Signed-off-by: James Tanner <[email protected]>

.github/workflows/ci_oci-env-integration.yml

@@ -37,6 +37,8 @@ jobs:
         run: |
           echo "OCI_ENV_PATH=${HOME}/work/galaxy_ng/oci_env" >> $GITHUB_ENV
           echo "COMPOSE_INTERACTIVE_NO_CLI=1" >> $GITHUB_ENV
+          echo "OCI_VERBOSE=1" >> $GITHUB_ENV
+          echo "GH_DUMP_LOGS=1" >> $GITHUB_ENV
 
       - name: Update apt
         run: sudo apt -y update

dev/oci_env_integration/actions/action_lib.py

@@ -48,6 +48,7 @@ class OCIEnvIntegrationTest:
     do_teardown = True
     flags = ""
     envs = {}
+    failed = None
 
     def __init__(self, envs):
         for env in envs:
@@ -59,18 +60,18 @@ def __init__(self, envs):
         self.do_teardown = os.environ.get("GH_TEARDOWN", "1") == "1"
         self.flags = os.environ.get("GH_FLAGS", "")
 
-        failed = False
+        self.failed = False
         try:
             self.set_up_env()
             self.run_test()
         except Exception as e:
             print(e)
-            failed = True
+            self.failed = True
         finally:
             self.dump_logs()
             self.teardown()
 
-        if failed:
+        if self.failed:
             exit(1)
 
     def set_up_env(self):
@@ -129,8 +130,9 @@ def run_test(self):
     def dump_logs(self):
         if not self.do_dump_logs:
             return
-        for env in self.envs:
-            self.exec_cmd(env, "compose logs")
+        if self.failed:
+            for env in self.envs:
+                self.exec_cmd(env, "compose logs")
 
     def teardown(self):
         if not self.do_teardown:

profiles/insights/proxy/main.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"math/rand"
@@ -223,6 +224,25 @@ func isConnectionResetByPeer(err error) bool {
 	return false
 }
 
+func isUseOfClosedNetworkConnection(err error) bool {
+	return strings.Contains(err.Error(), "use of closed network connection")
+}
+
+func isWriteBrokenPipe(err error) bool {
+	return strings.Contains(err.Error(), "write: broken pipe")
+}
+
+func isInvalidReadOnClosedBody(err error) bool {
+	return strings.Contains(err.Error(), "invalid Read on closed Body")
+}
+
+func isEOFerror(err error) bool {
+	if err == io.EOF {
+		return true
+	}
+	return false
+}
+
 func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*http.Response, error) {
 	for retry := 0; retry < maxRetries; retry++ {
 		resp, err := client.Do(req)
@@ -237,6 +257,26 @@ func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*
 				time.Sleep(1 * time.Second) // Wait before retrying
 				continue
 			}
+			if isUseOfClosedNetworkConnection(err) {
+				fmt.Printf("Retry attempt %d: use of closed network connection\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isWriteBrokenPipe(err) {
+				fmt.Printf("Retry attempt %d: write broken pipe\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isInvalidReadOnClosedBody(err) {
+				fmt.Printf("Retry attempt %d: invalid read on closed body\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
+			if isEOFerror(err) {
+				fmt.Printf("Retry attempt %d: EOF error\n", retry+1)
+				time.Sleep(1 * time.Second) // Wait before retrying
+				continue
+			}
 			return nil, err
 		}
 
@@ -247,7 +287,7 @@ func retryHTTPRequest(client *http.Client, req *http.Request, maxRetries int) (*
 }
 
 func main() {
-	fmt.Println("Staring insights proxy.")
+	fmt.Println("Starting insights proxy.")
 
 	// define origin server URL
 	urlToProxyTo, err := url.Parse(getEnv("UPSTREAM_URL", "http://localhost:5001"))
@@ -326,6 +366,8 @@ func main() {
 		data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
 		modified := downloadUrlReg.ReplaceAll(data, replacementURL)
 
+		fmt.Printf("MODIFIED DATA: %s\n", modified)
+
 		// Write the response
 		rw.WriteHeader(upstreamServerResponse.StatusCode)
 		rw.Write(modified)