Fix namespace logos on s3

CHANGES/2575.bugfix

@@ -0,0 +1 @@
+Fix bug where namespace logos would fail to download with S3 backends.

dev/insights/proxy/main.go

@@ -26,13 +26,6 @@ var refreshTokens = map[string]string{
 	"abcdefghijklmnopqrstuvwxyz1234567894": "notifications_admin",
 }
 
-var passwordMap = map[string]string{
-	"jdoe":                "redhat",
-	"iqe_normal_user":     "redhat",
-	"org-admin":           "redhat",
-	"notifications_admin": "redhat",
-}
-
 // Access tokens will be stored here, and they can be generated using the refresh
 // tokens listed above using the:
 // curl -X POST localhost:8080/auth/realms/redhat-external/protocol/openid-connect/token -d refresh_token=1234567890
@@ -116,13 +109,10 @@ func randomString(length int) string {
 
 func getAccessToken(rw http.ResponseWriter, req *http.Request) {
 	req.ParseForm()
-    grant_type := req.FormValue("grant_type")
 	refresh_token := req.FormValue("refresh_token")
-    username := req.FormValue("username")
-    password := req.FormValue("password")
 
 	if accountID, ok := refreshTokens[refresh_token]; ok {
-		fmt.Printf("Creating refresh token for: %s", accountID)
+		fmt.Printf("xxx Creating refresh token for: %s", accountID)
 
 		acces_token := randomString(32)
 		accessTokens[acces_token] = accountID
@@ -134,33 +124,12 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) {
 
 		rw.WriteHeader(http.StatusAccepted)
 		json.NewEncoder(rw).Encode(resp)
-
-    } else if _, found := passwordMap[username]; found{
-
-        if grant_type == "password" && passwordMap[username] == password {
-            fmt.Printf("Creating refresh token for: %s", accountID)
-
-            acces_token := randomString(32)
-            accessTokens[acces_token] = accountID
-
-            rw.Header().Set("Content-Type", "application/json")
-            resp := map[string]string{
-                "access_token": acces_token,
-            }
-
-            rw.WriteHeader(http.StatusAccepted)
-            json.NewEncoder(rw).Encode(resp)
-
-        } else {
-		    rw.WriteHeader(http.StatusUnauthorized)
-        }
-
 	} else {
 		rw.WriteHeader(http.StatusUnauthorized)
 	}
 }
 
-func userToIdentityHeader(account Account) string {
+func userToIentityHeader(account Account) string {
 
 	data, _ := json.Marshal(XRHItentity{
 		Entitlements: Entitlement{
@@ -187,21 +156,20 @@ func setRHIdentityHeader(req *http.Request) {
 			fmt.Printf("Authenticating with basic auth: %s:%s\n", user, pass)
 
 			if account, ok := accounts[user]; ok {
-				req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(account))
+				req.Header.Set("X-RH-IDENTITY", userToIentityHeader(account))
 			} else {
 				fmt.Printf("User not found: %s", user)
 			}
 
 		} else if strings.Contains(auth_header, "Bearer") {
-
 			reqToken := req.Header.Get("Authorization")
 			splitToken := strings.Split(reqToken, "Bearer ")
 			reqToken = splitToken[1]
 
 			fmt.Printf("Authenticating with refresh token: %s\n", reqToken)
 
 			if userID, ok := accessTokens[reqToken]; ok {
-				req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(accounts[userID]))
+				req.Header.Set("X-RH-IDENTITY", userToIentityHeader(accounts[userID]))
 			} else {
 				fmt.Printf("Token not found: %s", reqToken)
 			}
@@ -218,6 +186,22 @@ func getEnv(key string, fallback string) string {
 	return fallback
 }
 
+func createHTTPClient(url string) *http.Client {
+
+    if strings.Contains(url, ".tar.gz") {
+        return &http.Client{
+            CheckRedirect: func(req *http.Request, via []*http.Request) error {
+                return http.ErrUseLastResponse
+            },
+        }
+
+    }
+
+    //return &http.Client{}
+    //return &http.DefaultClient
+    return &http.Client{}
+}
+
 func main() {
 	fmt.Println("Staring insights proxy.")
 
@@ -241,34 +225,69 @@ func main() {
 
 		// Handle the keycloak auth url
 		if req.URL.Path == "/auth/realms/redhat-external/protocol/openid-connect/token" {
-            fmt.Println(req.URL.Path)
 			getAccessToken(rw, req)
-
+			fmt.Println("")
 			return
 		}
 
 		// Set X-RH-IDENTITY header
 		setRHIdentityHeader(req)
 
+        fmt.Printf("req.Host: %s\n", req.Host)
+        fmt.Printf("req.URL.Host: %s\n", req.URL.Host)
+        fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme)
+        fmt.Printf("req.URL.Path: %s\n", req.URL.Path)
+        fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host)
+        fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme)
+
 		// Rewrite the url on the incoming request and resend it
 		req.Host = urlToProxyTo.Host
 		req.URL.Host = urlToProxyTo.Host
 		req.URL.Scheme = urlToProxyTo.Scheme
 		req.RequestURI = ""
 		req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/")
 
-		fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
+        // fixme ...
+        // change http://localhost:11651 to http://pminio:8000
+
+        fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
+
+        /*
+        client := &http.Client{
+            CheckRedirect: func(req *http.Request, via []*http.Request) error {
+                return http.ErrUseLastResponse
+            },
+        }
+        */
+
+        client := createHTTPClient(req.URL.Path)
+
+		//upstreamServerResponse, err := http.DefaultClient.Do(req)
+		upstreamServerResponse, err := client.Do(req)
 
-		// save the response from the origin server
-		upstreamServerResponse, err := http.DefaultClient.Do(req)
 		if err != nil {
+            fmt.Println("error ...")
+            fmt.Println(err)
 			rw.WriteHeader(http.StatusInternalServerError)
 			_, _ = fmt.Fprint(rw, err)
 			fmt.Println("")
 
 			return
 		}
 
+        // if it's a 302 redirect, write the new url into the response headers ...
+        headers := upstreamServerResponse.Header
+        for key, values := range headers {
+            for _, value := range values {
+                fmt.Printf("HEADER %s: %s\n", key, value)
+            }
+        }
+
+        location := upstreamServerResponse.Header.Get("Location")
+        if location != "" {
+            rw.Header().Set("Location", location)
+        }
+
 		// replace any download urls that are found on the response so that they
 		// get redirected through the proxy
 		data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
@@ -278,8 +297,9 @@ func main() {
 		rw.WriteHeader(upstreamServerResponse.StatusCode)
 		rw.Write(modified)
 
+        fmt.Println("request complete")
 		fmt.Println()
 	})
 
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy))
-}
+}

dev/oci_env_integration/actions/insights.py

@@ -6,6 +6,9 @@
             "env_file": "insights.compose.env",
             "run_tests": True,
             "db_restore": None,
+
+            # The minio client can take a long time to install
+            "wait_before_tests": 120
         }
     ]
 )

dev/oci_env_integration/oci_env_configs/insights.compose.env

@@ -1,4 +1,4 @@
-COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights
+COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights:pminio
 COMPOSE_PROJECT_NAME=ci-insights
 
 DEV_SOURCE_PATH=galaxy_ng
@@ -13,4 +13,8 @@ DJANGO_SUPERUSER_PASSWORD=admin
 PULP_GALAXY_API_PATH_PREFIX=/api/automation-hub/
 
 API_PORT=55001
-INSIGHTS_PROXY_PORT=38080
+INSIGHTS_PROXY_PORT=38080
+
+S3_ENDPOINT_URL=http://localhost:11651
+MINIO_CONSOLE_PORT=7812
+MINIO_PORT=11651

galaxy_ng/app/api/ui/views/auth.py

@@ -89,7 +89,11 @@ def _oidc_logout(self, request):
             log.warning(msg)
             return
 
-        extra_data = json.loads(social.extra_data)
+        # FIXME - workaround for custom json type from dynaconf
+        if isinstance(social.extra_data, dict):
+            extra_data = social.extra_data
+        else:
+            extra_data = json.loads(social.extra_data)
 
         access_token = extra_data['access_token']
         refresh_token = extra_data['refresh_token']

galaxy_ng/app/dynaconf_hooks.py

@@ -101,7 +101,7 @@ def configure_keycloak(settings: Dynaconf) -> Dict[str, Any]:
             "SOCIAL_AUTH_LOGIN_REDIRECT_URL", default="/ui/"
         )
         data["SOCIAL_AUTH_JSONFIELD_ENABLED"] = True
-        data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField"
+        # data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField"
         data["SOCIAL_AUTH_URL_NAMESPACE"] = "social"
         data["SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA"] = [
             ("refresh_token", "refresh_token"),

galaxy_ng/app/tasks/namespaces.py

@@ -55,7 +55,12 @@ def _download_avatar(url):
             print("file is not an image")
             return
 
-    return Artifact.init_and_validate(tf)
+        # the artifact has to be saved before the file is closed, or s3transfer
+        # will throw an error.
+        artifact = Artifact.init_and_validate(tf)
+        artifact.save()
+
+        return artifact
 
 
 def _create_pulp_namespace(galaxy_ns_pk, download_logo):
@@ -99,8 +104,6 @@ def _create_pulp_namespace(galaxy_ns_pk, download_logo):
     else:
         with transaction.atomic():
             metadata.save()
-            if avatar_artifact:
-                avatar_artifact.save()
             ContentArtifact.objects.create(
                 artifact=avatar_artifact,
                 content=metadata,

galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py

@@ -61,7 +61,7 @@ def ansible_galaxy(
         if token:
             f.write(f"token={token}\n")
 
-    command_string = f"ansible-galaxy -vvv {command} --server={server} --ignore-certs"
+    command_string = f"ansible-galaxy {command} -vvv --server={server} --ignore-certs"
 
     for x in range(0, retries + 1):
         try:

profiles/insights/proxy/main.go

@@ -112,7 +112,7 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) {
 	refresh_token := req.FormValue("refresh_token")
 
 	if accountID, ok := refreshTokens[refresh_token]; ok {
-		fmt.Printf("Creating refresh token for: %s", accountID)
+		fmt.Printf("xxx Creating refresh token for: %s", accountID)
 
 		acces_token := randomString(32)
 		accessTokens[acces_token] = accountID
@@ -186,6 +186,22 @@ func getEnv(key string, fallback string) string {
 	return fallback
 }
 
+func createHTTPClient(url string) *http.Client {
+
+    if strings.Contains(url, ".tar.gz") {
+        return &http.Client{
+            CheckRedirect: func(req *http.Request, via []*http.Request) error {
+                return http.ErrUseLastResponse
+            },
+        }
+
+    }
+
+    //return &http.Client{}
+    //return &http.DefaultClient
+    return &http.Client{}
+}
+
 func main() {
 	fmt.Println("Staring insights proxy.")
 
@@ -217,25 +233,61 @@ func main() {
 		// Set X-RH-IDENTITY header
 		setRHIdentityHeader(req)
 
+        fmt.Printf("req.Host: %s\n", req.Host)
+        fmt.Printf("req.URL.Host: %s\n", req.URL.Host)
+        fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme)
+        fmt.Printf("req.URL.Path: %s\n", req.URL.Path)
+        fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host)
+        fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme)
+
 		// Rewrite the url on the incoming request and resend it
 		req.Host = urlToProxyTo.Host
 		req.URL.Host = urlToProxyTo.Host
 		req.URL.Scheme = urlToProxyTo.Scheme
 		req.RequestURI = ""
 		req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/")
 
-		fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
+        // fixme ...
+        // change http://localhost:11651 to http://pminio:8000
+
+        fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
+
+        /*
+        client := &http.Client{
+            CheckRedirect: func(req *http.Request, via []*http.Request) error {
+                return http.ErrUseLastResponse
+            },
+        }
+        */
+
+        client := createHTTPClient(req.URL.Path)
+
+		//upstreamServerResponse, err := http.DefaultClient.Do(req)
+		upstreamServerResponse, err := client.Do(req)
 
-		// save the response from the origin server
-		upstreamServerResponse, err := http.DefaultClient.Do(req)
 		if err != nil {
+            fmt.Println("error ...")
+            fmt.Println(err)
 			rw.WriteHeader(http.StatusInternalServerError)
 			_, _ = fmt.Fprint(rw, err)
 			fmt.Println("")
 
 			return
 		}
 
+        // if it's a 302 redirect, write the new url into the response headers ...
+        headers := upstreamServerResponse.Header
+        for key, values := range headers {
+            for _, value := range values {
+                fmt.Printf("HEADER %s: %s\n", key, value)
+            }
+        }
+
+        location := upstreamServerResponse.Header.Get("Location")
+        if location != "" {
+            rw.Header().Set("Location", location)
+        }
+
 		// replace any download urls that are found on the response so that they
 		// get redirected through the proxy
 		data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
@@ -245,8 +297,9 @@ func main() {
 		rw.WriteHeader(upstreamServerResponse.StatusCode)
 		rw.Write(modified)
 
+        fmt.Println("request complete")
 		fmt.Println()
 	})
 
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy))
-}
+}