diff --git a/CHANGES/2575.bugfix b/CHANGES/2575.bugfix new file mode 100644 index 0000000000..75c0d3a623 --- /dev/null +++ b/CHANGES/2575.bugfix @@ -0,0 +1 @@ +Fix bug where namespace logos would fail to download with S3 backends. \ No newline at end of file diff --git a/dev/insights/proxy/main.go b/dev/insights/proxy/main.go index e514e2e5c1..fd1e82b9e5 100644 --- a/dev/insights/proxy/main.go +++ b/dev/insights/proxy/main.go @@ -26,13 +26,6 @@ var refreshTokens = map[string]string{ "abcdefghijklmnopqrstuvwxyz1234567894": "notifications_admin", } -var passwordMap = map[string]string{ - "jdoe": "redhat", - "iqe_normal_user": "redhat", - "org-admin": "redhat", - "notifications_admin": "redhat", -} - // Access tokens will be stored here, and they can be generated using the refresh // tokens listed above using the: // curl -X POST localhost:8080/auth/realms/redhat-external/protocol/openid-connect/token -d refresh_token=1234567890 @@ -116,13 +109,10 @@ func randomString(length int) string { func getAccessToken(rw http.ResponseWriter, req *http.Request) { req.ParseForm() - grant_type := req.FormValue("grant_type") refresh_token := req.FormValue("refresh_token") - username := req.FormValue("username") - password := req.FormValue("password") if accountID, ok := refreshTokens[refresh_token]; ok { - fmt.Printf("Creating refresh token for: %s", accountID) + fmt.Printf("xxx Creating refresh token for: %s", accountID) acces_token := randomString(32) accessTokens[acces_token] = accountID @@ -134,33 +124,12 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) { rw.WriteHeader(http.StatusAccepted) json.NewEncoder(rw).Encode(resp) - - } else if _, found := passwordMap[username]; found{ - - if grant_type == "password" && passwordMap[username] == password { - fmt.Printf("Creating refresh token for: %s", accountID) - - acces_token := randomString(32) - accessTokens[acces_token] = accountID - - rw.Header().Set("Content-Type", "application/json") - resp := map[string]string{ - "access_token": acces_token, - } - - rw.WriteHeader(http.StatusAccepted) - json.NewEncoder(rw).Encode(resp) - - } else { - rw.WriteHeader(http.StatusUnauthorized) - } - } else { rw.WriteHeader(http.StatusUnauthorized) } } -func userToIdentityHeader(account Account) string { +func userToIentityHeader(account Account) string { data, _ := json.Marshal(XRHItentity{ Entitlements: Entitlement{ @@ -187,13 +156,12 @@ func setRHIdentityHeader(req *http.Request) { fmt.Printf("Authenticating with basic auth: %s:%s\n", user, pass) if account, ok := accounts[user]; ok { - req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(account)) + req.Header.Set("X-RH-IDENTITY", userToIentityHeader(account)) } else { fmt.Printf("User not found: %s", user) } } else if strings.Contains(auth_header, "Bearer") { - reqToken := req.Header.Get("Authorization") splitToken := strings.Split(reqToken, "Bearer ") reqToken = splitToken[1] @@ -201,7 +169,7 @@ func setRHIdentityHeader(req *http.Request) { fmt.Printf("Authenticating with refresh token: %s\n", reqToken) if userID, ok := accessTokens[reqToken]; ok { - req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(accounts[userID])) + req.Header.Set("X-RH-IDENTITY", userToIentityHeader(accounts[userID])) } else { fmt.Printf("Token not found: %s", reqToken) } @@ -218,6 +186,22 @@ func getEnv(key string, fallback string) string { return fallback } +func createHTTPClient(url string) *http.Client { + + if strings.Contains(url, ".tar.gz") { + return &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + } + + //return &http.Client{} + //return &http.DefaultClient + return &http.Client{} +} + func main() { fmt.Println("Staring insights proxy.") @@ -241,15 +225,21 @@ func main() { // Handle the keycloak auth url if req.URL.Path == "/auth/realms/redhat-external/protocol/openid-connect/token" { - fmt.Println(req.URL.Path) getAccessToken(rw, req) - + fmt.Println("") return } // Set X-RH-IDENTITY header setRHIdentityHeader(req) + fmt.Printf("req.Host: %s\n", req.Host) + fmt.Printf("req.URL.Host: %s\n", req.URL.Host) + fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme) + fmt.Printf("req.URL.Path: %s\n", req.URL.Path) + fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host) + fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme) + // Rewrite the url on the incoming request and resend it req.Host = urlToProxyTo.Host req.URL.Host = urlToProxyTo.Host @@ -257,11 +247,27 @@ func main() { req.RequestURI = "" req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/") - fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI()) + // fixme ... + // change http://localhost:11651 to http://pminio:8000 + + fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI()) + + /* + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + */ + + client := createHTTPClient(req.URL.Path) + + //upstreamServerResponse, err := http.DefaultClient.Do(req) + upstreamServerResponse, err := client.Do(req) - // save the response from the origin server - upstreamServerResponse, err := http.DefaultClient.Do(req) if err != nil { + fmt.Println("error ...") + fmt.Println(err) rw.WriteHeader(http.StatusInternalServerError) _, _ = fmt.Fprint(rw, err) fmt.Println("") @@ -269,6 +275,19 @@ func main() { return } + // if it's a 302 redirect, write the new url into the response headers ... + headers := upstreamServerResponse.Header + for key, values := range headers { + for _, value := range values { + fmt.Printf("HEADER %s: %s\n", key, value) + } + } + + location := upstreamServerResponse.Header.Get("Location") + if location != "" { + rw.Header().Set("Location", location) + } + // replace any download urls that are found on the response so that they // get redirected through the proxy data, _ := ioutil.ReadAll(upstreamServerResponse.Body) @@ -278,8 +297,9 @@ func main() { rw.WriteHeader(upstreamServerResponse.StatusCode) rw.Write(modified) + fmt.Println("request complete") fmt.Println() }) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy)) -} +} \ No newline at end of file diff --git a/dev/oci_env_integration/actions/insights.py b/dev/oci_env_integration/actions/insights.py index 53ac8b5b6a..b5e59243ed 100644 --- a/dev/oci_env_integration/actions/insights.py +++ b/dev/oci_env_integration/actions/insights.py @@ -6,6 +6,9 @@ "env_file": "insights.compose.env", "run_tests": True, "db_restore": None, + + # The minio client can take a long time to install + "wait_before_tests": 120 } ] ) diff --git a/dev/oci_env_integration/oci_env_configs/insights.compose.env b/dev/oci_env_integration/oci_env_configs/insights.compose.env index 645d08447a..7a8d9a8ccf 100644 --- a/dev/oci_env_integration/oci_env_configs/insights.compose.env +++ b/dev/oci_env_integration/oci_env_configs/insights.compose.env @@ -1,4 +1,4 @@ -COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights +COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights:pminio COMPOSE_PROJECT_NAME=ci-insights DEV_SOURCE_PATH=galaxy_ng @@ -13,4 +13,8 @@ DJANGO_SUPERUSER_PASSWORD=admin PULP_GALAXY_API_PATH_PREFIX=/api/automation-hub/ API_PORT=55001 -INSIGHTS_PROXY_PORT=38080 \ No newline at end of file +INSIGHTS_PROXY_PORT=38080 + +S3_ENDPOINT_URL=http://localhost:11651 +MINIO_CONSOLE_PORT=7812 +MINIO_PORT=11651 \ No newline at end of file diff --git a/galaxy_ng/app/api/ui/views/auth.py b/galaxy_ng/app/api/ui/views/auth.py index 7d9fd3eb07..457cb37f1e 100644 --- a/galaxy_ng/app/api/ui/views/auth.py +++ b/galaxy_ng/app/api/ui/views/auth.py @@ -89,7 +89,11 @@ def _oidc_logout(self, request): log.warning(msg) return - extra_data = json.loads(social.extra_data) + # FIXME - workaround for custom json type from dynaconf + if isinstance(social.extra_data, dict): + extra_data = social.extra_data + else: + extra_data = json.loads(social.extra_data) access_token = extra_data['access_token'] refresh_token = extra_data['refresh_token'] diff --git a/galaxy_ng/app/dynaconf_hooks.py b/galaxy_ng/app/dynaconf_hooks.py index d031ef550b..09505227ba 100755 --- a/galaxy_ng/app/dynaconf_hooks.py +++ b/galaxy_ng/app/dynaconf_hooks.py @@ -101,7 +101,7 @@ def configure_keycloak(settings: Dynaconf) -> Dict[str, Any]: "SOCIAL_AUTH_LOGIN_REDIRECT_URL", default="/ui/" ) data["SOCIAL_AUTH_JSONFIELD_ENABLED"] = True - data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField" + # data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField" data["SOCIAL_AUTH_URL_NAMESPACE"] = "social" data["SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA"] = [ ("refresh_token", "refresh_token"), diff --git a/galaxy_ng/app/tasks/namespaces.py b/galaxy_ng/app/tasks/namespaces.py index 2518ac795e..aa093d830a 100644 --- a/galaxy_ng/app/tasks/namespaces.py +++ b/galaxy_ng/app/tasks/namespaces.py @@ -55,7 +55,12 @@ def _download_avatar(url): print("file is not an image") return - return Artifact.init_and_validate(tf) + # the artifact has to be saved before the file is closed, or s3transfer + # will throw an error. + artifact = Artifact.init_and_validate(tf) + artifact.save() + + return artifact def _create_pulp_namespace(galaxy_ns_pk, download_logo): @@ -99,8 +104,6 @@ def _create_pulp_namespace(galaxy_ns_pk, download_logo): else: with transaction.atomic(): metadata.save() - if avatar_artifact: - avatar_artifact.save() ContentArtifact.objects.create( artifact=avatar_artifact, content=metadata, diff --git a/galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py b/galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py index 729800fbf2..27981812f8 100644 --- a/galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py +++ b/galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py @@ -61,7 +61,7 @@ def ansible_galaxy( if token: f.write(f"token={token}\n") - command_string = f"ansible-galaxy -vvv {command} --server={server} --ignore-certs" + command_string = f"ansible-galaxy {command} -vvv --server={server} --ignore-certs" for x in range(0, retries + 1): try: diff --git a/profiles/insights/proxy/main.go b/profiles/insights/proxy/main.go index 2bf1116fd0..fd1e82b9e5 100644 --- a/profiles/insights/proxy/main.go +++ b/profiles/insights/proxy/main.go @@ -112,7 +112,7 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) { refresh_token := req.FormValue("refresh_token") if accountID, ok := refreshTokens[refresh_token]; ok { - fmt.Printf("Creating refresh token for: %s", accountID) + fmt.Printf("xxx Creating refresh token for: %s", accountID) acces_token := randomString(32) accessTokens[acces_token] = accountID @@ -186,6 +186,22 @@ func getEnv(key string, fallback string) string { return fallback } +func createHTTPClient(url string) *http.Client { + + if strings.Contains(url, ".tar.gz") { + return &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + + } + + //return &http.Client{} + //return &http.DefaultClient + return &http.Client{} +} + func main() { fmt.Println("Staring insights proxy.") @@ -217,6 +233,13 @@ func main() { // Set X-RH-IDENTITY header setRHIdentityHeader(req) + fmt.Printf("req.Host: %s\n", req.Host) + fmt.Printf("req.URL.Host: %s\n", req.URL.Host) + fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme) + fmt.Printf("req.URL.Path: %s\n", req.URL.Path) + fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host) + fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme) + // Rewrite the url on the incoming request and resend it req.Host = urlToProxyTo.Host req.URL.Host = urlToProxyTo.Host @@ -224,11 +247,27 @@ func main() { req.RequestURI = "" req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/") - fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI()) + // fixme ... + // change http://localhost:11651 to http://pminio:8000 + + fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI()) + + /* + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + */ + + client := createHTTPClient(req.URL.Path) + + //upstreamServerResponse, err := http.DefaultClient.Do(req) + upstreamServerResponse, err := client.Do(req) - // save the response from the origin server - upstreamServerResponse, err := http.DefaultClient.Do(req) if err != nil { + fmt.Println("error ...") + fmt.Println(err) rw.WriteHeader(http.StatusInternalServerError) _, _ = fmt.Fprint(rw, err) fmt.Println("") @@ -236,6 +275,19 @@ func main() { return } + // if it's a 302 redirect, write the new url into the response headers ... + headers := upstreamServerResponse.Header + for key, values := range headers { + for _, value := range values { + fmt.Printf("HEADER %s: %s\n", key, value) + } + } + + location := upstreamServerResponse.Header.Get("Location") + if location != "" { + rw.Header().Set("Location", location) + } + // replace any download urls that are found on the response so that they // get redirected through the proxy data, _ := ioutil.ReadAll(upstreamServerResponse.Body) @@ -245,8 +297,9 @@ func main() { rw.WriteHeader(upstreamServerResponse.StatusCode) rw.Write(modified) + fmt.Println("request complete") fmt.Println() }) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy)) -} +} \ No newline at end of file