Skip to content

Commit

Permalink
Fix namespace logos on s3 (#1807)
Browse files Browse the repository at this point in the history 
* Fix namespace logos on S3

Issue: AAH-2575

* Fix deprecation warning

AssertionError: [DEPRECATION WARNING]: Setting verbosity before the arg sub command is
deprecated, set the verbosity after the sub command. This feature will be
removed from ansible-core in version 2.13. Deprecation warnings can be disabled
by setting deprecation_warnings=False in ansible.cfg.

No-Issue

* Updated proxy for tests
Issue: AAH-2575

* Copy the new proxy script to the oci profile path.

No-Issue

Signed-off-by: James Tanner <[email protected]>

* Add workaround for keycloak issues.

No-Issue

Signed-off-by: James Tanner <[email protected]>

* Add another workaround for keycloak.

No-Issue

Signed-off-by: James Tanner <[email protected]>

---------

Signed-off-by: James Tanner <[email protected]>
Co-authored-by: Brian McLaughlin <[email protected]>
Co-authored-by: James Tanner <[email protected]>
  • Loading branch information
@newswangerd @bmclaughlin @jctanner
3 people authored Sep 9, 2023
1 parent b078d82 commit 08c2985
Show file tree
Hide file tree
Showing 9 changed files with 143 additions and 55 deletions.
1 change: 1 addition & 0 deletions CHANGES/2575.bugfix
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix bug where namespace logos would fail to download with S3 backends.
104 changes: 62 additions & 42 deletions dev/insights/proxy/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,6 @@ var refreshTokens = map[string]string{
"abcdefghijklmnopqrstuvwxyz1234567894": "notifications_admin",
}

var passwordMap = map[string]string{
"jdoe": "redhat",
"iqe_normal_user": "redhat",
"org-admin": "redhat",
"notifications_admin": "redhat",
}

// Access tokens will be stored here, and they can be generated using the refresh
// tokens listed above using the:
// curl -X POST localhost:8080/auth/realms/redhat-external/protocol/openid-connect/token -d refresh_token=1234567890
Expand Down Expand Up @@ -116,13 +109,10 @@ func randomString(length int) string {

func getAccessToken(rw http.ResponseWriter, req *http.Request) {
req.ParseForm()
grant_type := req.FormValue("grant_type")
refresh_token := req.FormValue("refresh_token")
username := req.FormValue("username")
password := req.FormValue("password")

if accountID, ok := refreshTokens[refresh_token]; ok {
fmt.Printf("Creating refresh token for: %s", accountID)
fmt.Printf("xxx Creating refresh token for: %s", accountID)

acces_token := randomString(32)
accessTokens[acces_token] = accountID
Expand All @@ -134,33 +124,12 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) {

rw.WriteHeader(http.StatusAccepted)
json.NewEncoder(rw).Encode(resp)

} else if _, found := passwordMap[username]; found{

if grant_type == "password" && passwordMap[username] == password {
fmt.Printf("Creating refresh token for: %s", accountID)

acces_token := randomString(32)
accessTokens[acces_token] = accountID

rw.Header().Set("Content-Type", "application/json")
resp := map[string]string{
"access_token": acces_token,
}

rw.WriteHeader(http.StatusAccepted)
json.NewEncoder(rw).Encode(resp)

} else {
rw.WriteHeader(http.StatusUnauthorized)
}

} else {
rw.WriteHeader(http.StatusUnauthorized)
}
}

func userToIdentityHeader(account Account) string {
func userToIentityHeader(account Account) string {

data, _ := json.Marshal(XRHItentity{
Entitlements: Entitlement{
Expand All @@ -187,21 +156,20 @@ func setRHIdentityHeader(req *http.Request) {
fmt.Printf("Authenticating with basic auth: %s:%s\n", user, pass)

if account, ok := accounts[user]; ok {
req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(account))
req.Header.Set("X-RH-IDENTITY", userToIentityHeader(account))
} else {
fmt.Printf("User not found: %s", user)
}

} else if strings.Contains(auth_header, "Bearer") {

reqToken := req.Header.Get("Authorization")
splitToken := strings.Split(reqToken, "Bearer ")
reqToken = splitToken[1]

fmt.Printf("Authenticating with refresh token: %s\n", reqToken)

if userID, ok := accessTokens[reqToken]; ok {
req.Header.Set("X-RH-IDENTITY", userToIdentityHeader(accounts[userID]))
req.Header.Set("X-RH-IDENTITY", userToIentityHeader(accounts[userID]))
} else {
fmt.Printf("Token not found: %s", reqToken)
}
Expand All @@ -218,6 +186,22 @@ func getEnv(key string, fallback string) string {
return fallback
}

func createHTTPClient(url string) *http.Client {

if strings.Contains(url, ".tar.gz") {
return &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}

}

//return &http.Client{}
//return &http.DefaultClient
return &http.Client{}
}

func main() {
fmt.Println("Staring insights proxy.")

Expand All @@ -241,34 +225,69 @@ func main() {

// Handle the keycloak auth url
if req.URL.Path == "/auth/realms/redhat-external/protocol/openid-connect/token" {
fmt.Println(req.URL.Path)
getAccessToken(rw, req)

fmt.Println("")
return
}

// Set X-RH-IDENTITY header
setRHIdentityHeader(req)

fmt.Printf("req.Host: %s\n", req.Host)
fmt.Printf("req.URL.Host: %s\n", req.URL.Host)
fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme)
fmt.Printf("req.URL.Path: %s\n", req.URL.Path)
fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host)
fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme)

// Rewrite the url on the incoming request and resend it
req.Host = urlToProxyTo.Host
req.URL.Host = urlToProxyTo.Host
req.URL.Scheme = urlToProxyTo.Scheme
req.RequestURI = ""
req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/")

fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
// fixme ...
// change http://localhost:11651 to http://pminio:8000

fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())

/*
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
*/

client := createHTTPClient(req.URL.Path)

//upstreamServerResponse, err := http.DefaultClient.Do(req)
upstreamServerResponse, err := client.Do(req)

// save the response from the origin server
upstreamServerResponse, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println("error ...")
fmt.Println(err)
rw.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprint(rw, err)
fmt.Println("")

return
}

// if it's a 302 redirect, write the new url into the response headers ...
headers := upstreamServerResponse.Header
for key, values := range headers {
for _, value := range values {
fmt.Printf("HEADER %s: %s\n", key, value)
}
}

location := upstreamServerResponse.Header.Get("Location")
if location != "" {
rw.Header().Set("Location", location)
}

// replace any download urls that are found on the response so that they
// get redirected through the proxy
data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
Expand All @@ -278,8 +297,9 @@ func main() {
rw.WriteHeader(upstreamServerResponse.StatusCode)
rw.Write(modified)

fmt.Println("request complete")
fmt.Println()
})

log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy))
}
}
3 changes: 3 additions & 0 deletions dev/oci_env_integration/actions/insights.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
"env_file": "insights.compose.env",
"run_tests": True,
"db_restore": None,

# The minio client can take a long time to install
"wait_before_tests": 120
}
]
)
8 changes: 6 additions & 2 deletions dev/oci_env_integration/oci_env_configs/insights.compose.env
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights
COMPOSE_PROFILE=galaxy_ng/base:galaxy_ng/insights:pminio
COMPOSE_PROJECT_NAME=ci-insights

DEV_SOURCE_PATH=galaxy_ng
Expand All @@ -13,4 +13,8 @@ DJANGO_SUPERUSER_PASSWORD=admin
PULP_GALAXY_API_PATH_PREFIX=/api/automation-hub/

API_PORT=55001
INSIGHTS_PROXY_PORT=38080
INSIGHTS_PROXY_PORT=38080

S3_ENDPOINT_URL=http://localhost:11651
MINIO_CONSOLE_PORT=7812
MINIO_PORT=11651
6 changes: 5 additions & 1 deletion galaxy_ng/app/api/ui/views/auth.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,11 @@ def _oidc_logout(self, request):
log.warning(msg)
return

extra_data = json.loads(social.extra_data)
# FIXME - workaround for custom json type from dynaconf
if isinstance(social.extra_data, dict):
extra_data = social.extra_data
else:
extra_data = json.loads(social.extra_data)

access_token = extra_data['access_token']
refresh_token = extra_data['refresh_token']
Expand Down
2 changes: 1 addition & 1 deletion galaxy_ng/app/dynaconf_hooks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def configure_keycloak(settings: Dynaconf) -> Dict[str, Any]:
"SOCIAL_AUTH_LOGIN_REDIRECT_URL", default="/ui/"
)
data["SOCIAL_AUTH_JSONFIELD_ENABLED"] = True
data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField"
# data["SOCIAL_AUTH_JSONFIELD_CUSTOM"] = "django.db.models.JSONField"
data["SOCIAL_AUTH_URL_NAMESPACE"] = "social"
data["SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA"] = [
("refresh_token", "refresh_token"),
Expand Down
9 changes: 6 additions & 3 deletions galaxy_ng/app/tasks/namespaces.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,12 @@ def _download_avatar(url):
print("file is not an image")
return

return Artifact.init_and_validate(tf)
# the artifact has to be saved before the file is closed, or s3transfer
# will throw an error.
artifact = Artifact.init_and_validate(tf)
artifact.save()

return artifact


def _create_pulp_namespace(galaxy_ns_pk, download_logo):
Expand Down Expand Up @@ -99,8 +104,6 @@ def _create_pulp_namespace(galaxy_ns_pk, download_logo):
else:
with transaction.atomic():
metadata.save()
if avatar_artifact:
avatar_artifact.save()
ContentArtifact.objects.create(
artifact=avatar_artifact,
content=metadata,
Expand Down
2 changes: 1 addition & 1 deletion galaxy_ng/tests/integration/utils/client_ansible_galaxy_cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ def ansible_galaxy(
if token:
f.write(f"token={token}\n")

command_string = f"ansible-galaxy -vvv {command} --server={server} --ignore-certs"
command_string = f"ansible-galaxy {command} -vvv --server={server} --ignore-certs"

for x in range(0, retries + 1):
try:
Expand Down
63 changes: 58 additions & 5 deletions profiles/insights/proxy/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ func getAccessToken(rw http.ResponseWriter, req *http.Request) {
refresh_token := req.FormValue("refresh_token")

if accountID, ok := refreshTokens[refresh_token]; ok {
fmt.Printf("Creating refresh token for: %s", accountID)
fmt.Printf("xxx Creating refresh token for: %s", accountID)

acces_token := randomString(32)
accessTokens[acces_token] = accountID
Expand Down Expand Up @@ -186,6 +186,22 @@ func getEnv(key string, fallback string) string {
return fallback
}

func createHTTPClient(url string) *http.Client {

if strings.Contains(url, ".tar.gz") {
return &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}

}

//return &http.Client{}
//return &http.DefaultClient
return &http.Client{}
}

func main() {
fmt.Println("Staring insights proxy.")

Expand Down Expand Up @@ -217,25 +233,61 @@ func main() {
// Set X-RH-IDENTITY header
setRHIdentityHeader(req)

fmt.Printf("req.Host: %s\n", req.Host)
fmt.Printf("req.URL.Host: %s\n", req.URL.Host)
fmt.Printf("req.URL.Scheme: %s\n", req.URL.Scheme)
fmt.Printf("req.URL.Path: %s\n", req.URL.Path)
fmt.Printf("urlToProxyTo.Host: %s\n", urlToProxyTo.Host)
fmt.Printf("urlToProxyTo.Scheme: %s\n", urlToProxyTo.Scheme)

// Rewrite the url on the incoming request and resend it
req.Host = urlToProxyTo.Host
req.URL.Host = urlToProxyTo.Host
req.URL.Scheme = urlToProxyTo.Scheme
req.RequestURI = ""
req.URL.Path = strings.ReplaceAll(req.URL.Path, "//", "/")

fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())
// fixme ...
// change http://localhost:11651 to http://pminio:8000

fmt.Printf("Proxying request to: %s\n", req.URL.RequestURI())

/*
client := &http.Client{
CheckRedirect: func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
},
}
*/

client := createHTTPClient(req.URL.Path)

//upstreamServerResponse, err := http.DefaultClient.Do(req)
upstreamServerResponse, err := client.Do(req)

// save the response from the origin server
upstreamServerResponse, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println("error ...")
fmt.Println(err)
rw.WriteHeader(http.StatusInternalServerError)
_, _ = fmt.Fprint(rw, err)
fmt.Println("")

return
}

// if it's a 302 redirect, write the new url into the response headers ...
headers := upstreamServerResponse.Header
for key, values := range headers {
for _, value := range values {
fmt.Printf("HEADER %s: %s\n", key, value)
}
}

location := upstreamServerResponse.Header.Get("Location")
if location != "" {
rw.Header().Set("Location", location)
}

// replace any download urls that are found on the response so that they
// get redirected through the proxy
data, _ := ioutil.ReadAll(upstreamServerResponse.Body)
Expand All @@ -245,8 +297,9 @@ func main() {
rw.WriteHeader(upstreamServerResponse.StatusCode)
rw.Write(modified)

fmt.Println("request complete")
fmt.Println()
})

log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", proxyPort), reverseProxy))
}
}

0 comments on commit 08c2985

Please sign in to comment.