Merge branch 'master' into feature/guardrails-policy-m2

.github/CODEOWNERS

@@ -1 +1 @@
-* @jewzaam @bennerv @hawkowl @rogbas @petrkotas @jharrington22 @cblecker @facchettos @cadenmarchese @ulrichschlueter @s-amann @SudoBrendan @ellis-johnson
+* @jewzaam @bennerv @hawkowl @rogbas @petrkotas @jharrington22 @cblecker @facchettos @cadenmarchese @ulrichschlueter @s-amann @SudoBrendan @ellis-johnson @yjst2012 @anshulvermapatel

.github/workflows/codeql-analysis.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        language: ['go', 'javascript', 'python']
+        language: ['javascript', 'python']
         # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
         # Learn more:
         # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

.golangci.yml

@@ -5,6 +5,12 @@ run:
     - vendor
   skip-dirs-use-default: true
   modules-download-mode: vendor
+  build-tags:
+    - "aro"
+    - "containers_image_openpgp"
+    - "exclude_graphdriver_devicemapper"
+    - "exclude_graphdriver_btrfs"
+  go: "1.18"
 
 issues:
   exclude-rules:
@@ -60,8 +66,6 @@ linters-settings:
         alias: testdynamichelper
       - pkg: github.com/Azure/ARO-RP/test/util/log
         alias: testlog
-      - pkg: github.com/Azure/azure-sdk-for-go/services/graphrbac/1.6/graphrbac
-        alias: azgraphrbac
       - pkg: github.com/Azure/azure-sdk-for-go/services/keyvault/v7.0/keyvault
         alias: azkeyvault
       - pkg: github.com/Azure/azure-sdk-for-go/storage

.pipelines/ci.yml

@@ -1,4 +1,17 @@
 # Azure DevOps Pipeline running CI
+#
+# Note: This pipeline uses a secret variable "github_codeql_upload_token".
+#       This is a GitHub Personal Access Token (Classic) owned by mbarnes.
+#       It has no expiration and only has the "security_events" scope for
+#       the purpose of uploading CodeQL results.
+#
+#       However, for this secret to be available to pull requests from
+#       forked ARO-RP repositories, the pipeline option "Make secrets
+#       available to builds of forks" is enabled.
+#
+#       More information:
+#       https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/github#contributions-from-forks
+#
 trigger:
   branches:
     include:
@@ -31,6 +44,20 @@ variables:
 - template: vars.yml
 
 jobs:
+- job: Golang_CodeQL
+  pool:
+    name: 1es-aro-ci-pool
+  variables:
+    HOME: $(Agent.BuildDirectory)
+  steps:
+    - template: ./templates/template-checkout.yml
+    - template: ./templates/template-codeql.yml
+      parameters:
+        language: go
+        target: golang
+        github_token: $(github_codeql_upload_token)
+  timeoutInMinutes: 120
+
 - job: Python_Unit_Tests
   pool:
     name: 1es-aro-ci-pool

.pipelines/e2e.yml

@@ -8,12 +8,8 @@ resources:
     trigger: true
   containers:
   - container: container
-    image: registry.access.redhat.com/ubi8/toolbox:8.7
-    options: --user=0 --cap-add=NET_ADMIN --device /dev/net/tun --name vpn
-  - container: selenium
-    image: docker.io/selenium/standalone-edge:latest
-    options: --shm-size=2g
-
+    image: registry.access.redhat.com/ubi8/toolbox:8.8
+    options: --user=0 --privileged -v /dev/shm:/dev/shm --device /dev/net/tun --name vpn
 
 # Azure DevOps Pipeline running e2e tests
 variables:
@@ -23,17 +19,18 @@ variables:
 jobs:
 - job: E2E
   timeoutInMinutes: 180
+  variables:
+    ARO_PODMAN_SOCKET: "tcp://localhost:8888"
+    ARO_SELENIUM_HOSTNAME: "localhost"
   pool:
     name: 1es-aro-ci-pool
-  #services:
-  #  selenium: selenium
   steps:
   - template: ./templates/template-checkout.yml
 
   - script: |
       set -xe
       sudo rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-      sudo dnf install -y openvpn make
+      sudo dnf install -y openvpn make podman
     displayName: Setup (Container)
     target: container
 
@@ -63,8 +60,15 @@ jobs:
       . ./hack/e2e/run-rp-and-e2e.sh
 
       run_vpn
-      # run_portal
-      # validate_portal_running
+
+      run_podman
+      validate_podman_running
+
+      run_portal
+      validate_portal_running
+
+      run_selenium
+      validate_selenium_running
 
       run_rp
       validate_rp_running
@@ -82,25 +86,25 @@ jobs:
       hack/get-admin-kubeconfig.sh /subscriptions/$AZURE_SUBSCRIPTION_ID/resourceGroups/$CLUSTER/providers/Microsoft.RedHatOpenShift/openShiftClusters/$CLUSTER >admin.kubeconfig
       export KUBECONFIG=admin.kubeconfig
 
-      wget https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/$(OpenShiftVersion)/openshift-client-linux-$(OpenShiftVersion).tar.gz
+      wget -nv https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/$(OpenShiftVersion)/openshift-client-linux-$(OpenShiftVersion).tar.gz
       tar xf openshift-client-linux-$(OpenShiftVersion).tar.gz
       ./oc adm must-gather
       tar cf must-gather.tar.gz must-gather.local.*
     displayName: Collect must-gather
     condition: failed()
-    target: container
   - publish: must-gather.tar.gz
     artifact: must-gather
     displayName: Append must-gather to Pipeline
     condition: failed()
-    target: container
 
   - script: |
       export CI=true
       . ./hack/e2e/run-rp-and-e2e.sh
 
       delete_e2e_cluster
       kill_rp
+      kill_selenium
+      kill_podman
       kill_vpn
     displayName: Cleanup
     condition: always()

.pipelines/onebranch/pipeline.buildrp.official.yml

@@ -46,6 +46,8 @@ extends:
       suppression:
         suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress
         suppressionSet: default
+    git:
+      longpaths: true
 
     stages:
     - stage: Build_ARO

.pipelines/onebranch/pipeline.buildrp.pullrequest.yml

@@ -46,6 +46,8 @@ extends:
       suppression:
         suppressionFile: $(Build.SourcesDirectory)\.gdn\.gdnsuppress
         suppressionSet: default
+    git:
+      longpaths: true
 
     stages:
     - stage: Build_ARO

.pipelines/templates/template-codeql.yml

@@ -0,0 +1,36 @@
+parameters:
+- name: language
+  type: string
+  values:
+  # Based on "codeql resolve languages"
+  - cpp
+  - csharp
+  - csv
+  - go
+  - html
+  - java
+  - javascript
+  - properties
+  - python
+  - ruby
+  - xml
+- name: target
+  type: string
+  default: host
+- name: github_token
+  type: string
+
+# Based on "Use CodeQL in CI system" documentation:
+# https://docs.github.com/en/code-security/code-scanning/using-codeql-code-scanning-with-your-existing-ci-system/about-codeql-code-scanning-in-your-ci-system
+steps:
+- script: |
+    set -xe
+    sarif_file=codeql-results-${{ parameters.language }}.sarif
+    wget --quiet --output-document=- https://github.com/github/codeql-action/releases/latest/download/codeql-bundle-linux64.tar.gz | tar --extract --gunzip
+    ./codeql/codeql database create ./codeql-db --language=${{ parameters.language }}
+    ./codeql/codeql database analyze ./codeql-db --format=sarif-latest --sarif-category=no --output=${sarif_file}
+    ./codeql/codeql github upload-results --sarif=${sarif_file} --ref=$(Build.SourceBranch)
+  env:
+    GITHUB_TOKEN: ${{ parameters.github_token }}
+  displayName: ⚙️ CodeQL Analysis (${{ parameters.language }})
+  target: ${{ parameters.target }}

.sha256sum

@@ -2,4 +2,5 @@
 12d47b965d8a83b06ae3a44c632b624c57b129b5fbcc8c3f8de4fd80e2273f97  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2021-09-01-preview/redhatopenshift.json
 239c63228da1db172f298cd81d0c3cc0d52ecca907915efe61be98c42b6d8f1d  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2022-04-01/redhatopenshift.json
 1d167031baf0209fe8c46df9654585c64e8cc9a0c89555d7479c4ed6dc150251  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2022-09-04/redhatopenshift.json
-f8fc07701fe5624ed70436b287c531ee34e196db3e88d5d59263d51bf7f41b8a  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-04-01/redhatopenshift.json
+622404e8311c62f27fba778e30e760bb1901e5bd221b23de72f449cafbdf0c45  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/stable/2023-04-01/redhatopenshift.json
+4df3ebacaf35d77d09f5eab75fb9608241929b6ef8d00fb506455cd38e383640  swagger/redhatopenshift/resource-manager/Microsoft.RedHatOpenShift/preview/2023-07-01-preview/redhatopenshift.json

Dockerfile.gatekeeper

@@ -1,15 +1,14 @@
 ARG REGISTRY
 FROM ${REGISTRY}/ubi8/go-toolset:1.18.10-1 AS builder
 ARG GATEKEEPER_VERSION
-ENV DOWNLOAD_URL=https://github.com/open-policy-agent/gatekeeper/archive/v${GATEKEEPER_VERSION}.tar.gz
+ENV DOWNLOAD_URL=https://github.com/open-policy-agent/gatekeeper/archive/${GATEKEEPER_VERSION}.tar.gz
 
 ARG BUILDKIT_SBOM_SCAN_STAGE=true
 
 ENV GO111MODULE=on \
     GOOS=linux \
     GOPATH=/go/ \
-    GOARCH=amd64 \
-    CGO_ENABLED=0
+    GOARCH=amd64
 
 WORKDIR ${GOPATH}/src/github.com/open-policy-agent/gatekeeper
 USER root

Makefile

@@ -3,14 +3,17 @@ TAG ?= $(shell git describe --exact-match 2>/dev/null)
 COMMIT = $(shell git rev-parse --short=7 HEAD)$(shell [[ $$(git status --porcelain) = "" ]] || echo -dirty)
 ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro
 E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml
+GO_FLAGS ?= -tags=aro,containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper
+
+export GOFLAGS=$(GO_FLAGS)
 
 # fluentbit version must also be updated in RP code, see pkg/util/version/const.go
 MARINER_VERSION = 20230321
 FLUENTBIT_VERSION = 1.9.10
 FLUENTBIT_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/fluentbit:$(FLUENTBIT_VERSION)-cm$(MARINER_VERSION)
 AUTOREST_VERSION = 3.6.3
 AUTOREST_IMAGE = quay.io/openshift-on-azure/autorest:${AUTOREST_VERSION}
-GATEKEEPER_VERSION = 3.10.0
+GATEKEEPER_VERSION = v3.10.0
 GATEKEEPER_IMAGE ?= ${RP_IMAGE_ACR}.azurecr.io/gatekeeper:$(GATEKEEPER_VERSION)
 
 ifneq ($(shell uname -s),Darwin)
@@ -47,13 +50,13 @@ endif
 endif
 
 build-all:
-	go build -tags aro,containers_image_openpgp ./...
+	go build ./...
 
 aro: check-release generate
-	go build -tags aro,containers_image_openpgp,codec.safe -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro
+	go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro
 
 runlocal-rp:
-	go run -tags aro,containers_image_openpgp -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp
 
 az: pyenv
 	. pyenv/bin/activate && \
@@ -69,12 +72,12 @@ clean:
 	find -type d -name 'gomock_reflect_[0-9]*' -exec rm -rf {} \+ 2>/dev/null
 
 client: generate
-	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01
+	hack/build-client.sh "${AUTOREST_IMAGE}" 2020-04-30 2021-09-01-preview 2022-04-01 2022-09-04 2023-04-01 2023-07-01-preview
 
 # TODO: hard coding dev-config.yaml is clunky; it is also probably convenient to
 # override COMMIT.
 deploy:
-	go run -tags aro,containers_image_openpgp -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro deploy dev-config.yaml ${LOCATION}
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro deploy dev-config.yaml ${LOCATION}
 
 dev-config.yaml:
 	go run ./hack/gendevconfig >dev-config.yaml
@@ -149,7 +152,7 @@ proxy:
 	CGO_ENABLED=0 go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./hack/proxy
 
 run-portal:
-	go run -tags aro,containers_image_openpgp -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal
+	go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro portal
 
 build-portal:
 	cd portal/v1 && npm install && npm run build && cd ../v2 && npm install && npm run build
@@ -202,7 +205,7 @@ validate-go:
 	go test -tags e2e -run ^$$ ./test/e2e/...
 
 validate-go-action:
-	go run ./hack/licenses -validate -ignored-go vendor,pkg/client,.git -ignored-python python/client,vendor,.git
+	go run ./hack/licenses -validate -ignored-go vendor,pkg/client,.git -ignored-python python/client,python/az/aro/azext_aro/aaz,vendor,.git
 	go run ./hack/validate-imports cmd hack pkg test
 	@[ -z "$$(ls pkg/util/*.go 2>/dev/null)" ] || (echo error: go files are not allowed in pkg/util, use a subpackage; exit 1)
 	@[ -z "$$(find -name "*:*")" ] || (echo error: filenames with colons are not allowed on Windows, please rename; exit 1)
@@ -212,10 +215,10 @@ validate-fips:
 	hack/fips/validate-fips.sh
 
 unit-test-go:
-	go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -tags=aro,containers_image_openpgp -coverprofile=cover.out ./...
+	go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -coverprofile=cover.out ./...
 
 unit-test-go-coverpkg:
-	go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -tags=aro,containers_image_openpgp -coverpkg=./... -coverprofile=cover_coverpkg.out ./...
+	go run gotest.tools/[email protected] --format pkgname --junitfile report.xml -- -coverpkg=./... -coverprofile=cover_coverpkg.out ./...
 
 lint-go:
 	hack/lint-go.sh

cmd/aro/const.go

@@ -0,0 +1,11 @@
+package main
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+const (
+	DatabaseName        = "DATABASE_NAME"
+	DatabaseAccountName = "DATABASE_ACCOUNT_NAME"
+	KeyVaultPrefix      = "KEYVAULT_PREFIX"
+	DBTokenUrl          = "DBTOKEN_URL"
+)