Skip to content

Commit

Permalink
Make runlocal-rp with Container Image (#3593)
Browse files Browse the repository at this point in the history
Runlocal-RP is Containerized

- Modified Makefile to execute a local `podman run` for the RP on 127.0.0.1
- Local RPs now by default interact with Hive due to MacOS limitations
- Updated RP dev config to serve on all IPs due to MacOS limitations
- Doc updates
  • Loading branch information
shubhadapaithankar authored Aug 16, 2024
1 parent c74906d commit a387947
Show file tree
Hide file tree
Showing 5 changed files with 307 additions and 10 deletions.
70 changes: 66 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,72 @@ build-all:
aro: check-release generate
go build -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro

.PHONY: runlocal-rp
runlocal-rp:
go run -ldflags "-X github.com/Azure/ARO-RP/pkg/util/version.GitCommit=$(VERSION)" ./cmd/aro rp
# Target to create docker secrets
.PHONY: docker-secrets
docker-secrets: aks.kubeconfig
docker secret rm --ignore aks.kubeconfig
docker secret create aks.kubeconfig ./aks.kubeconfig

docker secret rm --ignore proxy-client.key
docker secret create proxy-client.key ./secrets/proxy-client.key

docker secret rm --ignore proxy-client.crt
docker secret create proxy-client.crt ./secrets/proxy-client.crt

docker secret rm --ignore proxy.crt
docker secret create proxy.crt ./secrets/proxy.crt

# Target to run the local RP
.PHONY: runlocal-rp
runlocal-rp: ci-rp docker-secrets
docker run --rm -p 127.0.0.1:8443:8443 \
--name aro-rp \
-w /app \
-e ARO_IMAGE \
-e RP_MODE="development" \
-e PROXY_HOSTNAME \
-e DOMAIN_NAME \
-e AZURE_RP_CLIENT_ID \
-e AZURE_FP_CLIENT_ID \
-e AZURE_SUBSCRIPTION_ID \
-e AZURE_TENANT_ID \
-e AZURE_RP_CLIENT_SECRET \
-e LOCATION \
-e RESOURCEGROUP \
-e AZURE_ARM_CLIENT_ID \
-e AZURE_FP_SERVICE_PRINCIPAL_ID \
-e AZURE_DBTOKEN_CLIENT_ID \
-e AZURE_PORTAL_CLIENT_ID \
-e AZURE_PORTAL_ACCESS_GROUP_IDS \
-e AZURE_CLIENT_ID \
-e AZURE_SERVICE_PRINCIPAL_ID \
-e AZURE_CLIENT_SECRET \
-e AZURE_GATEWAY_CLIENT_ID \
-e AZURE_GATEWAY_SERVICE_PRINCIPAL_ID \
-e AZURE_GATEWAY_CLIENT_SECRET \
-e DATABASE_NAME \
-e PULL_SECRET \
-e SECRET_SA_ACCOUNT_NAME \
-e DATABASE_ACCOUNT_NAME \
-e KEYVAULT_PREFIX \
-e ADMIN_OBJECT_ID \
-e PARENT_DOMAIN_NAME \
-e PARENT_DOMAIN_RESOURCEGROUP \
-e AZURE_ENVIRONMENT \
-e STORAGE_ACCOUNT_DOMAIN \
-e OIDC_STORAGE_ACCOUNT_NAME \
-e KUBECONFIG="/app/secrets/aks.kubeconfig" \
-e HIVE_KUBE_CONFIG_PATH="/app/secrets/aks.kubeconfig" \
-e ARO_CHECKOUT_PATH="/app" \
-e ARO_INSTALL_VIA_HIVE="true" \
-e ARO_ADOPT_BY_HIVE="true" \
--secret aks.kubeconfig,target=/app/secrets/aks.kubeconfig \
--secret proxy-client.key,target=/app/secrets/proxy-client.key \
--secret proxy-client.crt,target=/app/secrets/proxy-client.crt \
--secret proxy.crt,target=/app/secrets/proxy.crt \
$(RP_IMAGE_LOCAL) rp


.PHONY: az
az: pyenv
. pyenv/bin/activate && \
Expand Down Expand Up @@ -366,4 +428,4 @@ vendor:

.PHONY: install-go-tools
install-go-tools:
go install ${GOTESTSUM}
go install ${GOTESTSUM}
49 changes: 49 additions & 0 deletions docs/deploy-development-rp.md
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,55 @@ To run fake metrics socket:
```bash
go run ./hack/monitor
```
### Run the RP and create a Hive cluster

**Steps to perform on Mac**
1. Mount your local MacOS filesystem into the podman machine:
```bash
podman machine init --now --cpus=4 --memory=4096 -v $HOME:$HOME
```
2. Use the openvpn config file (which is now mounted inside the podman machine) to start the VPN connection:
```bash
podman machine ssh
sudo rpm-ostree install openvpn
sudo systemctl reboot
podman machine ssh
sudo openvpn --config /Users/<user_name>/go/src/github.com/Azure/ARO-RP/secrets/vpn-aks-westeurope.ovpn --daemon --writepid vpnpid
ps aux | grep openvpn
```
### Instructions for Modifying Environment File
**Update the env File**
- Open the `env` file.
- Update env file instructions: set `OPENSHIFT_VERSION`, update `INSTALLER_PULLSPEC` and `OCP_PULLSPEC`, mention quay.io for SHA256 hash.
- Update INSTALLER_PULLSPEC with the appropriate name and tag, typically matching the OpenShift version, e.g., `release-4.13.`(for more detail see the `env.example`)
* Source the environment file before creating the cluster using the `setup_resources.sh` script(Added the updated env in the PR)
```bash
cd /hack
./setup_resources.sh
```
* Once the cluster create verify connectivity with the ARO cluster:
- Download the admin kubeconfig file
```bash
az aro get-admin-kubeconfig --name <cluster_name> --resource-group v4-westeurope --file ~/.kube/aro-admin-kubeconfig
```
- Set the KUBECONFIG environment variable
```bash
export KUBECONFIG=~/.kube/aro-admin-kubeconfig
```
- Verify connectivity with the ARO cluster
```bash
kubectl get nodes
```
```bash
kubectl get nodes
NAME STATUS ROLES AGE VERSION
shpaitha-aro-cluster-4sp5c-master-0 Ready control-plane,master 39m v1.25.11+1485cc9
shpaitha-aro-cluster-4sp5c-master-1 Ready control-plane,master 39m v1.25.11+1485cc9
shpaitha-aro-cluster-4sp5c-master-2 Ready control-plane,master 39m v1.25.11+1485cc9
shpaitha-aro-cluster-4sp5c-worker-westeurope1-j9c76 Ready worker 29m v1.25.11+1485cc9
shpaitha-aro-cluster-4sp5c-worker-westeurope2-j9zrs Ready worker 27m v1.25.11+1485cc9
shpaitha-aro-cluster-4sp5c-worker-westeurope3-56tk7 Ready worker 28m v1.25.11+1485cc9
```

## Troubleshooting

Expand Down
10 changes: 7 additions & 3 deletions env.example
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
# use unique prefix for Azure resources when it is set, otherwise use your user's name
export AZURE_PREFIX="${AZURE_PREFIX:-$USER}"
export LOCATION=eastus
export ARO_IMAGE=arointsvc.azurecr.io/aro:latest
export LOCATION=westeurope
export NO_CACHE=false
export AZURE_EXTENSION_DEV_SOURCES="$(pwd)/python"

. secrets/env
export CLUSTER_RESOURCEGROUP="${USER}-v4-$LOCATION"
export CLUSTER_NAME="${USER}-aro-cluster"
export CLUSTER_VNET="${USER}-aro-vnet"
export ARO_IMAGE=arointsvc.azurecr.io/aro:latest

. secrets/env
184 changes: 184 additions & 0 deletions hack/setup_resources.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
#!/bin/bash

set -e

# Determine the base directory of the script
BASE_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)

# Construct the path to const.go using the base directory
CONST_GO_PATH="$BASE_DIR/pkg/util/version/const.go"

# Debugging: Print paths for verification
echo "Base directory: $BASE_DIR"
echo "Path to const.go: $CONST_GO_PATH"

# Check if const.go exists
if [ ! -f "$CONST_GO_PATH" ]; then
echo "Error: File $CONST_GO_PATH not found."
exit 1
fi

# Extract version and pullspec from const.go
OPENSHIFT_VERSION=$(awk -F'[(,)]' '/NewVersion/ {gsub(/ /, ""); print $2"."$3"."$4; exit}' "$CONST_GO_PATH")
OCP_PULLSPEC=$(awk -F'"' '/PullSpec:/ {print $2; exit}' "$CONST_GO_PATH")

# Set the INSTALLER_PULLSPEC
INSTALLER_PULLSPEC="arointsvc.azurecr.io/aro-installer:release-$(echo $OPENSHIFT_VERSION | sed 's/\.[^.]*$//')"
echo "Using OpenShift version: $OPENSHIFT_VERSION"
echo "Using OCP_PULLSPEC: $OCP_PULLSPEC"
echo "Using INSTALLER_PULLSPEC: $INSTALLER_PULLSPEC"

# Function to validate RP running
validate_rp_running() {
echo "########## Checking ARO RP Status ##########"
ELAPSED=0
while true; do
sleep 5
http_code=$(curl -k -s -o /dev/null -w '%{http_code}' https://localhost:8443/healthz/ready || true)
case $http_code in
"200")
echo "########## ✅ ARO RP Running ##########"
break
;;
*)
echo "Attempt $ELAPSED - local RP is NOT up. Code : $http_code, waiting"
sleep 2
# after 40 secs return exit 1 to not block ci
ELAPSED=$((ELAPSED + 1))
if [ $ELAPSED -eq 20 ]; then
exit 1
fi
;;
esac
done
}

# Ensure all env vars are set (LOCATION, CLUSTER_RESOURCEGROUP, CLUSTER_NAME)
ALL_SET="true"
if [ -z "${AZURE_SUBSCRIPTION_ID}" ]; then ALL_SET="false" && echo "AZURE_SUBSCRIPTION_ID is unset"; else echo "AZURE_SUBSCRIPTION_ID is set to '$AZURE_SUBSCRIPTION_ID'"; fi
if [ -z "${LOCATION}" ]; then ALL_SET="false" && echo "LOCATION is unset"; else echo "LOCATION is set to '$LOCATION'"; fi
if [ -z "${CLUSTER_RESOURCEGROUP}" ]; then ALL_SET="false" && echo "CLUSTER_RESOURCEGROUP is unset"; else echo "CLUSTER_RESOURCEGROUP is set to '$CLUSTER_RESOURCEGROUP'"; fi
if [ -z "${CLUSTER_NAME}" ]; then ALL_SET="false" && echo "CLUSTER_NAME is unset"; else echo "CLUSTER_NAME is set to '$CLUSTER_NAME'"; fi
if [ -z "${CLUSTER_VNET}" ]; then CLUSTER_VNET="aro-vnet2"; fi; echo "CLUSTER_VNET is ${CLUSTER_VNET}"
if [ -z "${CLUSTER_MASTER_SUBNET}" ]; then CLUSTER_MASTER_SUBNET="master-subnet"; fi; echo "CLUSTER_MASTER_SUBNET is ${CLUSTER_MASTER_SUBNET}"
if [ -z "${CLUSTER_WORKER_SUBNET}" ]; then CLUSTER_WORKER_SUBNET="worker-subnet"; fi; echo "CLUSTER_WORKER_SUBNET is ${CLUSTER_WORKER_SUBNET}"

if [[ "${ALL_SET}" != "true" ]]; then exit 1; fi

# Check Azure CLI version
echo "Checking Azure CLI version..."
az_version=$(az --version | grep 'azure-cli' | awk '{print $2}')
required_version="2.30.0"
if [ "$(printf '%s\n' "$required_version" "$az_version" | sort -V | head -n1)" = "$required_version" ]; then
echo "Azure CLI version is compatible"
else
echo "Azure CLI version must be $required_version or later. Please upgrade."
exit 1
fi

# Set the subscription
echo "Setting the subscription..."
az account set --subscription $AZURE_SUBSCRIPTION_ID

# Register the subscription directly
echo "Registering the subscription directly..."
curl -k -X PUT \
-H 'Content-Type: application/json' \
-d '{
"state": "Registered",
"properties": {
"tenantId": "'"$AZURE_TENANT_ID"'",
"registeredFeatures": [
{
"name": "Microsoft.RedHatOpenShift/RedHatEngineering",
"state": "Registered"
}
]
}
}' "https://localhost:8443/subscriptions/$AZURE_SUBSCRIPTION_ID?api-version=2.0"

# Validate RP running
validate_rp_running

# Function to add supported OpenShift version
add_openshift_version() {
local version=$1
local openshift_pullspec=$2
local installer_pullspec=$3

echo "Adding OpenShift version $version..."
curl -k -X PUT "https://localhost:8443/admin/versions" --header "Content-Type: application/json" -d '{
"properties": {
"version": "'"$version"'",
"enabled": true,
"openShiftPullspec": "'"$openshift_pullspec"'",
"installerPullspec": "'"$installer_pullspec"'"
}
}'
}

# Add the required OpenShift version
add_openshift_version "$OPENSHIFT_VERSION" "$OCP_PULLSPEC" "$INSTALLER_PULLSPEC"

# Delete the existing cluster if it exists
echo "Deleting the existing cluster if it exists..."
az aro delete --resource-group $CLUSTER_RESOURCEGROUP --name $CLUSTER_NAME --yes --no-wait || true

# Wait for the cluster deletion to complete
echo "Waiting for the cluster to be deleted..."
while az aro show --name $CLUSTER_NAME --resource-group $CLUSTER_RESOURCEGROUP &> /dev/null; do
echo "Cluster is still being deleted...waiting 30 seconds."
sleep 30
done

# Create resource group
echo "Creating resource group $CLUSTER_RESOURCEGROUP in $LOCATION..."
az group create --name $CLUSTER_RESOURCEGROUP --location $LOCATION

# Create virtual network
echo "Creating virtual network $CLUSTER_VNET in $CLUSTER_RESOURCEGROUP..."
az network vnet create --resource-group $CLUSTER_RESOURCEGROUP --name $CLUSTER_VNET --address-prefixes 10.0.0.0/22

# Delete any existing subnets and associated resources
echo "Deleting any existing master subnet resources..."
az network vnet subnet delete --resource-group $CLUSTER_RESOURCEGROUP --vnet-name $CLUSTER_VNET --name $CLUSTER_MASTER_SUBNET || true

echo "Deleting any existing worker subnet resources..."
az network vnet subnet delete --resource-group $CLUSTER_RESOURCEGROUP --vnet-name $CLUSTER_VNET --name $CLUSTER_WORKER_SUBNET || true

# Create master subnet
echo "Creating master subnet $CLUSTER_MASTER_SUBNET in $CLUSTER_VNET..."
az network vnet subnet create --resource-group $CLUSTER_RESOURCEGROUP --vnet-name $CLUSTER_VNET --name $CLUSTER_MASTER_SUBNET --address-prefixes 10.0.0.0/23 --service-endpoints Microsoft.ContainerRegistry

# Create worker subnet
echo "Creating worker subnet $CLUSTER_WORKER_SUBNET in $CLUSTER_VNET..."
az network vnet subnet create --resource-group $CLUSTER_RESOURCEGROUP --vnet-name $CLUSTER_VNET --name $CLUSTER_WORKER_SUBNET --address-prefixes 10.0.2.0/23 --service-endpoints Microsoft.ContainerRegistry

# Create cluster
echo "Creating cluster $CLUSTER_NAME in $CLUSTER_RESOURCEGROUP..."
az aro create --resource-group $CLUSTER_RESOURCEGROUP --name $CLUSTER_NAME --vnet $CLUSTER_VNET --master-subnet $CLUSTER_MASTER_SUBNET --worker-subnet $CLUSTER_WORKER_SUBNET --pull-secret "$PULL_SECRET" --location $LOCATION --version $OPENSHIFT_VERSION || {
echo "Cluster creation failed. Fetching deployment logs..."

# Fetch the deployment logs for further analysis
deployment_name=$(az deployment group list --resource-group $CLUSTER_RESOURCEGROUP --query '[0].name' -o tsv)
if [ -n "$deployment_name" ]; then
az deployment group show --name $deployment_name --resource-group $CLUSTER_RESOURCEGROUP
else
echo "No deployment found for resource group $CLUSTER_RESOURCEGROUP."
fi

exit 1
}

# Check for the existence of the cluster
if az aro show --name $CLUSTER_NAME --resource-group $CLUSTER_RESOURCEGROUP &> /dev/null; then
echo "Cluster creation successful."
else
echo "Cluster creation failed. Please check the logs for more details."
exit 1
fi

echo "To list cluster credentials, run:"
echo " az aro list-credentials --name $CLUSTER_NAME --resource-group $CLUSTER_RESOURCEGROUP"

echo "Note: Do not manually delete any resources. Let the script handle the deletions to avoid issues."
4 changes: 1 addition & 3 deletions pkg/env/dev.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,7 @@ func (d *dev) AROOperatorImage() string {
}

func (d *dev) Listen() (net.Listener, error) {
// in dev mode there is no authentication, so for safety we only listen on
// localhost
return net.Listen("tcp", "localhost:8443")
return net.Listen("tcp", ":8443")
}

// TODO: Delete FPAuthorizer once the replace from track1 to track2 is done.
Expand Down

0 comments on commit a387947

Please sign in to comment.