Initial ansible proof of concept

Dockerfile.ansible

@@ -0,0 +1,44 @@
+ARG REGISTRY
+ARG VERSION
+
+###############################################################################
+# final is our slim image with minimal layers and tools
+###############################################################################
+FROM ${REGISTRY}/ubi9/python-311:1-66 AS final
+ARG PIPX_VERSION=1.5.0 \
+    ANSIBLE_VERSION=9.5.1 \
+    AZURE_CLI_VERSION=2.60.0 \
+    ANSIBLE_AZCOLLECTION_VERSION=2.3.0
+
+# Have Ansible to print task timing information
+ENV ANSIBLE_CALLBACKS_ENABLED=profile_tasks
+USER root
+COPY ansible /ansible
+WORKDIR /ansible
+
+# Using pipx here because ansible and azure-cli have differing required core Azure modules
+# They each need a separate venv to avoid collisions
+RUN ${APP_ROOT}/bin/pip install "pipx==${PIPX_VERSION}" && \
+    ${APP_ROOT}/bin/pipx install "azure-cli==${AZURE_CLI_VERSION}" && \
+    ${APP_ROOT}/bin/pipx install "ansible==${ANSIBLE_VERSION}" --include-deps && \
+    ${APP_ROOT}/bin/pipx runpip ansible install -r "/ansible/ansible-requirements.txt" && \
+    ${HOME}/.local/bin/ansible-galaxy collection install "azure.azcollection==${ANSIBLE_AZCOLLECTION_VERSION}" && \
+    ${APP_ROOT}/bin/pipx runpip ansible install -r "${HOME}/.ansible/collections/ansible_collections/azure/azcollection/requirements-azure.txt" && \
+    ${APP_ROOT}/bin/pipx list && \
+    rm -rf ${HOME}/.ansible ${HOME}/.azure
+
+###############################################################################
+# linter takes the final image and injects ansible-lint. Ansible-lint needs
+# ansible itself and all ansible modules and python modules installed to correctly lint
+###############################################################################
+FROM final AS linter
+ARG ANSIBLE_LINT_VERSION=24.6.0
+RUN ${APP_ROOT}/bin/pipx inject ansible --include-apps "ansible-lint==${ANSIBLE_LINT_VERSION}" && \
+    ${HOME}/.local/bin/ansible-lint -v -c /ansible/.ansible_lint.yaml --project-dir /ansible --format sarif | tee /opt/app-root/src/sarif.txt
+
+###############################################################################
+# Re-FROM final so that it's the output container
+###############################################################################
+FROM final
+COPY --from=linter /opt/app-root/src/sarif.txt /opt/app-root/src/sarif.txt
+ENTRYPOINT ["/opt/app-root/src/.local/bin/ansible-playbook"]

Makefile

@@ -5,6 +5,7 @@ ARO_IMAGE_BASE = ${RP_IMAGE_ACR}.azurecr.io/aro
 E2E_FLAGS ?= -test.v --ginkgo.v --ginkgo.timeout 180m --ginkgo.flake-attempts=2 --ginkgo.junit-report=e2e-report.xml
 GO_FLAGS ?= -tags=containers_image_openpgp,exclude_graphdriver_btrfs,exclude_graphdriver_devicemapper
 NO_CACHE ?= true
+PODMAN_VOLUME_OVERLAY=$(shell if [[ $$(getenforce) == "Enforcing" ]]; then echo ":O"; else echo ""; fi 2>/dev/null)
 
 export GOFLAGS=$(GO_FLAGS)
 
@@ -285,4 +286,20 @@ vendor:
 install-go-tools:
 	go install ${GOTESTSUM}
 
-.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-rp ci-clean clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools
+ansible-image:
+	docker image exists aro-ansible:$(VERSION) || docker build . -f Dockerfile.ansible --build-arg REGISTRY=$(REGISTRY) --build-arg VERSION=$(VERSION) --no-cache=$(NO_CACHE) --tag aro-ansible:$(VERSION)
+
+LOCATION := eastus
+CLUSTERPREFIX := $(USER)
+CLUSTERPATTERN := basic
+CLEANUP := False
+SSH_CONFIG_DIR := $(HOME)/.ssh/
+SSH_KEY_BASENAME := id_rsa
+# Note: When running this from a pipeline, don't mount the ansible directory, use the one baked into the ansible image. This is for reproducibility purposes.
+cluster: ansible-image
+	docker run --rm -it -v $${AZURE_CONFIG_DIR:-~/.azure}:/opt/app-root/src/.azure$(PODMAN_VOLUME_OVERLAY) -v ./ansible:/ansible$(PODMAN_VOLUME_OVERLAY) -v $(SSH_CONFIG_DIR):/root/.ssh$(PODMAN_VOLUME_OVERLAY) aro-ansible:$(VERSION) -i hosts.yaml -l $(CLUSTERPATTERN) -e location=$(LOCATION) -e CLUSTERPREFIX=$(CLUSTERPREFIX) -e CLEANUP=$(CLEANUP) -e SSH_KEY_BASENAME=$(SSH_KEY_BASENAME) clusters.yaml
+
+lint-ansible:
+	cd ansible; ansible-lint -c .ansible_lint.yaml
+
+.PHONY: admin.kubeconfig aks.kubeconfig aro az ci-rp ci-clean clean client deploy dev-config.yaml discoverycache fix-macos-vendor generate image-aro-multistage image-fluentbit image-proxy init-contrib lint-go runlocal-rp proxy publish-image-aro-multistage publish-image-fluentbit publish-image-proxy secrets secrets-update e2e.test tunnel test-e2e test-go test-python vendor build-all validate-go unit-test-go coverage-go validate-fips install-go-tools cluster ansible-image cluster-dev lint-ansible

ansible/.ansible_lint.yaml

@@ -0,0 +1,10 @@
+profile: production
+exclude_paths: []
+use_default_rules: true
+skip_list:
+  - no-changed-when
+enable_list:
+  - args
+  - empty-string-compare
+  - no-same-owner
+  - name[prefix]

ansible/ansible-requirements.txt

@@ -0,0 +1 @@
+kubernetes==29.0.0

ansible/clusters.yaml

@@ -0,0 +1,19 @@
+---
+- name: Deploy simple clusters
+  hosts: simple_clusters
+  gather_facts: false
+  serial: "{{ max_simultaneous_clusters | default(1) }}"
+  environment:
+    AZURE_CORE_SURVEY_MESSAGE: "false"
+  roles:
+    - simple_cluster
+    - cleanup
+- name: Bring your own keys disk encryption
+  hosts: byok_clusters
+  gather_facts: false
+  serial: "{{ max_simultaneous_clusters | default(1) }}"
+  environment:
+    AZURE_CORE_SURVEY_MESSAGE: "false"
+  roles:
+    - byok_cluster
+    - cleanup

ansible/group_vars/all.yaml

@@ -0,0 +1 @@
+delegation: localhost

ansible/hosts.yaml

@@ -0,0 +1,93 @@
+---
+simple_clusters:
+  hosts:
+    basic:
+      # The simplest possible cluster
+      name: aro
+      resource_group: "{{ CLUSTERPREFIX }}-basic-{{ location }}"
+    enc:
+      # Basic cluster with encryption-at-host enabled
+      name: aro-414
+      resource_group: "{{ CLUSTERPREFIX }}-enc-{{ location }}-414"
+      version: 4.14.16
+      master_size: Standard_E8s_v5
+      master_encryption_at_host: true
+      worker_size: Standard_D4s_v5
+      worker_encryption_at_host: true
+    sre-shared-cluster:
+      name: sre-shared-cluster
+      resource_group: eastus
+      version: 4.12.25
+      cluster_resource_group: aro-a
+  vars:
+    network_prefix_cidr: 10.0.0.0/22
+    master_cidr: 10.0.0.0/23
+    master_size: Standard_D8s_v3
+    worker_cidr: 10.0.2.0/23
+    worker_size: Standard_D4s_v3
+    delegation: localhost
+  children:
+    udr_clusters:
+    private_clusters:
+
+udr_clusters:
+  hosts:
+    udr414:
+      name: aro-414
+      resource_group: "{{ CLUSTERPREFIX }}-udr-{{ location }}-414"
+      version: "4.14.16"
+      routes:
+        - name: Blackhole
+          address_prefix: 0.0.0.0/0
+          next_hop_type: none
+    udr_no_null414:
+      name: aro-414
+      resource_group: "{{ CLUSTERPREFIX }}-udrnonull-{{ location }}-414"
+      version: "4.14.16"
+      routes:
+        - name: The Interwebs
+          address_prefix: 0.0.0.0/0
+          next_hop_type: internet
+    udr413:
+      name: aro-413
+      resource_group: "{{ CLUSTERPREFIX }}-udr-{{ location }}-413"
+      version: "4.13.23"
+      routes:
+        - name: Blackhole
+          address_prefix: 0.0.0.0/0
+          next_hop_type: none
+  vars:
+    network_prefix_cidr: 10.0.0.0/22
+    master_cidr: 10.0.0.0/23
+    worker_cidr: 10.0.2.0/23
+    apiserver_visibility: Private
+    ingress_visibility: Private
+    outbound_type: UserDefinedRouting
+
+private_clusters:
+  hosts:
+    private:
+      # Private cluster
+      name: aro
+      resource_group: "{{ CLUSTERPREFIX }}-private-{{ location }}"
+      apiserver_visibility: Private
+      ingress_visibility: Private
+      network_prefix_cidr: 10.0.0.0/22
+      master_cidr: 10.0.0.0/23
+      master_size: Standard_D8s_v3
+      worker_cidr: 10.0.2.0/23
+      worker_size: Standard_D4s_v3
+
+byok_clusters:
+  hosts:
+    byok:
+      # Cluster with customer-managed disk encryption key
+      # https://learn.microsoft.com/en-us/azure/openshift/howto-byok
+      name: aro-414
+      resource_group: "{{ CLUSTERPREFIX }}-byok-{{ location }}-414"
+      version: 4.14.16
+      network_prefix_cidr: 10.0.0.0/22
+      master_cidr: 10.0.0.0/23
+      master_size: Standard_E8s_v5
+      worker_cidr: 10.0.2.0/23
+      worker_size: Standard_D4s_v5

ansible/roles/byok_cluster/tasks/main.yaml

@@ -0,0 +1,103 @@
+- name: Create resource group
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_resourcegroup.yaml
+- name: Create vnet and subnets
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_vnet.yaml
+
+- name: Generate keyvault name
+  ansible.builtin.set_fact:
+    keyvault_name: "byok-{{ lookup('password', '/dev/null chars=ascii_letters,digits') | to_uuid | replace('-', '') | truncate(24 - 5, end='') }}"
+- name: Debug keyvault_name
+  ansible.builtin.debug:
+    var: keyvault_name
+- name: Byok keyvault
+  delegate_to: localhost
+  register: byok_keyvault_status
+  azure.azcollection.azure_rm_keyvault:
+    resource_group: "{{ resource_group }}"
+    vault_name: "{{ keyvault_name }}"
+    location: "{{ location }}"
+    enable_purge_protection: true
+    vault_tenant: "{{ sub_info.tenant_id }}"
+    sku:
+      name: standard
+      family: "A"
+    access_policies:
+      - tenant_id: "{{ sub_info.tenant_id }}"
+        object_id: "{{ currentuser_info.id }}"
+        keys: ["encrypt", "decrypt", "wrapkey", "unwrapkey", "sign", "verify", "get", "list", "create", "update", "import", "delete", "backup",
+               "restore", "recover", "purge"]
+    tags:
+      createdby: "{{ currentuser_info.userPrincipalName }}"
+      createdwith: "ansible"
+      purge: "true"
+- name: Debug byok_keyvault_status
+  ansible.builtin.debug:
+    var: byok_keyvault_status
+- name: Get byok keyvault
+  delegate_to: localhost
+  register: byok_keyvault_info
+  azure.azcollection.azure_rm_keyvault_info:
+    resource_group: "{{ resource_group }}"
+    name: "{{ keyvault_name }}"
+- name: Debug byok_keyvault_info
+  ansible.builtin.debug:
+    var: byok_keyvault_info
+
+- name: Byok key
+  delegate_to: localhost
+  register: byok_keyvault_key_status
+  azure.azcollection.azure_rm_keyvaultkey:
+    key_name: "{{ name }}-key"
+    keyvault_uri: "{{ byok_keyvault_info.keyvaults[0].vault_uri }}"
+- name: Debug byok_keyvault_key_status
+  ansible.builtin.debug:
+    var: byok_keyvault_key_status
+- name: Get byok key
+  delegate_to: localhost
+  register: byok_keyvault_key_info
+  azure.azcollection.azure_rm_keyvaultkey_info:
+    vault_uri: "{{ byok_keyvault_info.keyvaults[0].vault_uri }}"
+    name: "{{ name }}-key"
+- name: Debug byok_keyvault_key_info
+  ansible.builtin.debug:
+    var: byok_keyvault_key_info
+
+- name: Byok disk encryption set
+  delegate_to: localhost
+  register: byok_des_status
+  azure.azcollection.azure_rm_diskencryptionset:
+    resource_group: "{{ resource_group }}"
+    name: "{{ name }}-des"
+    location: "{{ location }}"
+    source_vault: "{{ keyvault_name }}"
+    key_url: "{{ byok_keyvault_key_info['keys'][0].kid }}"
+- name: Debug byok_des_status
+  ansible.builtin.debug:
+    var: byok_des_status
+
+- name: Byok keyvault
+  delegate_to: localhost
+  register: byok_keyvault_status
+  azure.azcollection.azure_rm_keyvault:
+    resource_group: "{{ resource_group }}"
+    vault_name: "{{ keyvault_name }}"
+    location: "{{ location }}"
+    enable_purge_protection: true
+    vault_tenant: "{{ sub_info.tenant_id }}"
+    sku:
+      name: standard
+      family: "A"
+    access_policies:
+      - tenant_id: "{{ sub_info.tenant_id }}"
+        object_id: "{{ currentuser_info.id }}"
+        keys: ["encrypt", "decrypt", "wrapkey", "unwrapkey", "sign", "verify", "get", "list", "create", "update", "import", "delete", "backup",
+               "restore", "recover", "purge"]
+      - tenant_id: "{{ byok_des_status.state.identity.tenant_id }}"
+        object_id: "{{ byok_des_status.state.identity.principal_id }}"
+        keys: ["wrapkey", "unwrapkey", "get"]
+
+- name: Create aro cluster
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_aro_cluster.yaml

ansible/roles/cleanup/tasks/main.yaml

@@ -0,0 +1,7 @@
+---
+- name: Delete aro cluster
+  ansible.builtin.include_tasks:
+    file: ../tasks/delete_aro_cluster.yaml
+- name: Create resource group
+  ansible.builtin.include_tasks:
+    file: ../tasks/delete_resourcegroup.yaml

ansible/roles/pucm/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+- name: Az aro update
+  ansible.builtin.command:
+    argv: [az, aro, update, "--name={{ name }}", "--resource-group={{ resource_group }}", -o=yaml]
+  delegate_to: localhost
+- name: Wait for provisioningState to become Succeeded
+  azure.azcollection.azure_rm_openshiftmanagedcluster_info:
+    name: "{{ name }}"
+    resource_group: "{{ resource_group }}"
+  delegate_to: localhost
+  register: cluster_status
+  until: cluster_status.clusters.properties.provisioningState == 'Succeeded'
+  retries: 6
+  delay: 10

ansible/roles/refresh_credentials/tasks/main.yaml

@@ -0,0 +1,14 @@
+---
+- name: Az aro update refresh-credentials
+  ansible.builtin.command:
+    argv: [az, aro, update, "--name={{ name }}", "--resource-group={{ resource_group }}", --refresh-credentials, -o=yaml]
+  delegate_to: localhost
+- name: Wait for provisioningState to become Succeeded
+  azure.azcollection.azure_rm_openshiftmanagedcluster_info:
+    name: "{{ name }}"
+    resource_group: "{{ resource_group }}"
+  delegate_to: localhost
+  register: cluster_status
+  until: cluster_status.clusters.properties.provisioningState == 'Succeeded'
+  retries: 6
+  delay: 10

ansible/roles/simple_cluster/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Create resource group
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_resourcegroup.yaml
+- name: Create vnet and subnets
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_vnet.yaml
+- name: Create aro cluster
+  ansible.builtin.include_tasks:
+    file: ../tasks/create_aro_cluster.yaml