Automation of Rosa Scaling Benchmark (#462)

Relates to (#444)

Co-authored-by: Anna Manukyan <[email protected]>
Co-authored-by: Alexander Schwartz <[email protected]>

.github/actions/ec2-create-instances/action.yml

@@ -0,0 +1,24 @@
+name: Create EC2 instances
+description: Creates EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to host the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: aws-ec2-requirements-install
+      name: Install the required Ansible AWS collections.
+      shell: bash
+      run: |
+        ./aws_ec2.sh requirements
+        pipx inject ansible-core boto3 botocore
+      working-directory: ansible
+
+    - id: create-ec2-instances
+      shell: bash
+      run: |
+        ./aws_ec2.sh create ${{ inputs.region }}
+      working-directory: ansible

.github/actions/ec2-delete-instances/action.yml

@@ -0,0 +1,15 @@
+name: Delete EC2 instances
+description: Deletes EC2 instances
+
+inputs:
+  region:
+    description: 'The AWS region used to delete the EC2 instances.'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - id: delete-load-runners
+      shell: bash
+      run: ./aws_ec2.sh delete ${{ inputs.region }}
+      working-directory: ansible

.github/actions/keycloak-create-dataset/action.yml

@@ -14,6 +14,15 @@ inputs:
   clients:
     description: 'Number of clients to create'
     default: '1'
+  createClientForSpecificRealm:
+    description: 'Create client for realm'
+    default: 'false'
+  clientsPerRealm:
+    description: 'Number of clients per Realm.'
+    default: '1'
+  realmNameForClients:
+    description: 'Name of the realm'
+    default: 'realm-0'
   maxWaitEntityCreation:
     description: 'Maximum number of seconds to wait for creation of entities'
     default: '300'
@@ -32,3 +41,11 @@ runs:
         ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
         ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
       working-directory: dataset
+
+    - id: create_clients_for_realm
+      if: ${{ inputs.createClientForSpecificRealm }}
+      shell: bash
+      run: |
+        ./dataset-import.sh -a create-clients -c ${{ inputs.clientsPerRealm }} -n ${{ inputs.realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+        ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset
+      working-directory: dataset

.github/workflows/rosa-scaling-benchmark.yml

@@ -0,0 +1,217 @@
+name: Rosa Cluster - Scaling Benchmark
+
+on:
+  workflow_dispatch:
+    inputs:
+      clusterName:
+        description: 'Name of the cluster'
+        type: string
+        default: 'gh-keycloak'
+      region:
+        description: 'Name of the region where EC2 instances should be installed'
+        type: string
+        default: 'eu-west-1'
+      numberOfEntitiesInRealm:
+        description: 'Number of entities for the scenario in DB'
+        type: number
+        default: 100000
+      maxWaitEntityCreation:
+        description: 'Maximum number of seconds to wait for creation of entities'
+        type: number
+        default: 900
+      numberOfUsersPerSecond:
+        description: 'User logins per second'
+        type: number
+        default: 200
+      numberOfClientsPerSecond:
+        description: 'Client credential grants per second'
+        type: number
+        default: 1000
+      measurement:
+        description: 'Measurement period (seconds)'
+        type: number
+        default: 600
+      skipCreateDeployment:
+        description: 'Skip creating Keycloak deployment'
+        type: boolean
+        default: false
+      skipCreateDataset:
+        description: 'Skip creating dataset'
+        type: boolean
+        default: false
+      skipDeleteProject:
+        description: 'Skip deleting project'
+        type: boolean
+        default: false
+
+concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+env:
+  PROJECT_PREFIX: runner- # same as default
+  PROJECT: runner-keycloak
+  KC_CPU_REQUESTS: 6
+  KC_INSTANCES: 3
+  KC_DISABLE_STICKY_SESSION: true
+  KC_MEMORY_REQUESTS_MB: 4000
+  KC_MEMORY_LIMITS_MB: 4000
+  KC_HEAP_MAX_MB: 2048
+  KC_DB_POOL_INITIAL_SIZE: 30
+  KC_DB_POOL_MAX_SIZE: 30
+  KC_DB_POOL_MIN_SIZE: 30
+  ANSIBLE_CUSTOM_VARS_ARG: '-e @env_rosa_benchmark.yml'
+
+jobs:
+  run:
+    name: Run Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup ROSA CLI
+        uses: ./.github/actions/rosa-cli-setup
+        with:
+          aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-default-region: ${{ vars.AWS_DEFAULT_REGION }}
+          rosa-token: ${{ secrets.ROSA_TOKEN }}
+
+      - name: Login to OpenShift cluster
+        uses: ./.github/actions/oc-keycloak-login
+        with:
+          clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }}
+
+      - name: Set up JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '17'
+          cache: 'maven'
+
+      - name: Cache Maven Wrapper
+        uses: actions/cache@v3
+        with:
+          path: |
+            .mvn/wrapper/maven-wrapper.jar
+          key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-wrapper-
+
+      - name: Build with Maven
+        run: |
+          ./mvnw -B clean package -DskipTests -pl benchmark
+          tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz
+          mv keycloak-benchmark-* keycloak-benchmark
+
+      - name: Allow cluster to scale
+        if: ${{ !inputs.skipCreateDeployment }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling
+
+      - name: Create Keycloak deployment
+        if: ${{ !inputs.skipCreateDeployment }}
+        uses: ./.github/actions/keycloak-create-deployment
+        with:
+          projectPrefix: ${{ env.PROJECT_PREFIX }}
+          disableStickySessions: ${{ !env.KC_DISABLE_STICKY_SESSION }}
+          replicas: ${{ env.KC_INSTANCES }}
+          podCpuRequests: ${{ env.KC_CPU_REQUESTS }}
+          podMemoryRequests: ${{ env.KC_MEMORY_REQUESTS_MB }}
+          podMemoryLimit: ${{ env.KC_MEMORY_LIMITS_MB }}
+          heapMaxSizeMB: ${{ env.KC_HEAP_MAX_MB }}
+
+      - name: Get URLs
+        uses: ./.github/actions/get-keycloak-url
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users and clients
+        if: ${{ !inputs.skipCreateDataset }}
+        uses: ./.github/actions/keycloak-create-dataset
+        with:
+          project: ${{ env.PROJECT }}
+          users: ${{ inputs.numberOfEntitiesInRealm }}
+          clients: 100
+          clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }}
+          createClientForSpecificRealm: true
+          maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }}
+
+      - name: Create AWS EC2 instances
+        id: create_aws_ec2_instances
+        uses: ./.github/actions/ec2-create-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Testing memory for creating sessions
+        id: kcb-authorization-code-1
+        run: |
+          ./benchmark.sh ${{ inputs.region }} \
+          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=0 \
+          --measurement=${{ inputs.measurement }} \
+          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Testing CPU usage for user logins
+        id: kcb-authorization-code-2
+        run: |
+          ./benchmark.sh ${{ inputs.region }}  \
+          --scenario=keycloak.scenario.authentication.AuthorizationCode \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \
+          --ramp-up=20 \
+          --logout-percentage=100 \
+          --measurement=${{ inputs.measurement }} \
+          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Testing CPU usage for client credential grants
+        id: kcb-client-secret
+        run: |
+          ./benchmark.sh ${{ inputs.region }} \
+          --scenario=keycloak.scenario.authentication.ClientSecret \
+          --server-url=${{ env.KEYCLOAK_URL }} \
+          --realm-name=realm-0 \
+          --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \
+          --ramp-up=20 \
+          --measurement=${{ inputs.measurement }} \
+          --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \
+          --log-http-on-failure
+        working-directory: ansible
+
+      - name: Archive Gatling reports
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: gatling-results
+          path: ansible/files/benchmark/*/results
+          retention-days: 5
+
+      - name: Archive Summary
+        uses: actions/upload-artifact@v3
+        with:
+          name: summary
+          path: ansible/files/benchmark/*/results/*/js/stats.json
+          retention-days: 5
+
+      - name: Delete EC2 instances
+        if: ${{ (success() || failure()) && steps.create_aws_ec2_instances.conclusion != 'skipped' }}
+        uses: ./.github/actions/ec2-delete-instances
+        with:
+          region: ${{ inputs.region }}
+
+      - name: Delete Keycloak deployment
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        uses: ./.github/actions/keycloak-delete-deployment
+        with:
+          project: ${{ env.PROJECT }}
+
+      - name: Scale down the cluster
+        if: ${{ (success() || failure()) && !inputs.skipDeleteProject }}
+        run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling

ansible/aws_ec2.sh

@@ -11,8 +11,8 @@ case $OPERATION in
     pip3 install --user boto3 botocore
   ;;
   create|delete|start|stop)
-    if [ -f "env.yml" ]; then CUSTOM_VARS_ARG="-e @env.yml"; fi
-    ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" $CUSTOM_VARS_ARG
+    if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi
+    ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" $ANSIBLE_CUSTOM_VARS_ARG "${@:3}"
   ;;
   *)
     echo "Invalid option!"

ansible/benchmark.sh

@@ -7,8 +7,8 @@ KCB_PARAMS=${@:2}
 
 CLUSTER_NAME=${CLUSTER_NAME:-"benchmark_$(whoami)"}
 
-if [ -f "env.yml" ]; then CUSTOM_VARS_ARG="-e @env.yml"; fi
+if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi
 
 ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml benchmark.yml \
-  $CUSTOM_VARS_ARG \
+  $ANSIBLE_CUSTOM_VARS_ARG \
   -e "kcb_params=\"${KCB_PARAMS}\""

ansible/env_rosa_benchmark.yml

@@ -0,0 +1,5 @@
+# Setting use for ROSA default benchmark
+cluster_size: 5
+kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip
+# GitHub actions seem to use different IP addresses for a single runner, which would then be rejected
+cidr_ip: '0.0.0.0/0'

ansible/roles/aws_ec2/defaults/main.yml

@@ -2,10 +2,12 @@ cluster_identifier: "{{ lookup('env', 'USER') }}"
 cluster_name: "benchmark_{{ cluster_identifier }}"
 cluster_size: 1
 
+cidr_ip: "{{ control_host_ip.stdout }}/32"
+
 ami_name: RHEL-8.8.0_HVM-20230503-x86_64-54-Hourly2-GP2
 
 instance_type: t3.micro
 instance_volume_size: 20
 instance_device: /dev/sda1
 
-no_log_sensitive: yes
+no_log_sensitive: yes

ansible/roles/aws_ec2/tasks/create-resources.yml

@@ -13,7 +13,7 @@
       - proto: tcp
         from_port: 22
         to_port: 22
-        cidr_ip: '{{ control_host_ip.stdout }}/32'
+        cidr_ip: '{{cidr_ip}}'
   register: group
   no_log: "{{ no_log_sensitive }}"
 

ansible/roles/aws_ec2/tasks/main.yml

@@ -1,12 +1,13 @@
 - debug: var=cluster_identifier
 - debug: var=region
 - debug: var=cluster_name
+- debug: var=cidr_ip
 
 - include_tasks: create-resources.yml
   when: operation == "create"
 
 - include_tasks: manage-instances.yml
   when: operation == "start" or operation == "stop"
-  
+
 - include_tasks: delete-resources.yml
   when: operation == "delete"

doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc

@@ -198,7 +198,7 @@ cd ../../ansible
 [source,bash,subs="+quotes"]
 ----
 ./benchmark.sh eu-west-1 \
---scenario=keycloak.scenario.authentication.AuthorizationCode \
+--scenario=keycloak.scenario.authentication.ClientSecret \
 --server-url=${KEYCLOAK_URL} \
 --realm-name=realm-0 \
 --users-per-sec=**<number of clients per second>** \