From 406553252e6c5eee7ae0d2d2869e1838d101adfd Mon Sep 17 00:00:00 2001 From: andyuk1986 Date: Wed, 9 Aug 2023 09:13:41 +0200 Subject: [PATCH] Automation of Rosa Scaling Benchmark (#462) Relates to (#444) Co-authored-by: Anna Manukyan Co-authored-by: Alexander Schwartz --- .../actions/ec2-create-instances/action.yml | 24 ++ .../actions/ec2-delete-instances/action.yml | 15 ++ .../keycloak-create-dataset/action.yml | 17 ++ .github/workflows/rosa-scaling-benchmark.yml | 217 ++++++++++++++++++ ansible/aws_ec2.sh | 4 +- ansible/benchmark.sh | 4 +- ansible/env_rosa_benchmark.yml | 5 + ansible/roles/aws_ec2/defaults/main.yml | 4 +- .../roles/aws_ec2/tasks/create-resources.yml | 2 +- ansible/roles/aws_ec2/tasks/main.yml | 3 +- .../report/rosa-benchmark-key-results.adoc | 2 +- 11 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 .github/actions/ec2-create-instances/action.yml create mode 100644 .github/actions/ec2-delete-instances/action.yml create mode 100644 .github/workflows/rosa-scaling-benchmark.yml create mode 100644 ansible/env_rosa_benchmark.yml diff --git a/.github/actions/ec2-create-instances/action.yml b/.github/actions/ec2-create-instances/action.yml new file mode 100644 index 000000000..52ca53cd0 --- /dev/null +++ b/.github/actions/ec2-create-instances/action.yml @@ -0,0 +1,24 @@ +name: Create EC2 instances +description: Creates EC2 instances + +inputs: + region: + description: 'The AWS region used to host the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: aws-ec2-requirements-install + name: Install the required Ansible AWS collections. + shell: bash + run: | + ./aws_ec2.sh requirements + pipx inject ansible-core boto3 botocore + working-directory: ansible + + - id: create-ec2-instances + shell: bash + run: | + ./aws_ec2.sh create ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/ec2-delete-instances/action.yml b/.github/actions/ec2-delete-instances/action.yml new file mode 100644 index 000000000..75cdaffd0 --- /dev/null +++ b/.github/actions/ec2-delete-instances/action.yml @@ -0,0 +1,15 @@ +name: Delete EC2 instances +description: Deletes EC2 instances + +inputs: + region: + description: 'The AWS region used to delete the EC2 instances.' + required: true + +runs: + using: composite + steps: + - id: delete-load-runners + shell: bash + run: ./aws_ec2.sh delete ${{ inputs.region }} + working-directory: ansible diff --git a/.github/actions/keycloak-create-dataset/action.yml b/.github/actions/keycloak-create-dataset/action.yml index fe9bbca67..f6187b238 100644 --- a/.github/actions/keycloak-create-dataset/action.yml +++ b/.github/actions/keycloak-create-dataset/action.yml @@ -14,6 +14,15 @@ inputs: clients: description: 'Number of clients to create' default: '1' + createClientForSpecificRealm: + description: 'Create client for realm' + default: 'false' + clientsPerRealm: + description: 'Number of clients per Realm.' + default: '1' + realmNameForClients: + description: 'Name of the realm' + default: 'realm-0' maxWaitEntityCreation: description: 'Maximum number of seconds to wait for creation of entities' default: '300' @@ -32,3 +41,11 @@ runs: ./dataset-import.sh -a create-realms -r ${{ inputs.realms }} -c ${{ inputs.clients }} -u ${{ inputs.users }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset working-directory: dataset + + - id: create_clients_for_realm + if: ${{ inputs.createClientForSpecificRealm }} + shell: bash + run: | + ./dataset-import.sh -a create-clients -c ${{ inputs.clientsPerRealm }} -n ${{ inputs.realmNameForClients }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + ./dataset-import.sh -a status-completed -t ${{ inputs.maxWaitEntityCreation }} -l ${{ env.KEYCLOAK_URL }}/realms/master/dataset + working-directory: dataset diff --git a/.github/workflows/rosa-scaling-benchmark.yml b/.github/workflows/rosa-scaling-benchmark.yml new file mode 100644 index 000000000..324217518 --- /dev/null +++ b/.github/workflows/rosa-scaling-benchmark.yml @@ -0,0 +1,217 @@ +name: Rosa Cluster - Scaling Benchmark + +on: + workflow_dispatch: + inputs: + clusterName: + description: 'Name of the cluster' + type: string + default: 'gh-keycloak' + region: + description: 'Name of the region where EC2 instances should be installed' + type: string + default: 'eu-west-1' + numberOfEntitiesInRealm: + description: 'Number of entities for the scenario in DB' + type: number + default: 100000 + maxWaitEntityCreation: + description: 'Maximum number of seconds to wait for creation of entities' + type: number + default: 900 + numberOfUsersPerSecond: + description: 'User logins per second' + type: number + default: 200 + numberOfClientsPerSecond: + description: 'Client credential grants per second' + type: number + default: 1000 + measurement: + description: 'Measurement period (seconds)' + type: number + default: 600 + skipCreateDeployment: + description: 'Skip creating Keycloak deployment' + type: boolean + default: false + skipCreateDataset: + description: 'Skip creating dataset' + type: boolean + default: false + skipDeleteProject: + description: 'Skip deleting project' + type: boolean + default: false + +concurrency: cluster_${{ github.event.inputs.clusterName || format('gh-{0}', github.repository_owner) }} + +env: + PROJECT_PREFIX: runner- # same as default + PROJECT: runner-keycloak + KC_CPU_REQUESTS: 6 + KC_INSTANCES: 3 + KC_DISABLE_STICKY_SESSION: true + KC_MEMORY_REQUESTS_MB: 4000 + KC_MEMORY_LIMITS_MB: 4000 + KC_HEAP_MAX_MB: 2048 + KC_DB_POOL_INITIAL_SIZE: 30 + KC_DB_POOL_MAX_SIZE: 30 + KC_DB_POOL_MIN_SIZE: 30 + ANSIBLE_CUSTOM_VARS_ARG: '-e @env_rosa_benchmark.yml' + +jobs: + run: + name: Run Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup ROSA CLI + uses: ./.github/actions/rosa-cli-setup + with: + aws-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-default-region: ${{ vars.AWS_DEFAULT_REGION }} + rosa-token: ${{ secrets.ROSA_TOKEN }} + + - name: Login to OpenShift cluster + uses: ./.github/actions/oc-keycloak-login + with: + clusterName: ${{ inputs.clusterName || format('gh-{0}', github.repository_owner) }} + + - name: Set up JDK + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + cache: 'maven' + + - name: Cache Maven Wrapper + uses: actions/cache@v3 + with: + path: | + .mvn/wrapper/maven-wrapper.jar + key: ${{ runner.os }}-maven-wrapper-${{ hashFiles('**/maven-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-maven-wrapper- + + - name: Build with Maven + run: | + ./mvnw -B clean package -DskipTests -pl benchmark + tar xfvz benchmark/target/keycloak-benchmark-*.tar.gz + mv keycloak-benchmark-* keycloak-benchmark + + - name: Allow cluster to scale + if: ${{ !inputs.skipCreateDeployment }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 3 --max-replicas 10 scaling + + - name: Create Keycloak deployment + if: ${{ !inputs.skipCreateDeployment }} + uses: ./.github/actions/keycloak-create-deployment + with: + projectPrefix: ${{ env.PROJECT_PREFIX }} + disableStickySessions: ${{ !env.KC_DISABLE_STICKY_SESSION }} + replicas: ${{ env.KC_INSTANCES }} + podCpuRequests: ${{ env.KC_CPU_REQUESTS }} + podMemoryRequests: ${{ env.KC_MEMORY_REQUESTS_MB }} + podMemoryLimit: ${{ env.KC_MEMORY_LIMITS_MB }} + heapMaxSizeMB: ${{ env.KC_HEAP_MAX_MB }} + + - name: Get URLs + uses: ./.github/actions/get-keycloak-url + with: + project: ${{ env.PROJECT }} + + - name: Create Keycloak dataset with "${{ inputs.numberOfEntitiesInRealm }}" users and clients + if: ${{ !inputs.skipCreateDataset }} + uses: ./.github/actions/keycloak-create-dataset + with: + project: ${{ env.PROJECT }} + users: ${{ inputs.numberOfEntitiesInRealm }} + clients: 100 + clientsPerRealm: ${{ inputs.numberOfEntitiesInRealm }} + createClientForSpecificRealm: true + maxWaitEntityCreation: ${{ inputs.maxWaitEntityCreation }} + + - name: Create AWS EC2 instances + id: create_aws_ec2_instances + uses: ./.github/actions/ec2-create-instances + with: + region: ${{ inputs.region }} + + - name: Testing memory for creating sessions + id: kcb-authorization-code-1 + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.AuthorizationCode \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ + --ramp-up=20 \ + --logout-percentage=0 \ + --measurement=${{ inputs.measurement }} \ + --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \ + --log-http-on-failure + working-directory: ansible + + - name: Testing CPU usage for user logins + id: kcb-authorization-code-2 + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.AuthorizationCode \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=${{ inputs.numberOfUsersPerSecond }} \ + --ramp-up=20 \ + --logout-percentage=100 \ + --measurement=${{ inputs.measurement }} \ + --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \ + --log-http-on-failure + working-directory: ansible + + - name: Testing CPU usage for client credential grants + id: kcb-client-secret + run: | + ./benchmark.sh ${{ inputs.region }} \ + --scenario=keycloak.scenario.authentication.ClientSecret \ + --server-url=${{ env.KEYCLOAK_URL }} \ + --realm-name=realm-0 \ + --users-per-sec=${{ inputs.numberOfClientsPerSecond }} \ + --ramp-up=20 \ + --measurement=${{ inputs.measurement }} \ + --users-per-realm=${{ inputs.numberOfEntitiesInRealm }} \ + --log-http-on-failure + working-directory: ansible + + - name: Archive Gatling reports + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: gatling-results + path: ansible/files/benchmark/*/results + retention-days: 5 + + - name: Archive Summary + uses: actions/upload-artifact@v3 + with: + name: summary + path: ansible/files/benchmark/*/results/*/js/stats.json + retention-days: 5 + + - name: Delete EC2 instances + if: ${{ (success() || failure()) && steps.create_aws_ec2_instances.conclusion != 'skipped' }} + uses: ./.github/actions/ec2-delete-instances + with: + region: ${{ inputs.region }} + + - name: Delete Keycloak deployment + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + uses: ./.github/actions/keycloak-delete-deployment + with: + project: ${{ env.PROJECT }} + + - name: Scale down the cluster + if: ${{ (success() || failure()) && !inputs.skipDeleteProject }} + run: rosa edit machinepool -c ${{ inputs.clusterName }} --min-replicas 0 --max-replicas 0 scaling diff --git a/ansible/aws_ec2.sh b/ansible/aws_ec2.sh index cae8774e0..57dd8ea56 100755 --- a/ansible/aws_ec2.sh +++ b/ansible/aws_ec2.sh @@ -11,8 +11,8 @@ case $OPERATION in pip3 install --user boto3 botocore ;; create|delete|start|stop) - if [ -f "env.yml" ]; then CUSTOM_VARS_ARG="-e @env.yml"; fi - ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" $CUSTOM_VARS_ARG + if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi + ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" $ANSIBLE_CUSTOM_VARS_ARG "${@:3}" ;; *) echo "Invalid option!" diff --git a/ansible/benchmark.sh b/ansible/benchmark.sh index d44f4d76f..cf858f701 100755 --- a/ansible/benchmark.sh +++ b/ansible/benchmark.sh @@ -7,8 +7,8 @@ KCB_PARAMS=${@:2} CLUSTER_NAME=${CLUSTER_NAME:-"benchmark_$(whoami)"} -if [ -f "env.yml" ]; then CUSTOM_VARS_ARG="-e @env.yml"; fi +if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml benchmark.yml \ - $CUSTOM_VARS_ARG \ + $ANSIBLE_CUSTOM_VARS_ARG \ -e "kcb_params=\"${KCB_PARAMS}\"" diff --git a/ansible/env_rosa_benchmark.yml b/ansible/env_rosa_benchmark.yml new file mode 100644 index 000000000..f62fc6f1c --- /dev/null +++ b/ansible/env_rosa_benchmark.yml @@ -0,0 +1,5 @@ +# Setting use for ROSA default benchmark +cluster_size: 5 +kcb_zip: ../benchmark/target/keycloak-benchmark-0.10-SNAPSHOT.zip +# GitHub actions seem to use different IP addresses for a single runner, which would then be rejected +cidr_ip: '0.0.0.0/0' diff --git a/ansible/roles/aws_ec2/defaults/main.yml b/ansible/roles/aws_ec2/defaults/main.yml index e62eb6855..9639ad292 100644 --- a/ansible/roles/aws_ec2/defaults/main.yml +++ b/ansible/roles/aws_ec2/defaults/main.yml @@ -2,10 +2,12 @@ cluster_identifier: "{{ lookup('env', 'USER') }}" cluster_name: "benchmark_{{ cluster_identifier }}" cluster_size: 1 +cidr_ip: "{{ control_host_ip.stdout }}/32" + ami_name: RHEL-8.8.0_HVM-20230503-x86_64-54-Hourly2-GP2 instance_type: t3.micro instance_volume_size: 20 instance_device: /dev/sda1 -no_log_sensitive: yes \ No newline at end of file +no_log_sensitive: yes diff --git a/ansible/roles/aws_ec2/tasks/create-resources.yml b/ansible/roles/aws_ec2/tasks/create-resources.yml index cedd203ac..ace8bd99e 100644 --- a/ansible/roles/aws_ec2/tasks/create-resources.yml +++ b/ansible/roles/aws_ec2/tasks/create-resources.yml @@ -13,7 +13,7 @@ - proto: tcp from_port: 22 to_port: 22 - cidr_ip: '{{ control_host_ip.stdout }}/32' + cidr_ip: '{{cidr_ip}}' register: group no_log: "{{ no_log_sensitive }}" diff --git a/ansible/roles/aws_ec2/tasks/main.yml b/ansible/roles/aws_ec2/tasks/main.yml index 520506193..4937fe631 100644 --- a/ansible/roles/aws_ec2/tasks/main.yml +++ b/ansible/roles/aws_ec2/tasks/main.yml @@ -1,12 +1,13 @@ - debug: var=cluster_identifier - debug: var=region - debug: var=cluster_name +- debug: var=cidr_ip - include_tasks: create-resources.yml when: operation == "create" - include_tasks: manage-instances.yml when: operation == "start" or operation == "stop" - + - include_tasks: delete-resources.yml when: operation == "delete" diff --git a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc index 24925cfb3..5739b9d1d 100644 --- a/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc +++ b/doc/benchmark/modules/ROOT/pages/report/rosa-benchmark-key-results.adoc @@ -198,7 +198,7 @@ cd ../../ansible [source,bash,subs="+quotes"] ---- ./benchmark.sh eu-west-1 \ ---scenario=keycloak.scenario.authentication.AuthorizationCode \ +--scenario=keycloak.scenario.authentication.ClientSecret \ --server-url=${KEYCLOAK_URL} \ --realm-name=realm-0 \ --users-per-sec=**** \