WIP Active/Active Failover test

keycloak · May 31, 2024 · f19a01b · f19a01b
1 parent 241a6c7
commit f19a01b
Show file tree

Hide file tree

Showing 6 changed files with 137 additions and 8 deletions.
diff --git a/provision/aws/global-accelerator/accelerator_multi_az_create.sh b/provision/aws/global-accelerator/accelerator_multi_az_create.sh
@@ -29,6 +29,7 @@ function createLoadBalancer() {
   export CLUSTER_NAME=$1
   SVC_NAME=$2
   NAMESPACE=$3
+  ACCELERATOR_NAME=$4
 
   bash ${SCRIPT_DIR}/../rosa_oc_login.sh > /dev/null
   oc create namespace ${NAMESPACE}  > /dev/null || true
@@ -38,7 +39,7 @@ function createLoadBalancer() {
   metadata:
     name: ${SVC_NAME}
     annotations:
-      service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: site=${CLUSTER_NAME},namespace=${NAMESPACE}
+      service.beta.kubernetes.io/aws-load-balancer-additional-resource-tags: accelerator=${ACCELERATOR_NAME},site=${CLUSTER_NAME},namespace=${NAMESPACE}
       service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
       service.beta.kubernetes.io/aws-load-balancer-healthcheck-path: "/lb-check"
       service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol: "https"
@@ -71,8 +72,8 @@ if [[ "${CLUSTER_1_REGION}" != "${CLUSTER_2_REGION}" ]]; then
   exit 1
 fi
 
-createLoadBalancer ${CLUSTER_1} ${ACCELERATOR_LB_NAME} ${KEYCLOAK_NAMESPACE}
-createLoadBalancer ${CLUSTER_2} ${ACCELERATOR_LB_NAME} ${KEYCLOAK_NAMESPACE}
+createLoadBalancer ${CLUSTER_1} ${ACCELERATOR_LB_NAME} ${KEYCLOAK_NAMESPACE} ${ACCELERATOR_NAME}
+createLoadBalancer ${CLUSTER_2} ${ACCELERATOR_LB_NAME} ${KEYCLOAK_NAMESPACE} ${ACCELERATOR_NAME}
 
 TOFU_CMD="tofu apply -auto-approve \
   -var aws_region=${CLUSTER_1_REGION} \

diff --git a/provision/rosa-cross-dc/Taskfile.yaml b/provision/rosa-cross-dc/Taskfile.yaml
@@ -682,6 +682,11 @@ tasks:
         - ACCELERATOR_NAME
     cmd: KEYCLOAK_NAMESPACE="{{.KC_NAMESPACE_PREFIX}}keycloak" ACCELERATOR_NAME={{.ACCELERATOR_NAME}} CLUSTER_1={{.ROSA_CLUSTER_NAME_1}} CLUSTER_2={{.ROSA_CLUSTER_NAME_2}} ./accelerator_multi_az_delete.sh
 
+  global-accelerator-recover:
+    desc: "Recover from Global Accelerator split-brain"
+    cmds:
+      - task: global-accelerator-create
+
   route53:
     desc: "Creates Route53 primary/backup DNS records"
     dir: "{{.ROUTE53_DIR}}"

diff --git a/provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests/pom.xml b/provision/rosa-cross-dc/keycloak-benchmark-crossdc-tests/pom.xml
@@ -61,6 +61,28 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>globalaccelerator</artifactId>
+            <version>${aws.java.sdk.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>software.amazon.awssdk</groupId>
+                    <artifactId>netty-nio-client</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>elasticloadbalancingv2</artifactId>
+            <version>${aws.java.sdk.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>software.amazon.awssdk</groupId>
+                    <artifactId>netty-nio-client</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <profiles>
@@ -84,6 +106,9 @@
                         <groupId>org.apache.maven.plugins</groupId>
                         <artifactId>maven-surefire-plugin</artifactId>
                         <configuration>
+                            <systemPropertyVariables>
+                                <deployment.type>active-active</deployment.type>
+                            </systemPropertyVariables>
                             <excludedGroups>active-passive</excludedGroups>
                         </configuration>
                     </plugin>
@@ -101,6 +126,7 @@
                     <excludedGroups>active-active</excludedGroups>
                     <systemPropertyVariables>
                         <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager>
+                        <deployment.type>active-passive</deployment.type>
                         <!--suppress UnresolvedMavenProperty -->
                         <load-balancer.url>${LOAD_BALANCER_URL}</load-balancer.url>
                         <!--suppress UnresolvedMavenProperty -->

diff --git a/...hmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/AbstractCrossDCTest.java b/...hmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/AbstractCrossDCTest.java
@@ -13,6 +13,7 @@
 import org.keycloak.benchmark.crossdc.client.DatacenterInfo;
 import org.keycloak.benchmark.crossdc.client.KeycloakClient;
 import org.keycloak.benchmark.crossdc.junit.tags.ActiveActive;
+import org.keycloak.benchmark.crossdc.junit.tags.ActivePassive;
 import org.keycloak.benchmark.crossdc.util.HttpClientUtils;
 import org.keycloak.benchmark.crossdc.util.InfinispanUtils;
 import org.keycloak.representations.idm.ClientRepresentation;
@@ -51,7 +52,7 @@ public abstract class AbstractCrossDCTest {
     public AbstractCrossDCTest(TestInfo testInfo) {
         assertNotNull(MAIN_PASSWORD, "Main password must be set");
         var httpClient = HttpClientUtils.newHttpClient();
-        this.activePassive = !testInfo.getTags().contains(ActiveActive.TAG);
+        this.activePassive = !System.getProperty("deployment.type", "").equals(ActivePassive.TAG);
         this.DC_1 = new DatacenterInfo(httpClient, System.getProperty("keycloak.dc1.url"), System.getProperty("infinispan.dc1.url"), activePassive);
         this.DC_2 = new DatacenterInfo(httpClient, System.getProperty("keycloak.dc2.url"), System.getProperty("infinispan.dc2.url"), activePassive);
         this.LOAD_BALANCER_KEYCLOAK = new KeycloakClient(httpClient, System.getProperty("load-balancer.url"), activePassive);
@@ -154,21 +155,23 @@ public void tearDownTestEnvironment() throws URISyntaxException, IOException, In
               });
 
         MOCK_COOKIE_MANAGER.getCookieStore().removeAll();
-        failbackHealthChecks();
+        failbackLoadBalancers();
     }
 
     @AfterAll
     public void tearDown() throws URISyntaxException, IOException, InterruptedException {
-        failbackHealthChecks();
+        failbackLoadBalancers();
     }
 
-    private void failbackHealthChecks() throws URISyntaxException, IOException, InterruptedException {
+    private void failbackLoadBalancers() throws URISyntaxException, IOException, InterruptedException {
         DC_1.kc().markLBCheckUp();
         DC_2.kc().markLBCheckUp();
         if (activePassive) {
             String domain = DC_1.getKeycloakServerURL().substring("https://".length());
             AWSClient.updateRoute53HealthCheckPath(domain, "/lb-check");
             DC_1.kc().waitToBeActive(LOAD_BALANCER_KEYCLOAK);
+        } else {
+            AWSClient.acceleratorFallback(LOAD_BALANCER_KEYCLOAK.getKeycloakServerUrl());
         }
     }
 

diff --git a/...ak-benchmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/FailoverTest.java b/...ak-benchmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/FailoverTest.java
@@ -2,6 +2,7 @@
 
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
+import org.keycloak.benchmark.crossdc.junit.tags.ActiveActive;
 import org.keycloak.benchmark.crossdc.junit.tags.ActivePassive;
 
 import java.io.IOException;
@@ -38,4 +39,12 @@ public void logoutUserWithFailoverTest() throws IOException, URISyntaxException,
 
         LOAD_BALANCER_KEYCLOAK.refreshToken(REALM_NAME, (String) tokensMap.get("refresh_token"), CLIENTID, CLIENT_SECRET, 400);
     }
+
+    @Test
+    @ActiveActive
+    public void ensureAcceleratorUpdatedOnSplitBrainTest() {
+        // TODO Need a way to trigger a split-brain
+        // Make testsuite aware of K8s clusters so that GossipRouter can be enabled/disabled?
+        // Loadbalancer and Infinispan URLs can then be retrieved via Java client
+    }
 }
diff --git a/...enchmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/client/AWSClient.java b/...enchmark-crossdc-tests/src/test/java/org/keycloak/benchmark/crossdc/client/AWSClient.java
@@ -1,9 +1,10 @@
 package org.keycloak.benchmark.crossdc.client;
 
 import java.time.Duration;
+import java.util.List;
+import java.util.stream.Collectors;
 
 import org.jboss.logging.Logger;
-import org.keycloak.benchmark.crossdc.AbstractCrossDCTest;
 
 import software.amazon.awssdk.core.waiters.WaiterOverrideConfiguration;
 import software.amazon.awssdk.http.SdkHttpClient;
@@ -12,6 +13,14 @@
 import software.amazon.awssdk.services.cloudwatch.CloudWatchClient;
 import software.amazon.awssdk.services.cloudwatch.model.DescribeAlarmsRequest;
 import software.amazon.awssdk.services.cloudwatch.model.StateValue;
+import software.amazon.awssdk.services.elasticloadbalancingv2.ElasticLoadBalancingV2Client;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancer;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.LoadBalancerTypeEnum;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.Tag;
+import software.amazon.awssdk.services.elasticloadbalancingv2.model.TagDescription;
+import software.amazon.awssdk.services.globalaccelerator.GlobalAcceleratorClient;
+import software.amazon.awssdk.services.globalaccelerator.model.Accelerator;
+import software.amazon.awssdk.services.globalaccelerator.model.EndpointConfiguration;
 import software.amazon.awssdk.services.route53.Route53Client;
 import software.amazon.awssdk.services.route53.model.HealthCheck;
 import software.amazon.awssdk.services.route53.model.UpdateHealthCheckRequest;
@@ -54,4 +63,80 @@ public static void updateRoute53HealthCheckPath(String domainName, String path)
          );
       }
    }
+
+   public static void acceleratorFallback(String acceleratorDns) {
+      try (
+            SdkHttpClient httpClient = ApacheHttpClient.builder().build();
+            GlobalAcceleratorClient gaClient = GlobalAcceleratorClient.builder()
+                  .region(Region.US_WEST_2)
+                  .httpClient(httpClient)
+                  .build()
+      ) {
+         // Retrieve Accelerator instance based upon DNS
+         Accelerator accelerator = gaClient.listAccelerators().accelerators()
+               .stream()
+               .filter(a -> a.dnsName().equals(acceleratorDns))
+               .findFirst()
+               .orElseThrow();
+
+         var listenerArn = gaClient.listListeners(b -> b.acceleratorArn(accelerator.acceleratorArn()))
+               .listeners()
+               .stream()
+               .findFirst()
+               .orElseThrow()
+               .listenerArn();
+
+         var endpointGroup = gaClient.listEndpointGroups(b -> b.listenerArn(listenerArn))
+               .endpointGroups()
+               .stream()
+               .findFirst()
+               .orElseThrow();
+
+         var region = endpointGroup.endpointGroupRegion();
+
+         List<String> endpoints;
+         try (ElasticLoadBalancingV2Client elbClient =
+                    ElasticLoadBalancingV2Client.builder()
+                          .region(Region.of(region))
+                          .httpClient(httpClient)
+                          .build()
+         ) {
+            // Get all LBs associated with the Accelerator
+            var elbs = elbClient.describeLoadBalancers()
+                  .loadBalancers()
+                  .stream()
+                  .filter(lb -> lb.type() == LoadBalancerTypeEnum.NETWORK)
+                  .map(LoadBalancer::loadBalancerArn)
+                  .toList();
+
+            endpoints = elbClient.describeTags(b -> b.resourceArns(elbs))
+                  .tagDescriptions()
+                  .stream()
+                  .filter(td -> td.tags()
+                        .contains(
+                              Tag.builder()
+                                    .key("accelerator")
+                                    .value(accelerator.name())
+                                    .build()
+                        )
+                  )
+                  .map(TagDescription::resourceArn)
+                  .toList();
+         }
+
+         var endpointConfigs = endpoints.stream()
+               .map(elb -> EndpointConfiguration.builder()
+                     .clientIPPreservationEnabled(false)
+                     .endpointId(elb)
+                     .weight(50)
+                     .build()
+               ).toList();
+
+         // Add all LBs to the Accelerator EndpointGroup
+         gaClient.updateEndpointGroup(
+               g -> g.endpointGroupArn(endpointGroup.endpointGroupArn())
+                     .endpointConfigurations(endpointConfigs)
+         );
+      }
+   }
 }