diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index cf8f941b..68a39f8c 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -14,6 +14,7 @@ jobs: fail-fast: false matrix: python-version: [ "3.8", "3.11" ] + db-engine: ["aurora", "multi-az"] steps: - name: 'Clone repository' @@ -58,7 +59,7 @@ jobs: - name: 'Run Integration Tests' run: | - ./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }} --info + ./gradlew --no-parallel --no-daemon test-python-${{ matrix.python-version }}-${{ matrix.db-engine }} --info env: RDS_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }} RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} diff --git a/tests/integration/container/utils/rds_test_utility.py b/tests/integration/container/utils/rds_test_utility.py index 7e38b608..a4059188 100644 --- a/tests/integration/container/utils/rds_test_utility.py +++ b/tests/integration/container/utils/rds_test_utility.py @@ -156,6 +156,7 @@ def failover_cluster_and_wait_until_writer_changed( sleep(1) cluster_address = socket.gethostbyname(cluster_endpoint) + self.make_sure_instances_up(self.get_instance_ids()) self.logger.debug("Testing.FinishedFailover", initial_writer_id, str((perf_counter_ns() - start) / 1_000_000)) def failover_cluster(self, cluster_id: Optional[str] = None) -> None: @@ -403,7 +404,7 @@ def make_sure_instances_up(self, instances: List[str]) -> None: instance_info: TestInstanceInfo = database_info.get_instance(i) success: bool = False start_time = timeit.default_timer() - while (timeit.default_timer() - start_time) < 300: # 5 min + while (timeit.default_timer() - start_time) < 600: # 10 min try: conn = self._open_connection(instance_info) conn.close() diff --git a/tests/integration/host/build.gradle.kts b/tests/integration/host/build.gradle.kts index e0877122..eff3d0fb 100644 --- a/tests/integration/host/build.gradle.kts +++ b/tests/integration/host/build.gradle.kts @@ -62,21 +62,43 @@ tasks.withType { systemProperty("java.util.logging.config.file", "${project.buildDir}/resources/test/logging-test.properties") } -tasks.register("test-python-3.11") { +tasks.register("test-python-3.11-aurora") { group = "verification" filter.includeTestsMatching("integration.host.TestRunner.runTests") doFirst { systemProperty("exclude-performance", "true") systemProperty("exclude-python-38", "true") + systemProperty("exclude-multi-az", "true") + } +} + +tasks.register("test-python-3.8-aurora") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + systemProperty("exclude-performance", "true") + systemProperty("exclude-python-311", "true") + systemProperty("exclude-multi-az", "true") } } -tasks.register("test-python-3.8") { +tasks.register("test-python-3.11-multi-az") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + systemProperty("exclude-performance", "true") + systemProperty("exclude-python-38", "true") + systemProperty("exclude-aurora", "true") + } +} + +tasks.register("test-python-3.8-multi-az") { group = "verification" filter.includeTestsMatching("integration.host.TestRunner.runTests") doFirst { systemProperty("exclude-performance", "true") systemProperty("exclude-python-311", "true") + systemProperty("exclude-aurora", "true") } } diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironment.java b/tests/integration/host/src/test/java/integration/host/TestEnvironment.java index f99fc3bf..5e43d6b9 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironment.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironment.java @@ -16,8 +16,6 @@ package integration.host; -import static org.junit.jupiter.api.Assertions.assertEquals; - import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import eu.rekawek.toxiproxy.ToxiproxyClient; @@ -44,6 +42,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; @@ -70,6 +69,7 @@ public class TestEnvironment implements AutoCloseable { private static final TestEnvironmentConfiguration config = new TestEnvironmentConfiguration(); private static final boolean USE_OTLP_CONTAINER_FOR_TRACES = false; + private static final AtomicInteger ipAddressUsageRefCount = new AtomicInteger(0); private final TestEnvironmentInfo info = new TestEnvironmentInfo(); // only this info is passed to test container @@ -102,12 +102,22 @@ private TestEnvironment(TestEnvironmentRequest request) { } public static TestEnvironment build(TestEnvironmentRequest request) throws IOException { + DatabaseEngineDeployment deployment = request.getDatabaseEngineDeployment(); + if (deployment == DatabaseEngineDeployment.AURORA + || deployment == DatabaseEngineDeployment.RDS + || deployment == DatabaseEngineDeployment.RDS_MULTI_AZ) { + // These environment require creating external database cluster that should be publicly available. + // Corresponding AWS Security Groups should be configured and the test task runner IP address + // should be whitelisted. + ipAddressUsageRefCount.incrementAndGet(); + } + LOGGER.finest("Building test env: " + request.getEnvPreCreateIndex()); preCreateEnvironment(request.getEnvPreCreateIndex()); TestEnvironment env; - switch (request.getDatabaseEngineDeployment()) { + switch (deployment) { case DOCKER: env = new TestEnvironment(request); initDatabaseParams(env); @@ -894,7 +904,11 @@ public void close() throws Exception { private void deleteDbCluster() { if (!this.reuseAuroraDbCluster && !StringUtils.isNullOrEmpty(this.runnerIP)) { - auroraUtil.ec2DeauthorizesIP(runnerIP); + if (ipAddressUsageRefCount.decrementAndGet() == 0) { + // Another test environments are still in use of test task runner IP address. + // The last execute tst environment will do the cleanup. + auroraUtil.ec2DeauthorizesIP(runnerIP); + } } if (!this.reuseAuroraDbCluster) {