From bfa4824b40b2c8af844a0fdfafa5a9bd7c09ad03 Mon Sep 17 00:00:00 2001 From: Crystal Liang Date: Thu, 11 Jul 2024 16:49:39 -0700 Subject: [PATCH] test: aurora failover performance tests --- .github/workflows/aurora_performance.yml | 89 +++ .github/workflows/integration_tests.yml | 2 +- common/lib/plugin_service.ts | 3 +- .../efm/host_monitoring_connection_plugin.ts | 22 +- common/lib/plugins/efm/monitor.ts | 15 +- .../plugins/efm/monitor_connection_context.ts | 22 +- common/lib/plugins/efm/monitor_service.ts | 7 +- .../lib/plugins/failover/failover_plugin.ts | 2 +- common/lib/utils/client_utils.ts | 8 +- common/lib/wrapper_property.ts | 21 +- mysql/lib/client.ts | 2 +- mysql/lib/dialect/mysql_database_dialect.ts | 5 +- package-lock.json | 16 +- package.json | 6 +- pg/lib/client.ts | 2 +- .../container/tests/aurora_failover.test.ts | 252 ++++---- .../tests/basic_connectivity.test.ts | 252 ++++---- tests/integration/container/tests/config.ts | 7 + .../tests/iam_authentication.test.ts | 188 +++--- .../container/tests/performance.test.ts | 348 ++++++++++ .../tests/read_write_splitting.test.ts | 592 ++++++++++-------- .../container/tests/utils/driver_helper.ts | 24 +- .../container/tests/utils/proxy_helper.ts | 4 +- .../container/tests/utils/test_environment.ts | 78 +-- tests/integration/host/build.gradle.kts | 49 ++ .../host/TestEnvironmentProvider.java | 2 - .../host/util/ContainerHelper.java | 1 + tests/unit/monitor_connection_context.test.ts | 27 +- 28 files changed, 1355 insertions(+), 691 deletions(-) create mode 100644 .github/workflows/aurora_performance.yml create mode 100644 tests/integration/container/tests/performance.test.ts diff --git a/.github/workflows/aurora_performance.yml b/.github/workflows/aurora_performance.yml new file mode 100644 index 000000000..6eb262acb --- /dev/null +++ b/.github/workflows/aurora_performance.yml @@ -0,0 +1,89 @@ +name: Aurora Performance Tests + +on: + workflow_dispatch: + push: + branches: + - performance-tests + +jobs: + run-integration-tests: + strategy: + matrix: + db: [mysql, pg] + + name: Run Aurora ${{ matrix.db }} container performance tests + runs-on: ubuntu-latest + + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + fetch-depth: 50 + + - name: "Set up JDK 8" + uses: actions/setup-java@v3 + with: + distribution: "corretto" + java-version: 8 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "20.x" + - name: Install dependencies + run: npm install --no-save + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: Set up Temp AWS Credentials + run: | + creds=($(aws sts get-session-token \ + --duration-seconds 21600 \ + --query 'Credentials.[AccessKeyId, SecretAccessKey, SessionToken]' \ + --output text \ + | xargs)); + echo "::add-mask::${creds[0]}" + echo "::add-mask::${creds[1]}" + echo "::add-mask::${creds[2]}" + echo "TEMP_AWS_ACCESS_KEY_ID=${creds[0]}" >> $GITHUB_ENV + echo "TEMP_AWS_SECRET_ACCESS_KEY=${creds[1]}" >> $GITHUB_ENV + echo "TEMP_AWS_SESSION_TOKEN=${creds[2]}" >> $GITHUB_ENV + + - name: Run Integration Tests + run: | + ./gradlew --no-parallel --no-daemon test-aurora-${{ matrix.db }}-performance --info + env: + AURORA_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }} + AURORA_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ env.TEMP_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ env.TEMP_AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ env.TEMP_AWS_SESSION_TOKEN }} + NUM_INSTANCES: 5 + + - name: "Get Github Action IP" + if: always() + id: ip + uses: haythem/public-ip@v1.3 + + - name: "Remove Github Action IP" + if: always() + run: | + aws ec2 revoke-security-group-ingress \ + --group-name default \ + --protocol -1 \ + --port -1 \ + --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ + 2>&1 > /dev/null; + + - name: Archive Performance results + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.db }}-performance-results + path: ./tests/integration/container/reports + retention-days: 5 diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 592b2a98c..d6ff0ccf7 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -86,7 +86,7 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: integration-report path: ./tests/integration/container/reports diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 492a7c71b..e48d0b11e 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -282,9 +282,8 @@ export class PluginService implements ErrorHandler, HostListProviderService { createTargetClient(props: Map): any { const createClientFunc = this.getCurrentClient().getCreateClientFunc(); - const copy = WrapperProperties.removeWrapperProperties(Object.fromEntries(props.entries())); if (createClientFunc) { - return createClientFunc(Object.fromEntries(new Map(Object.entries(copy)))); + return createClientFunc(props); } throw new AwsWrapperError("AwsClient is missing create target client function."); // This should not be reached } diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts index 33fefb886..d88f4819d 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts @@ -111,7 +111,14 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp failureDetectionIntervalMillis, failureDetectionCount ); - result = await methodFunc(); + + result = await Promise.race([monitorContext.trackHealthStatus(), methodFunc()]) + .then((result: any) => { + return result; + }) + .catch((error: any) => { + throw error; + }); } finally { if (monitorContext != null) { await this.monitorService.stopMonitoring(monitorContext); @@ -123,13 +130,15 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp } const targetClient = this.pluginService.getCurrentClient().targetClient; - let isClientClosed = false; + let isClientValid = false; if (targetClient) { - isClientClosed = await this.pluginService.isClientValid(targetClient); + isClientValid = await this.pluginService.isClientValid(targetClient); } - if (!targetClient || !isClientClosed) { - await this.pluginService.getCurrentClient().end(); + if (!targetClient || !isClientValid) { + if (targetClient) { + await this.pluginService.tryClosingTargetClient(targetClient); + } // eslint-disable-next-line no-unsafe-finally throw new AwsWrapperError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo.host)); } @@ -192,7 +201,6 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp } async releaseResources(): Promise { - const hostKeys = (await this.getMonitoringHostInfo())?.allAliases; - return this.monitorService.releaseResources(hostKeys); + return this.monitorService.releaseResources(); } } diff --git a/common/lib/plugins/efm/monitor.ts b/common/lib/plugins/efm/monitor.ts index 742e29cdb..c721335f1 100644 --- a/common/lib/plugins/efm/monitor.ts +++ b/common/lib/plugins/efm/monitor.ts @@ -21,6 +21,7 @@ import { logger } from "../../../logutils"; import { Messages } from "../../utils/messages"; import { ClientWrapper } from "../../client_wrapper"; import { sleep } from "../../utils/utils"; +import { WrapperProperties } from "../../wrapper_property"; export interface Monitor { startMonitoring(context: MonitorConnectionContext): void; @@ -51,7 +52,6 @@ class ConnectionStatus { export class MonitorImpl implements Monitor { private readonly SLEEP_WHEN_INACTIVE_MILLIS: number = 100; private readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS: number = 3000; - private readonly MONITORING_PROPERTY_PREFIX: string = "monitoring_"; private readonly activeContexts: MonitorConnectionContext[] = []; private readonly newContexts: MonitorConnectionContext[] = []; @@ -63,7 +63,7 @@ export class MonitorImpl implements Monitor { private contextLastUsedTimestampNanos: number; private started = false; private stopped: boolean = false; - private cancel: boolean = false; + private cancelled: boolean = false; private monitoringClient: ClientWrapper | null = null; private delayMillisTimeoutId: any; private sleepWhenInactiveTimeoutId: any; @@ -110,8 +110,7 @@ export class MonitorImpl implements Monitor { logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); try { - this.stopped = false; - while (!this.cancel) { + while (!this.cancelled) { try { let newMonitorContext: MonitorConnectionContext | undefined; let firstAddedNewMonitorContext: MonitorConnectionContext | null = null; @@ -234,11 +233,11 @@ export class MonitorImpl implements Monitor { const monitoringConnProperties: Map = new Map(this.properties); for (const key of this.properties.keys()) { - if (!key.startsWith(this.MONITORING_PROPERTY_PREFIX)) { + if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { continue; } - monitoringConnProperties.set(key.substring(this.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); + monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); monitoringConnProperties.delete(key); } @@ -258,7 +257,7 @@ export class MonitorImpl implements Monitor { } isStopped(): boolean { - return this.stopped; + return this.stopped || this.cancelled; } protected getCurrentTimeNano() { @@ -266,7 +265,7 @@ export class MonitorImpl implements Monitor { } async releaseResources() { - this.cancel = true; + this.cancelled = true; clearTimeout(this.delayMillisTimeoutId); clearTimeout(this.sleepWhenInactiveTimeoutId); await this.endMonitoringClient(); diff --git a/common/lib/plugins/efm/monitor_connection_context.ts b/common/lib/plugins/efm/monitor_connection_context.ts index 616bc30d4..85f9ede93 100644 --- a/common/lib/plugins/efm/monitor_connection_context.ts +++ b/common/lib/plugins/efm/monitor_connection_context.ts @@ -18,6 +18,10 @@ import { Monitor } from "./monitor"; import { logger } from "../../../logutils"; import { Messages } from "../../utils/messages"; import { ClientWrapper } from "../../client_wrapper"; +import { sleep } from "../../utils/utils"; +import { AwsWrapperError } from "../../utils/errors"; +import { uniqueId } from "lodash"; +import { PluginService } from "../../plugin_service"; export class MonitorConnectionContext { readonly failureDetectionIntervalMillis: number; @@ -25,6 +29,7 @@ export class MonitorConnectionContext { private readonly failureDetectionCount: number; readonly clientToAbort: ClientWrapper; readonly monitor: Monitor; + readonly pluginService: PluginService; isActiveContext: boolean = true; isHostUnhealthy: boolean = false; @@ -33,19 +38,22 @@ export class MonitorConnectionContext { private invalidHostStartTimeNano: number = 0; private abortedConnectionCounter: number = 0; failureCount: number = 0; + id: string = uniqueId("_monitorContext"); constructor( monitor: Monitor, clientToAbort: any, failureDetectionTimeMillis: number, failureDetectionIntervalMillis: number, - failureDetectionCount: number + failureDetectionCount: number, + pluginService: PluginService ) { this.monitor = monitor; this.clientToAbort = clientToAbort; this.failureDetectionTimeMillis = failureDetectionTimeMillis; this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; this.failureDetectionCount = failureDetectionCount; + this.pluginService = pluginService; } resetInvalidHostStartTimeNano(): void { @@ -62,7 +70,7 @@ export class MonitorConnectionContext { } try { - await this.clientToAbort.client.end(); + await this.pluginService.tryClosingTargetClient(this.clientToAbort); } catch (error: any) { // ignore logger.debug(Messages.get("MonitorConnectionContext.exceptionAbortingConnection", error.message)); @@ -116,7 +124,7 @@ export class MonitorConnectionContext { const invalidHostDurationNano: number = statusCheckEndNano - this.invalidHostStartTimeNano; const maxInvalidHostDurationMillis: number = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount); - if (invalidHostDurationNano >= maxInvalidHostDurationMillis * 1_000_000) { + if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationMillis * 1_000_000) { logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); this.isHostUnhealthy = true; await this.abortConnection(); @@ -133,4 +141,12 @@ export class MonitorConnectionContext { logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); } + + async trackHealthStatus() { + while (!this.isHostUnhealthy && this.isActiveContext) { + await sleep(100); + } + + throw new AwsWrapperError("trackHealthStatus stopped"); + } } diff --git a/common/lib/plugins/efm/monitor_service.ts b/common/lib/plugins/efm/monitor_service.ts index 59f2d7a11..c2ed25d4b 100644 --- a/common/lib/plugins/efm/monitor_service.ts +++ b/common/lib/plugins/efm/monitor_service.ts @@ -38,7 +38,7 @@ export interface MonitorService { stopMonitoringForAllConnections(hostKeys: Set): void; - releaseResources(hostKeys: Set | undefined): Promise; + releaseResources(): Promise; } export class MonitorServiceImpl implements MonitorService { @@ -87,7 +87,8 @@ export class MonitorServiceImpl implements MonitorService { clientToAbort, failureDetectionTimeMillis, failureDetectionIntervalMillis, - failureDetectionCount + failureDetectionCount, + this.pluginService ); monitor.startMonitoring(context); return context; @@ -152,7 +153,7 @@ export class MonitorServiceImpl implements MonitorService { } } - async releaseResources(hostKeys: Set) { + async releaseResources() { for (const [key, monitor] of MonitorServiceImpl.monitors.entries) { if (monitor.item) { await monitor.item.releaseResources(); diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index 68ae3412c..7acf052d5 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -15,7 +15,7 @@ */ import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { uniqueId } from "lodash"; +import { method, uniqueId } from "lodash"; import { logger } from "../../../logutils"; import { HostInfo } from "../../host_info"; import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; diff --git a/common/lib/utils/client_utils.ts b/common/lib/utils/client_utils.ts index 98dbc96c9..3c2799fd6 100644 --- a/common/lib/utils/client_utils.ts +++ b/common/lib/utils/client_utils.ts @@ -21,9 +21,13 @@ import { logger } from "../../logutils"; import { WrapperProperties } from "../wrapper_property"; export class ClientUtils { - static async queryWithTimeout(newPromise: Promise, props: Map): Promise { + static async queryWithTimeout(newPromise: Promise, props: Map, timeValue?: number): Promise { const timer: any = {}; - const timeoutTask = getTimeoutTask(timer, Messages.get("ClientUtils.queryTaskTimeout"), WrapperProperties.INTERNAL_QUERY_TIMEOUT.get(props)); + const timeoutTask = getTimeoutTask( + timer, + Messages.get("ClientUtils.queryTaskTimeout"), + timeValue ?? WrapperProperties.INTERNAL_QUERY_TIMEOUT.get(props) + ); return await Promise.race([timeoutTask, newPromise]) .then((result) => { if (result) { diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index 48ae2f0e4..75e71e2e7 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -40,6 +40,7 @@ export class WrapperProperty { } export class WrapperProperties { + static readonly MONITORING_PROPERTY_PREFIX: string = "monitoring_"; static readonly DEFAULT_PLUGINS = "auroraConnectionTracker,failover,hostMonitoring"; static readonly DEFAULT_TOKEN_EXPIRATION_SEC = 15 * 60; @@ -240,8 +241,7 @@ export class WrapperProperties { 600000 // 10 minutes ); - static removeWrapperProperties(config: T): T { - const copy = Object.assign({}, config); + static removeWrapperProperties(props: Map): any { const persistingProperties = [ WrapperProperties.USER.name, WrapperProperties.PASSWORD.name, @@ -250,16 +250,25 @@ export class WrapperProperties { WrapperProperties.HOST.name ]; + const copy = new Map(props); + + for (const key of props.keys()) { + if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { + continue; + } + + copy.delete(key); + } + Object.values(WrapperProperties).forEach((prop) => { if (prop instanceof WrapperProperty) { const propertyName = (prop as WrapperProperty).name; - if (!persistingProperties.includes(propertyName) && Object.hasOwn(config as object, propertyName)) { - // @ts-expect-error - delete copy[propertyName]; + if (!persistingProperties.includes(propertyName) && copy.has(propertyName)) { + copy.delete(propertyName); } } }); - return copy; + return Object.fromEntries(copy.entries()); } } diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index aa2d0499a..a92a71fc3 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -40,7 +40,7 @@ export class AwsMySQLClient extends AwsClient { constructor(config: any) { super(config, new MySQLErrorHandler(), DatabaseType.MYSQL, AwsMySQLClient.knownDialectsByCode, new MySQLConnectionUrlParser()); - this._createClientFunc = (config: any) => { + this._createClientFunc = (config: Map) => { return createConnection(WrapperProperties.removeWrapperProperties(config)); }; this.resetState(); diff --git a/mysql/lib/dialect/mysql_database_dialect.ts b/mysql/lib/dialect/mysql_database_dialect.ts index 70eb868a5..ed233f13b 100644 --- a/mysql/lib/dialect/mysql_database_dialect.ts +++ b/mysql/lib/dialect/mysql_database_dialect.ts @@ -22,7 +22,6 @@ import { AwsWrapperError } from "../../../common/lib/utils/errors"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; import { TransactionIsolationLevel } from "../../../common/lib/utils/transaction_isolation_level"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { Utils } from "../utils"; import { ClientUtils } from "../../../common/lib/utils/client_utils"; export class MySQLDatabaseDialect implements DatabaseDialect { @@ -75,8 +74,8 @@ export class MySQLDatabaseDialect implements DatabaseDialect { async tryClosingTargetClient(targetClient: ClientWrapper) { try { - await ClientUtils.queryWithTimeout(targetClient.client.promise().end(), targetClient.properties); - } catch (error) { + await ClientUtils.queryWithTimeout(targetClient.client.promise().destroy(), targetClient.properties); + } catch (error: any) { // ignore } } diff --git a/package-lock.json b/package-lock.json index 774cac322..0ae60996a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -53,7 +53,8 @@ "ts-node": "^10.9.2", "tsx": "^4.18.0", "typescript": "^5.5.2", - "winston": "3.14.2" + "winston": "3.14.2", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" } }, "node_modules/@ampproject/remapping": { @@ -9906,6 +9907,19 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/xlsx": { + "version": "0.20.3", + "resolved": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz", + "integrity": "sha512-oLDq3jw7AcLqKWH2AhCpVTZl8mf6X2YReP+Neh0SJUzV/BdZYjth94tG5toiMB1PPrYtxOCfaoUCkvtuH+3AJA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xml2js": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", diff --git a/package.json b/package.json index f525f3adc..b87f32933 100644 --- a/package.json +++ b/package.json @@ -72,10 +72,12 @@ "ts-node": "^10.9.2", "tsx": "^4.18.0", "typescript": "^5.5.2", - "winston": "3.14.2" + "winston": "3.14.2", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "overrides": { "braces": "^3.0.3", - "fast-xml-parser": "^4.4.1" + "fast-xml-parser": "^4.4.1", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" } } diff --git a/pg/lib/client.ts b/pg/lib/client.ts index fa671205a..652990bbc 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -38,7 +38,7 @@ export class AwsPGClient extends AwsClient { constructor(config: any) { super(config, new PgErrorHandler(), DatabaseType.POSTGRES, AwsPGClient.knownDialectsByCode, new PgConnectionUrlParser()); - this._createClientFunc = (config: any) => { + this._createClientFunc = (config: Map) => { const targetClient: Client = new Client(WrapperProperties.removeWrapperProperties(config)); targetClient.on("error", (error: any) => { this.emit("error", error); diff --git a/tests/integration/container/tests/aurora_failover.test.ts b/tests/integration/container/tests/aurora_failover.test.ts index 7b48412c6..046b52a54 100644 --- a/tests/integration/container/tests/aurora_failover.test.ts +++ b/tests/integration/container/tests/aurora_failover.test.ts @@ -23,6 +23,10 @@ import { QueryResult } from "pg"; import { ProxyHelper } from "./utils/proxy_helper"; import { RdsUtils } from "../../../../common/lib/utils/rds_utils"; import { logger } from "../../../../common/logutils"; +import { features } from "./config"; +import { TestEnvironmentFeatures } from "./utils/test_environment_features"; + +const itIf = features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && !features.includes(TestEnvironmentFeatures.PERFORMANCE) ? it : it.skip; let env: TestEnvironment; let driver; @@ -82,151 +86,167 @@ describe("aurora failover", () => { logger.info(`Test finished: ${expect.getState().currentTestName}`); }, 1000000); - it("fails from writer to new writer on connection invocation", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await client.connect(); + itIf( + "fails from writer to new writer on connection invocation", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + client.on("error", (error: any) => { + logger.debug(error.message); + }); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + await client.connect(); - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, 1000000); + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); - it("writer fails within transaction", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1000000 + ); - client.on("error", (error: any) => { - logger.debug(error.message); - }); + itIf( + "writer fails within transaction", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + client.on("error", (error: any) => { + logger.debug(error.message); + }); - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - await expect(async () => { - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); - }).rejects.toThrow(TransactionResolutionUnknownError); + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - // Attempt to query the instance id. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + await expect(async () => { + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); + }).rejects.toThrow(TransactionResolutionUnknownError); - // Assert that we are connected to the new writer after failover happens. - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + // Attempt to query the instance id. + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); - expect(currentConnectionId).toBe(nextClusterWriterId); - expect(initialWriterId).not.toBe(nextClusterWriterId); + // Assert that we are connected to the new writer after failover happens. + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - // Assert that NO row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("0"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); - } + const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); + expect(currentConnectionId).toBe(nextClusterWriterId); + expect(initialWriterId).not.toBe(nextClusterWriterId); - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, 1000000); + // Assert that NO row has been inserted to the table. + const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); + if (env.engine === DatabaseEngine.PG) { + expect((result as QueryResult).rows[0]["count"]).toBe("0"); + } else if (env.engine === DatabaseEngine.MYSQL) { + expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); + } - it("fails from writer and transfers session state", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + }, + 1000000 + ); - client.on("error", (error: any) => { - logger.debug(error.message); - }); + itIf( + "fails from writer and transfers session state", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); + client.on("error", (error: any) => { + logger.debug(error.message); + }); - await client.setReadOnly(true); - const writerId = await auroraTestUtility.queryInstanceId(client); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); - // Failover cluster and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + await client.setReadOnly(true); + const writerId = await auroraTestUtility.queryInstanceId(client); - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); + // Failover cluster and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, 1000000); + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); - it("fails from reader to writer", async () => { - // Connect to writer instance - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - client.on("error", (err: any) => { - logger.debug(err); - }); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Get a reader instance - let readerInstanceHost; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - readerInstanceHost = host.host; + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1000000 + ); + + itIf( + "fails from reader to writer", + async () => { + // Connect to writer instance + const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + client = initClientFunc(writerConfig); + client.on("error", (err: any) => { + logger.debug(err); + }); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Get a reader instance + let readerInstanceHost; + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + readerInstanceHost = host.host; + } } - } - if (!readerInstanceHost) { - throw new Error("Could not find a reader instance"); - } - const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); + if (!readerInstanceHost) { + throw new Error("Could not find a reader instance"); + } + const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); - secondaryClient = initClientFunc(readerConfig); - secondaryClient.on("error", (err: any) => { - logger.debug(err); - }); + secondaryClient = initClientFunc(readerConfig); + secondaryClient.on("error", (err: any) => { + logger.debug(err); + }); - await secondaryClient.connect(); + await secondaryClient.connect(); - // Crash the reader instance - const rdsUtils = new RdsUtils(); - const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); - if (readerInstanceId) { - await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); + // Crash the reader instance + const rdsUtils = new RdsUtils(); + const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); + if (readerInstanceId) { + await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); - await expect(async () => { - await auroraTestUtility.queryInstanceId(secondaryClient); - }).rejects.toThrow(FailoverSuccessError); + await expect(async () => { + await auroraTestUtility.queryInstanceId(secondaryClient); + }).rejects.toThrow(FailoverSuccessError); - await ProxyHelper.enableConnectivity(readerInstanceId); + await ProxyHelper.enableConnectivity(readerInstanceId); - // Assert that we are currently connected to the writer instance - const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).toBe(initialWriterId); - } - }, 1000000); + // Assert that we are currently connected to the writer instance + const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).toBe(initialWriterId); + } + }, + 1000000 + ); }); diff --git a/tests/integration/container/tests/basic_connectivity.test.ts b/tests/integration/container/tests/basic_connectivity.test.ts index 6582638c1..6fcaa8e69 100644 --- a/tests/integration/container/tests/basic_connectivity.test.ts +++ b/tests/integration/container/tests/basic_connectivity.test.ts @@ -20,6 +20,10 @@ import { DriverHelper } from "./utils/driver_helper"; import { AuroraTestUtility } from "./utils/aurora_test_utility"; import { logger } from "../../../../common/logutils"; import { DatabaseEngine } from "./utils/database_engine"; +import { TestEnvironmentFeatures } from "./utils/test_environment_features"; +import { features } from "./config"; + +const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) ? it : it.skip; let client: any; const auroraTestUtility = new AuroraTestUtility(); @@ -53,118 +57,138 @@ afterEach(async () => { }, 1000000); describe("basic_connectivity", () => { - it("wrapper with failover plugins read only endpoint", async () => { - const env = await TestEnvironment.getCurrent(); - const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - const initClientFunc = DriverHelper.getClient(driver); - - let props = { - user: env.databaseInfo.username, - host: env.databaseInfo.clusterReadOnlyEndpoint, - database: env.databaseInfo.default_db_name, - password: env.databaseInfo.password, - port: env.databaseInfo.clusterEndpointPort, - plugins: "failover,efm" - }; - props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); - client = initClientFunc(props); - - await executeInstanceQuery(client, env.engine, props); - }, 1000000); - - it("wrapper with failover plugins cluster endpoint", async () => { - const env = await TestEnvironment.getCurrent(); - const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - const initClientFunc = DriverHelper.getClient(driver); - - let props = { - user: env.databaseInfo.username, - host: env.databaseInfo.clusterEndpoint, - database: env.databaseInfo.default_db_name, - password: env.databaseInfo.password, - port: env.databaseInfo.clusterEndpointPort, - plugins: "failover,efm" - }; - props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); - - client = initClientFunc(props); - await executeInstanceQuery(client, env.engine, props); - }, 1000000); - - it("wrapper with failover plugins instance endpoint", async () => { - const env = await TestEnvironment.getCurrent(); - const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - const initClientFunc = DriverHelper.getClient(driver); - - let props = { - user: env.databaseInfo.username, - host: env.databaseInfo.instances[0].host, - database: env.databaseInfo.default_db_name, - password: env.databaseInfo.password, - port: env.databaseInfo.clusterEndpointPort, - plugins: "failover,efm" - }; - props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); - - client = initClientFunc(props); - await executeInstanceQuery(client, env.engine, props); - }, 1000000); - - it("wrapper", async () => { - const env = await TestEnvironment.getCurrent(); - const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - const initClientFunc = DriverHelper.getClient(driver); - - let props = { - user: env.databaseInfo.username, - host: env.databaseInfo.instances[0].host, - database: env.databaseInfo.default_db_name, - password: env.databaseInfo.password, - port: env.databaseInfo.instanceEndpointPort, - plugins: "" - }; - props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); - - client = initClientFunc(props); - client.on("error", (error: any) => { - logger.debug(error.message); - }); - await client.connect(); - - const res = await DriverHelper.executeInstanceQuery(env.engine, client); - - expect(res).not.toBeNull(); - }, 1000000); - - it("wrapper_proxy", async () => { - const env = await TestEnvironment.getCurrent(); - const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - const initClientFunc = DriverHelper.getClient(driver); - - let props = { - user: env.databaseInfo.username, - host: env.proxyDatabaseInfo.instances[0].host, - database: env.databaseInfo.default_db_name, - password: env.databaseInfo.password, - port: env.proxyDatabaseInfo.instanceEndpointPort, - plugins: "", - clusterInstanceHostPattern: "?." + env.proxyDatabaseInfo.instanceEndpointSuffix + ":" + env.proxyDatabaseInfo.instanceEndpointPort - }; - props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); - - client = initClientFunc(props); - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await client.connect(); - - await ProxyHelper.disableAllConnectivity(env.engine); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(); - - await ProxyHelper.enableAllConnectivity(); - }, 1000000); + itIf( + "wrapper with failover plugins read only endpoint", + async () => { + const env = await TestEnvironment.getCurrent(); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + const initClientFunc = DriverHelper.getClient(driver); + + let props = { + user: env.databaseInfo.username, + host: env.databaseInfo.clusterReadOnlyEndpoint, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: env.databaseInfo.clusterEndpointPort, + plugins: "failover,efm" + }; + props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); + client = initClientFunc(props); + + await executeInstanceQuery(client, env.engine, props); + }, + 1000000 + ); + + itIf( + "wrapper with failover plugins cluster endpoint", + async () => { + const env = await TestEnvironment.getCurrent(); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + const initClientFunc = DriverHelper.getClient(driver); + + let props = { + user: env.databaseInfo.username, + host: env.databaseInfo.clusterEndpoint, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: env.databaseInfo.clusterEndpointPort, + plugins: "failover,efm" + }; + props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); + + client = initClientFunc(props); + await executeInstanceQuery(client, env.engine, props); + }, + 1000000 + ); + + itIf( + "wrapper with failover plugins instance endpoint", + async () => { + const env = await TestEnvironment.getCurrent(); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + const initClientFunc = DriverHelper.getClient(driver); + + let props = { + user: env.databaseInfo.username, + host: env.databaseInfo.instances[0].host, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: env.databaseInfo.clusterEndpointPort, + plugins: "failover,efm" + }; + props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); + + client = initClientFunc(props); + await executeInstanceQuery(client, env.engine, props); + }, + 1000000 + ); + + itIf( + "wrapper", + async () => { + const env = await TestEnvironment.getCurrent(); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + const initClientFunc = DriverHelper.getClient(driver); + + let props = { + user: env.databaseInfo.username, + host: env.databaseInfo.instances[0].host, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: env.databaseInfo.instanceEndpointPort, + plugins: "" + }; + props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); + + client = initClientFunc(props); + client.on("error", (error: any) => { + logger.debug(error.message); + }); + await client.connect(); + + const res = await DriverHelper.executeInstanceQuery(env.engine, client); + + expect(res).not.toBeNull(); + }, + 1000000 + ); + + itIf( + "wrapper_proxy", + async () => { + const env = await TestEnvironment.getCurrent(); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + const initClientFunc = DriverHelper.getClient(driver); + + let props = { + user: env.databaseInfo.username, + host: env.proxyDatabaseInfo.instances[0].host, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: env.proxyDatabaseInfo.instanceEndpointPort, + plugins: "", + clusterInstanceHostPattern: "?." + env.proxyDatabaseInfo.instanceEndpointSuffix + ":" + env.proxyDatabaseInfo.instanceEndpointPort + }; + props = DriverHelper.addDriverSpecificConfiguration(props, env.engine); + + client = initClientFunc(props); + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await client.connect(); + + await ProxyHelper.disableAllConnectivity(env.engine); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(); + + await ProxyHelper.enableAllConnectivity(); + }, + 1000000 + ); }); diff --git a/tests/integration/container/tests/config.ts b/tests/integration/container/tests/config.ts index eb8d6ca63..bea4da304 100644 --- a/tests/integration/container/tests/config.ts +++ b/tests/integration/container/tests/config.ts @@ -24,3 +24,10 @@ function simpleFormatter(type: LogType, message: LogMessage): string { } global.console = new CustomConsole(process.stdout, process.stderr, simpleFormatter); + +const infoJson = process.env.TEST_ENV_INFO_JSON; +if (infoJson === undefined) { + throw new Error("env var required"); +} +const testInfo = JSON.parse(infoJson); +export const features = testInfo.request.features; diff --git a/tests/integration/container/tests/iam_authentication.test.ts b/tests/integration/container/tests/iam_authentication.test.ts index da4b3ce69..ecd14c3e1 100644 --- a/tests/integration/container/tests/iam_authentication.test.ts +++ b/tests/integration/container/tests/iam_authentication.test.ts @@ -24,6 +24,10 @@ import { AwsPGClient } from "../../../../pg/lib"; import { AwsMySQLClient } from "../../../../mysql/lib"; import { IamAuthenticationPlugin } from "../../../../common/lib/authentication/iam_authentication_plugin"; import { logger } from "../../../../common/logutils"; +import { TestEnvironmentFeatures } from "./utils/test_environment_features"; +import { features } from "./config"; + +const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && features.includes(TestEnvironmentFeatures.IAM) ? it : it.skip; let env: TestEnvironment; let driver; @@ -81,87 +85,111 @@ describe("iam authentication", () => { logger.info(`Test finished: ${expect.getState().currentTestName}`); }, 1000000); - it("iam wrong database username", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); - config["user"] = `WRONG_${env.info.databaseInfo.username}_USER`; - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await expect(client.connect()).rejects.toThrow(); - }, 100000); - - it("iam no database username", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); - config["user"] = undefined; - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await expect(client.connect()).rejects.toBeInstanceOf(AwsWrapperError); - }, 100000); - - it("iam invalid host", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); - config["iamHost"] = "<>"; - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await expect(client.connect()).rejects.toBeInstanceOf(AwsWrapperError); - }, 100000); + itIf( + "iam wrong database username", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); + config["user"] = `WRONG_${env.info.databaseInfo.username}_USER`; + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await expect(client.connect()).rejects.toThrow(); + }, + 100000 + ); + + itIf( + "iam no database username", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); + config["user"] = undefined; + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await expect(client.connect()).rejects.toBeInstanceOf(AwsWrapperError); + }, + 100000 + ); + + itIf( + "iam invalid host", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); + config["iamHost"] = "<>"; + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await expect(client.connect()).rejects.toBeInstanceOf(AwsWrapperError); + }, + 100000 + ); // Currently, PG cannot connect to an IP address with SSL enabled, skip if PG - it("iam using ip address", async () => { - if (env.engine === "MYSQL") { - const instance = env.writer; - if (instance.host) { - const ip = await getIpAddress(instance.host); - const config = await initDefaultConfig(ip.address); - - config["password"] = "anything"; - config["iamHost"] = instance.host; - - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await validateConnection(client); - } else { - throw new AwsWrapperError("Host not found"); + itIf( + "iam using ip address", + async () => { + if (env.engine === "MYSQL") { + const instance = env.writer; + if (instance.host) { + const ip = await getIpAddress(instance.host); + const config = await initDefaultConfig(ip.address); + + config["password"] = "anything"; + config["iamHost"] = instance.host; + + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await validateConnection(client); + } else { + throw new AwsWrapperError("Host not found"); + } } - } - }, 100000); - - it("iam valid connection properties", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); - config["password"] = "anything"; - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await validateConnection(client); - }, 100000); - - it("iam valid connection properties no password", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); - config["password"] = undefined; - const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await validateConnection(client); - }, 100000); + }, + 100000 + ); + + itIf( + "iam valid connection properties", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); + config["password"] = "anything"; + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await validateConnection(client); + }, + 100000 + ); + + itIf( + "iam valid connection properties no password", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint); + config["password"] = undefined; + const client: AwsPGClient | AwsMySQLClient = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await validateConnection(client); + }, + 100000 + ); }); diff --git a/tests/integration/container/tests/performance.test.ts b/tests/integration/container/tests/performance.test.ts new file mode 100644 index 000000000..9fc16b010 --- /dev/null +++ b/tests/integration/container/tests/performance.test.ts @@ -0,0 +1,348 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { logger } from "../../../../common/logutils"; +import { TestEnvironment } from "./utils/test_environment"; +import { DriverHelper } from "./utils/driver_helper"; +import { ProxyHelper } from "./utils/proxy_helper"; +import { AuroraTestUtility } from "./utils/aurora_test_utility"; +import { TestEnvironmentFeatures } from "./utils/test_environment_features"; +import * as XLSX from "xlsx"; +import { anything } from "ts-mockito"; +import { WrapperProperties } from "../../../../common/lib/wrapper_property"; +import { features } from "./config"; +import { MonitorServiceImpl } from "../../../../common/lib/plugins/efm/monitor_service"; + +const itIf = + features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && + features.includes(TestEnvironmentFeatures.PERFORMANCE) && + features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED) + ? it + : it.skip; + +const REPEAT_TIMES: number = process.env.REPEAT_TIMES ? Number(process.env.REPEAT_TIMES) : 5; +const PERF_FAILOVER_TIMEOUT_MS = 120000; +const failureDetectionTimeParams = [ + // Defaults + [30000, 5000, 3, 5000], + [30000, 5000, 3, 10000], + [30000, 5000, 3, 15000], + [30000, 5000, 3, 20000], + [30000, 5000, 3, 25000], + [30000, 5000, 3, 30000], + [30000, 5000, 3, 35000], + [30000, 5000, 3, 40000], + [30000, 5000, 3, 50000], + [30000, 5000, 3, 60000], + + // Aggressive detection scheme + [6000, 1000, 1, 1000], + [6000, 1000, 1, 2000], + [6000, 1000, 1, 3000], + [6000, 1000, 1, 4000], + [6000, 1000, 1, 5000], + [6000, 1000, 1, 6000], + [6000, 1000, 1, 7000], + [6000, 1000, 1, 8000], + [6000, 1000, 1, 9000], + [6000, 1000, 1, 10000] +]; + +let env: TestEnvironment; +let driver; +let initClientFunc: (props: any) => any; + +const auroraTestUtility = new AuroraTestUtility(); +let enhancedFailureMonitoringPerfDataList: PerfStatMonitoring[] = []; + +async function initDefaultConfig(host: string, port: number): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.default_db_name, + password: env.databaseInfo.password, + port: port, + failoverTimeoutMs: 250000 + }; + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine, true); + return config; +} + +async function connectWithRetry(client: any): Promise { + let connectCount = 0; + let clientConnected = false; + while (!clientConnected && connectCount < 10) { + try { + await client.connect(); + clientConnected = true; + } catch (error: any) { + // ignore + connectCount++; + } + } + + expect(clientConnected).toBe(true); +} + +async function testFailureDetectionTimeEfmEnabled() { + try { + for (let i = 0; i < failureDetectionTimeParams.length; i++) { + await executeFailureDetectionTimeEfmEnabled( + failureDetectionTimeParams[i][0], + failureDetectionTimeParams[i][1], + failureDetectionTimeParams[i][2], + failureDetectionTimeParams[i][3] + ); + } + } finally { + doWritePerfDataToFile(`EnhancedMonitoringOnly_Db_${env.engine}_Instances_${env.instances.length}_Plugins_efm.xlsx`, "EfmOnly"); + } +} + +async function testFailureDetectionTimeFailoverAndEfmEnabled() { + try { + for (let i = 0; i < failureDetectionTimeParams.length; i++) { + await executeFailureDetectionTimeFailoverAndEfmEnabled( + failureDetectionTimeParams[i][0], + failureDetectionTimeParams[i][1], + failureDetectionTimeParams[i][2], + failureDetectionTimeParams[i][3] + ); + } + } finally { + doWritePerfDataToFile(`FailoverWithEnhancedMonitoring_Db_${env.engine}_Instances_${env.instances.length}_Plugins_efm.xlsx`, "FailoverWithEfm"); + } +} + +function doWritePerfDataToFile(fileName: string, worksheetName: string) { + const rows = []; + for (let i = 0; i < enhancedFailureMonitoringPerfDataList.length; i++) { + rows.push(enhancedFailureMonitoringPerfDataList[i].writeData()); + } + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.json_to_sheet(rows); + XLSX.utils.book_append_sheet(workbook, worksheet, worksheetName); + XLSX.utils.sheet_add_aoa(worksheet, enhancedFailureMonitoringPerfDataList[0].writeHeader(), { origin: "A1" }); + XLSX.writeFile(workbook, __dirname + "/../reports/" + fileName); +} + +async function executeFailureDetectionTimeEfmEnabled( + detectionTimeMillis: number, + detectionIntervalMillis: number, + detectionCount: number, + sleepDelayMillis: number +) { + const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.clusterEndpointPort); + config["plugins"] = "efm"; + config[WrapperProperties.FAILURE_DETECTION_TIME_MS.name] = detectionTimeMillis; + config[WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.name] = detectionIntervalMillis; + config[WrapperProperties.FAILURE_DETECTION_COUNT.name] = detectionCount; + + await executeTest(sleepDelayMillis, config, detectionTimeMillis, detectionIntervalMillis, detectionCount); +} + +async function executeFailureDetectionTimeFailoverAndEfmEnabled( + detectionTimeMillis: number, + detectionIntervalMillis: number, + detectionCount: number, + sleepDelayMillis: number +) { + const props = new Map(); + const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.clusterEndpointPort); + config["plugins"] = "efm,failover"; + config[WrapperProperties.FAILURE_DETECTION_TIME_MS.name] = detectionTimeMillis; + config[WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.name] = detectionIntervalMillis; + config[WrapperProperties.FAILURE_DETECTION_COUNT.name] = detectionCount; + config[WrapperProperties.FAILOVER_TIMEOUT_MS.name] = PERF_FAILOVER_TIMEOUT_MS; + config[WrapperProperties.FAILOVER_MODE.name] = "strict-reader"; + + await executeTest(sleepDelayMillis, config, detectionTimeMillis, detectionIntervalMillis, detectionCount); +} + +async function executeTest( + sleepDelayMillis: number, + config: any, + detectionTimeMillis: number, + detectionIntervalMillis: number, + detectionCount: number +) { + const data = new PerfStatMonitoring(); + await doMeasurePerformance(sleepDelayMillis, REPEAT_TIMES, data, config); + data.paramDetectionTime = detectionTimeMillis; + data.paramDetectionInterval = detectionIntervalMillis; + data.paramDetectionCount = detectionCount; + logger.debug("Collected data: " + data.toString()); + enhancedFailureMonitoringPerfDataList.push(data); +} + +async function doMeasurePerformance(sleepDelayMillis: number, repeatTimes: number, data: PerfStatMonitoring, config: any) { + let downTimeMillis: number = 0; + const elapsedTimeMillis: number[] = []; + + for (let i = 0; i < repeatTimes; i++) { + const client = initClientFunc(config); + try { + await connectWithRetry(client); + client.on("error", (err: any) => { + logger.debug(err.message); + }); + + const instanceHost = await auroraTestUtility.queryInstanceId(client); + setTimeout(async () => { + await ProxyHelper.disableConnectivity(env.engine, instanceHost); + downTimeMillis = Date.now(); + logger.debug("Network outages started."); + }, sleepDelayMillis); + + expect(await DriverHelper.executeQuery(env.engine, client, DriverHelper.getSleepQuery(env.engine, 60), 120000)).toThrow(anything()); + } catch (error: any) { + // Calculate and add detection time. + if (downTimeMillis === 0) { + logger.warn("Network outages start time is undefined!"); + } else { + const failureTimeMillis = Date.now() - downTimeMillis; + logger.debug(`Time to detect failure: ${failureTimeMillis}`); + elapsedTimeMillis.push(failureTimeMillis); + } + } finally { + downTimeMillis = 0; + try { + await ProxyHelper.enableAllConnectivity(); + await client.end(); + MonitorServiceImpl.clearMonitors(); + } catch (error: any) { + // ignore + } + } + } + + let min; + let max; + let total = 0; + let iterations = 0; + for (let i = 0; i < repeatTimes; i++) { + if (!isNaN(elapsedTimeMillis[i])) { + iterations++; + total += elapsedTimeMillis[i]; + if (!max || elapsedTimeMillis[i] > max) { + max = elapsedTimeMillis[i]; + } + if (!min || elapsedTimeMillis[i] < min) { + min = elapsedTimeMillis[i]; + } + } + } + const avg = Math.round(total / iterations); + logger.debug(`Calculated average failure detection time: ${total} / ${iterations} = ${avg}`); + + data.paramNetworkOutageDelayMillis = sleepDelayMillis; + data.minFailureDetectionTimeMillis = min; + data.maxFailureDetectionTimeMillis = max; + data.avgFailureDetectionTimeMillis = avg; +} + +describe("performance", () => { + beforeEach(async () => { + enhancedFailureMonitoringPerfDataList = []; + env = await TestEnvironment.getCurrent(); + driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + await ProxyHelper.enableAllConnectivity(); + }); + + afterEach(async () => { + await TestEnvironment.updateWriter(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1000000); + + itIf( + "failure detection with efm enabled", + async () => { + await testFailureDetectionTimeEfmEnabled(); + }, + 10000000 + ); + + itIf( + "failure detection with failover and efm enabled", + async () => { + await testFailureDetectionTimeFailoverAndEfmEnabled(); + }, + 10000000 + ); +}); + +abstract class PerfStatBase { + paramNetworkOutageDelayMillis?: number; + minFailureDetectionTimeMillis?: number; + maxFailureDetectionTimeMillis?: number; + avgFailureDetectionTimeMillis?: number; + + writeHeader(): string[][] { + return []; + } + + writeData(): (number | undefined)[] { + return []; + } +} + +class PerfStatMonitoring extends PerfStatBase { + paramDetectionTime?: number; + paramDetectionInterval?: number; + paramDetectionCount?: number; + + writeHeader() { + return [ + [ + "FailureDetectionGraceTime", + "FailureDetectionInterval", + "FailureDetectionCount", + "NetworkOutageDelayMillis", + "MinFailureDetectionTimeMillis", + "MaxFailureDetectionTimeMillis", + "AvgFailureDetectionTimeMillis" + ] + ]; + } + + writeData() { + return [ + this.paramDetectionTime, + this.paramDetectionInterval, + this.paramDetectionCount, + this.paramNetworkOutageDelayMillis, + this.minFailureDetectionTimeMillis, + this.maxFailureDetectionTimeMillis, + this.avgFailureDetectionTimeMillis + ]; + } + + toString(): string { + return ( + `[paramDetectionTime=${this.paramDetectionTime}, ` + + `paramDetectionInterval=${this.paramDetectionInterval}, ` + + `paramDetectionCount=${this.paramDetectionCount}, ` + + `paramNetworkOutageDelayMillis=${this.paramNetworkOutageDelayMillis}, ` + + `minFailureDetectionTimeMillis=${this.minFailureDetectionTimeMillis}, ` + + `maxFailureDetectionTimeMillis=${this.maxFailureDetectionTimeMillis} ` + + `avgFailureDetectionTimeMillis=${this.avgFailureDetectionTimeMillis}` + ); + } +} diff --git a/tests/integration/container/tests/read_write_splitting.test.ts b/tests/integration/container/tests/read_write_splitting.test.ts index 0273265d7..7a2f1c078 100644 --- a/tests/integration/container/tests/read_write_splitting.test.ts +++ b/tests/integration/container/tests/read_write_splitting.test.ts @@ -22,6 +22,10 @@ import { DatabaseEngine } from "./utils/database_engine"; import { QueryResult } from "pg"; import { ProxyHelper } from "./utils/proxy_helper"; import { logger } from "../../../../common/logutils"; +import { TestEnvironmentFeatures } from "./utils/test_environment_features"; +import { features } from "./config"; + +const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && features.includes(TestEnvironmentFeatures.IAM) ? it : it.skip; let env: TestEnvironment; let driver; @@ -87,316 +91,360 @@ describe("aurora read write splitting", () => { logger.info(`Test finished: ${expect.getState().currentTestName}`); }, 1000000); - it("test connect to writer switch set read only", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await client.setReadOnly(true); - const readerId = await auroraTestUtility.queryInstanceId(client); - expect(readerId).not.toBe(initialWriterId); + itIf( + "test connect to writer switch set read only", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); - await client.setReadOnly(true); - const currentId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentId0).toStrictEqual(readerId); + client.on("error", (error: any) => { + logger.debug(error.message); + }); - await client.setReadOnly(false); - const currentId1 = await auroraTestUtility.queryInstanceId(client); - expect(currentId1).toStrictEqual(initialWriterId); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - await client.setReadOnly(false); - const currentId2 = await auroraTestUtility.queryInstanceId(client); - expect(currentId2).toStrictEqual(initialWriterId); + await client.setReadOnly(true); + const readerId = await auroraTestUtility.queryInstanceId(client); + expect(readerId).not.toBe(initialWriterId); - await client.setReadOnly(true); - const currentId3 = await auroraTestUtility.queryInstanceId(client); - expect(currentId3).toStrictEqual(readerId); - expect(await auroraTestUtility.isDbInstanceWriter(currentId3)).toStrictEqual(false); - }, 1000000); - - it("test set read only false in read only transaction", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); + await client.setReadOnly(true); + const currentId0 = await auroraTestUtility.queryInstanceId(client); + expect(currentId0).toStrictEqual(readerId); - client.on("error", (error: any) => { - logger.debug(error.message); - }); + await client.setReadOnly(false); + const currentId1 = await auroraTestUtility.queryInstanceId(client); + expect(currentId1).toStrictEqual(initialWriterId); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + await client.setReadOnly(false); + const currentId2 = await auroraTestUtility.queryInstanceId(client); + expect(currentId2).toStrictEqual(initialWriterId); + + await client.setReadOnly(true); + const currentId3 = await auroraTestUtility.queryInstanceId(client); + expect(currentId3).toStrictEqual(readerId); + expect(await auroraTestUtility.isDbInstanceWriter(currentId3)).toStrictEqual(false); + }, + 1000000 + ); + + itIf( + "test set read only false in read only transaction", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await client.setReadOnly(true); + const initialReaderId = await auroraTestUtility.queryInstanceId(client); + expect(initialReaderId).not.toBe(initialWriterId); + + await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION READ ONLY"); // start transaction + await DriverHelper.executeQuery(env.engine, client, "SELECT 1"); - await client.setReadOnly(true); - const initialReaderId = await auroraTestUtility.queryInstanceId(client); - expect(initialReaderId).not.toBe(initialWriterId); + try { + await client.setReadOnly(false); + } catch (error: any) { + logger.debug(error.message); + if (!(error instanceof AwsWrapperError)) { + throw new Error("Resulting error type incorrect"); + } + } + const currentConnectionId0 = await auroraTestUtility.queryInstanceId(client); + expect(currentConnectionId0).toStrictEqual(initialReaderId); - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION READ ONLY"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "SELECT 1"); + await DriverHelper.executeQuery(env.engine, client, "COMMIT"); - try { await client.setReadOnly(false); - } catch (error: any) { - logger.debug(error.message); - if (!(error instanceof AwsWrapperError)) { - throw new Error("Resulting error type incorrect"); + const currentConnectionId1 = await auroraTestUtility.queryInstanceId(client); + expect(currentConnectionId1).toStrictEqual(initialWriterId); + }, + 1000000 + ); + + itIf( + "test set read only true in transaction", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); + + await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); + + await client.setReadOnly(true); + const currentReaderId = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId).toStrictEqual(initialWriterId); + + await DriverHelper.executeQuery(env.engine, client, "COMMIT"); + + // Assert that 1 row has been inserted to the table. + const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); + if (env.engine === DatabaseEngine.PG) { + expect((result as QueryResult).rows[0]["count"]).toBe("1"); + } else if (env.engine === DatabaseEngine.MYSQL) { + expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(1); } - } - const currentConnectionId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentConnectionId0).toStrictEqual(initialReaderId); - - await DriverHelper.executeQuery(env.engine, client, "COMMIT"); - - await client.setReadOnly(false); - const currentConnectionId1 = await auroraTestUtility.queryInstanceId(client); - expect(currentConnectionId1).toStrictEqual(initialWriterId); - }, 1000000); - - it("test set read only true in transaction", async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - - await client.setReadOnly(true); - const currentReaderId = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId).toStrictEqual(initialWriterId); - - await DriverHelper.executeQuery(env.engine, client, "COMMIT"); - - // Assert that 1 row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("1"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(1); - } - await client.setReadOnly(false); - const currentConnectionId1 = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId1)).toStrictEqual(true); - expect(currentConnectionId1).toStrictEqual(initialWriterId); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, 1000000); - - it("test set read only all instances down", async () => { - const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(config); - - client.on("error", (error: any) => { - logger.debug(error.message); - }); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await client.setReadOnly(true); - const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId0).not.toBe(initialWriterId); - - // Kill all instances - await ProxyHelper.disableAllConnectivity(env.engine); - await expect(async () => { await client.setReadOnly(false); - }).rejects.toThrow(); - }, 1000000); - - it("test set read only all readers down", async () => { - // Connect to writer instance - const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(config); - client.on("error", (err: any) => { - logger.debug(err); - }); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Kill all reader instances - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + const currentConnectionId1 = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId1)).toStrictEqual(true); + expect(currentConnectionId1).toStrictEqual(initialWriterId); + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + }, + 1000000 + ); + + itIf( + "test set read only all instances down", + async () => { + const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + client = initClientFunc(config); + + client.on("error", (error: any) => { + logger.debug(error.message); + }); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await client.setReadOnly(true); + const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId0).not.toBe(initialWriterId); + + // Kill all instances + await ProxyHelper.disableAllConnectivity(env.engine); + await expect(async () => { + await client.setReadOnly(false); + }).rejects.toThrow(); + }, + 1000000 + ); + + itIf( + "test set read only all readers down", + async () => { + // Connect to writer instance + const config = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + client = initClientFunc(config); + client.on("error", (err: any) => { + logger.debug(err); + }); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Kill all reader instances + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + } } - } - - await client.setReadOnly(true); - const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId0).toStrictEqual(initialWriterId); - await client.setReadOnly(false); - const currentReaderId1 = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId1).toStrictEqual(initialWriterId); + await client.setReadOnly(true); + const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId0).toStrictEqual(initialWriterId); - await ProxyHelper.enableAllConnectivity(); - await client.setReadOnly(true); - const currentReaderId2 = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId2).not.toBe(initialWriterId); - }, 1000000); - - it("test failover to new writer set read only true false", async () => { - // Connect to writer instance - const writerConfig = await initConfigWithFailover(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - client.on("error", (err: any) => { - logger.debug(err); - }); - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Kill all reader instances - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + await client.setReadOnly(false); + const currentReaderId1 = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId1).toStrictEqual(initialWriterId); + + await ProxyHelper.enableAllConnectivity(); + await client.setReadOnly(true); + const currentReaderId2 = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId2).not.toBe(initialWriterId); + }, + 1000000 + ); + + itIf( + "test failover to new writer set read only true false", + async () => { + // Connect to writer instance + const writerConfig = await initConfigWithFailover( + env.proxyDatabaseInfo.writerInstanceEndpoint, + env.proxyDatabaseInfo.instanceEndpointPort, + true + ); + client = initClientFunc(writerConfig); + client.on("error", (err: any) => { + logger.debug(err); + }); + await client.connect(); + + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Kill all reader instances + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + } } - } - // Force internal reader connection to the writer instance - await client.setReadOnly(true); - const currentId0 = await auroraTestUtility.queryInstanceId(client); + // Force internal reader connection to the writer instance + await client.setReadOnly(true); + const currentId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentId0).toStrictEqual(initialWriterId); + expect(currentId0).toStrictEqual(initialWriterId); - await client.setReadOnly(false); + await client.setReadOnly(false); - await ProxyHelper.enableAllConnectivity(); + await ProxyHelper.enableAllConnectivity(); - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - await TestEnvironment.updateWriter(); + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + await TestEnvironment.updateWriter(); - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - const newWriterId = await auroraTestUtility.queryInstanceId(client); + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + const newWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(newWriterId)).toStrictEqual(true); - expect(newWriterId).not.toBe(initialWriterId); + expect(await auroraTestUtility.isDbInstanceWriter(newWriterId)).toStrictEqual(true); + expect(newWriterId).not.toBe(initialWriterId); - await client.setReadOnly(true); - const currentReaderId = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId).not.toBe(newWriterId); + await client.setReadOnly(true); + const currentReaderId = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId).not.toBe(newWriterId); - await client.setReadOnly(false); - const currentId = await auroraTestUtility.queryInstanceId(client); - expect(currentId).toStrictEqual(newWriterId); - }, 1000000); - - it("test failover to new reader set read only false true", async () => { - // Connect to writer instance - const writerConfig = await initConfigWithFailover(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - writerConfig["failoverMode"] = "reader-or-writer"; - client = initClientFunc(writerConfig); - client.on("error", (err: any) => { - logger.debug(err); - }); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await client.setReadOnly(true); - - const readerConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(readerConnectionId).not.toBe(initialWriterId); - // Get a reader instance - let otherReaderId; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== readerConnectionId && host.instanceId !== initialWriterId) { - otherReaderId = host.instanceId; - break; + await client.setReadOnly(false); + const currentId = await auroraTestUtility.queryInstanceId(client); + expect(currentId).toStrictEqual(newWriterId); + }, + 1000000 + ); + + itIf( + "test failover to new reader set read only false true", + async () => { + // Connect to writer instance + const writerConfig = await initConfigWithFailover( + env.proxyDatabaseInfo.writerInstanceEndpoint, + env.proxyDatabaseInfo.instanceEndpointPort, + true + ); + writerConfig["failoverMode"] = "reader-or-writer"; + client = initClientFunc(writerConfig); + client.on("error", (err: any) => { + logger.debug(err); + }); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await client.setReadOnly(true); + + const readerConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(readerConnectionId).not.toBe(initialWriterId); + // Get a reader instance + let otherReaderId; + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== readerConnectionId && host.instanceId !== initialWriterId) { + otherReaderId = host.instanceId; + break; + } } - } - if (!otherReaderId) { - throw new Error("Could not find a reader instance"); - } - // Kill all instances except one other reader - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== otherReaderId) { - await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + if (!otherReaderId) { + throw new Error("Could not find a reader instance"); } - } - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); - - expect(currentReaderId0).toStrictEqual(otherReaderId); - expect(currentReaderId0).not.toBe(readerConnectionId); - - await ProxyHelper.enableAllConnectivity(); - await client.setReadOnly(false); + // Kill all instances except one other reader + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== otherReaderId) { + await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + } + } + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); - const currentId = await auroraTestUtility.queryInstanceId(client); - expect(currentId).toStrictEqual(initialWriterId); + const currentReaderId0 = await auroraTestUtility.queryInstanceId(client); - await client.setReadOnly(true); + expect(currentReaderId0).toStrictEqual(otherReaderId); + expect(currentReaderId0).not.toBe(readerConnectionId); - const currentReaderId2 = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId2).toStrictEqual(otherReaderId); - }, 1000000); + await ProxyHelper.enableAllConnectivity(); + await client.setReadOnly(false); - it("test failover reader to writer set read only true false", async () => { - // Connect to writer instance - const writerConfig = await initConfigWithFailover(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - client.on("error", (err: any) => { - logger.debug(err); - }); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - await client.setReadOnly(true); - - const currentReaderId = await auroraTestUtility.queryInstanceId(client); - expect(currentReaderId).not.toBe(initialWriterId); - - // Kill all reader instances - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + const currentId = await auroraTestUtility.queryInstanceId(client); + expect(currentId).toStrictEqual(initialWriterId); + + await client.setReadOnly(true); + + const currentReaderId2 = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId2).toStrictEqual(otherReaderId); + }, + 1000000 + ); + + itIf( + "test failover reader to writer set read only true false", + async () => { + // Connect to writer instance + const writerConfig = await initConfigWithFailover( + env.proxyDatabaseInfo.writerInstanceEndpoint, + env.proxyDatabaseInfo.instanceEndpointPort, + true + ); + client = initClientFunc(writerConfig); + client.on("error", (err: any) => { + logger.debug(err); + }); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + await client.setReadOnly(true); + + const currentReaderId = await auroraTestUtility.queryInstanceId(client); + expect(currentReaderId).not.toBe(initialWriterId); + + // Kill all reader instances + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + await ProxyHelper.disableConnectivity(env.engine, host.instanceId); + } } - } - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); - const currentId0 = await auroraTestUtility.queryInstanceId(client); + const currentId0 = await auroraTestUtility.queryInstanceId(client); - expect(currentId0).toStrictEqual(initialWriterId); + expect(currentId0).toStrictEqual(initialWriterId); - await ProxyHelper.enableAllConnectivity(); - await client.setReadOnly(true); + await ProxyHelper.enableAllConnectivity(); + await client.setReadOnly(true); - const currentId1 = await auroraTestUtility.queryInstanceId(client); - expect(currentId1).not.toBe(initialWriterId); + const currentId1 = await auroraTestUtility.queryInstanceId(client); + expect(currentId1).not.toBe(initialWriterId); - await client.setReadOnly(false); + await client.setReadOnly(false); - const currentId2 = await auroraTestUtility.queryInstanceId(client); - expect(currentId2).toStrictEqual(initialWriterId); - }, 1000000); + const currentId2 = await auroraTestUtility.queryInstanceId(client); + expect(currentId2).toStrictEqual(initialWriterId); + }, + 1000000 + ); }); diff --git a/tests/integration/container/tests/utils/driver_helper.ts b/tests/integration/container/tests/utils/driver_helper.ts index d01c5411b..a5cefe34e 100644 --- a/tests/integration/container/tests/utils/driver_helper.ts +++ b/tests/integration/container/tests/utils/driver_helper.ts @@ -81,26 +81,28 @@ export class DriverHelper { } } - static async executeQuery(engine: DatabaseEngine, client: AwsClient, sql: string) { + static async executeQuery(engine: DatabaseEngine, client: AwsClient, sql: string, timeoutValue?: number) { switch (engine) { case DatabaseEngine.PG: return await (client as AwsPGClient).query(sql); case DatabaseEngine.MYSQL: - return await (client as AwsMySQLClient).query({ sql: sql, timeout: 10000 }); + return await (client as AwsMySQLClient).query({ sql: sql, timeout: timeoutValue }); default: throw new Error("invalid engine"); } } - static addDriverSpecificConfiguration(props: any, engine: DatabaseEngine) { - switch (engine) { - case DatabaseEngine.PG: - props["query_timeout"] = 10000; - break; - case DatabaseEngine.MYSQL: - break; - default: - break; + static addDriverSpecificConfiguration(props: any, engine: DatabaseEngine, performance: boolean = false) { + if (engine === DatabaseEngine.PG && !performance) { + props["query_timeout"] = 10000; + } else if (engine === DatabaseEngine.PG && performance) { + props["query_timeout"] = 120000; + props["connectionTimeoutMillis"] = 400; + props["monitoring_query_timeout"] = 400; + } else if (engine === DatabaseEngine.MYSQL && performance) { + props["connectTimeout"] = 400; + props["monitoring_internal_query_timeout"] = 400; + props["internal_query_timeout"] = 120000; } return props; diff --git a/tests/integration/container/tests/utils/proxy_helper.ts b/tests/integration/container/tests/utils/proxy_helper.ts index e96dc2866..11a06d504 100644 --- a/tests/integration/container/tests/utils/proxy_helper.ts +++ b/tests/integration/container/tests/utils/proxy_helper.ts @@ -22,7 +22,9 @@ import { DatabaseEngine } from "./database_engine"; export class ProxyHelper { static async disableAllConnectivity(engine: DatabaseEngine) { const env = await TestEnvironment.getCurrent(); - env.proxyInfos.forEach((p) => ProxyHelper.disableProxyConnectivity(p)); + for (let i = 0; i < env.proxyInfos.length; i++) { + await ProxyHelper.disableProxyConnectivity(env.proxyInfos[i]); + } } static async enableAllConnectivity() { diff --git a/tests/integration/container/tests/utils/test_environment.ts b/tests/integration/container/tests/utils/test_environment.ts index 6496e49c8..548a8ec33 100644 --- a/tests/integration/container/tests/utils/test_environment.ts +++ b/tests/integration/container/tests/utils/test_environment.ts @@ -69,48 +69,50 @@ export class TestEnvironment { } static async initProxies(environment: TestEnvironment) { - environment.proxies = {}; - const proxyControlPort: number = environment.proxyDatabaseInfo.controlPort; - for (let i = 0; i < environment.proxyInstances.length; i++) { - const instance = environment.proxyInstances[i]; - if (instance.host === undefined || instance.instanceId === undefined) { - throw new Error("no valid host"); + if (environment.features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED)) { + environment.proxies = {}; + const proxyControlPort: number = environment.proxyDatabaseInfo.controlPort; + for (let i = 0; i < environment.proxyInstances.length; i++) { + const instance = environment.proxyInstances[i]; + if (instance.host === undefined || instance.instanceId === undefined) { + throw new Error("no valid host"); + } + + const client = new Toxiproxy(TestEnvironment.createProxyUrl(instance.host, proxyControlPort)); + const proxies = await client.getAll(); + + const host = environment.instances[i].host; + if (host === undefined) { + throw new Error("no host"); + } + + environment.proxies[instance.instanceId] = new ProxyInfo(proxies[environment.instances[i].url], host, proxyControlPort); } - const client = new Toxiproxy(TestEnvironment.createProxyUrl(instance.host, proxyControlPort)); - const proxies = await client.getAll(); - - const host = environment.instances[i].host; - if (host === undefined) { - throw new Error("no host"); - } - - environment.proxies[instance.instanceId] = new ProxyInfo(proxies[environment.instances[i].url], host, proxyControlPort); - } - - if (environment.proxyDatabaseInfo.clusterEndpoint !== undefined) { - const client = new Toxiproxy(TestEnvironment.createProxyUrl(environment.proxyDatabaseInfo.clusterEndpoint, proxyControlPort)); - const proxy = await client.get(`${environment.databaseInfo.clusterEndpoint}:${environment.databaseInfo.clusterEndpointPort}`); - - if (proxy !== undefined) { - environment.proxies[environment.proxyDatabaseInfo.clusterEndpoint] = new ProxyInfo( - proxy, - environment.databaseInfo.clusterEndpoint, - proxyControlPort - ); + if (environment.proxyDatabaseInfo.clusterEndpoint !== undefined) { + const client = new Toxiproxy(TestEnvironment.createProxyUrl(environment.proxyDatabaseInfo.clusterEndpoint, proxyControlPort)); + const proxy = await client.get(`${environment.databaseInfo.clusterEndpoint}:${environment.databaseInfo.clusterEndpointPort}`); + + if (proxy !== undefined) { + environment.proxies[environment.proxyDatabaseInfo.clusterEndpoint] = new ProxyInfo( + proxy, + environment.databaseInfo.clusterEndpoint, + proxyControlPort + ); + } } - } - - if (environment.proxyDatabaseInfo.clusterReadOnlyEndpoint !== undefined) { - const client = new Toxiproxy(TestEnvironment.createProxyUrl(environment.proxyDatabaseInfo.clusterReadOnlyEndpoint, proxyControlPort)); - const proxy = await client.get(`${environment.databaseInfo.clusterReadOnlyEndpoint}:${environment.databaseInfo.clusterReadOnlyEndpointPort}`); - if (proxy !== undefined) { - environment.proxies[environment.databaseInfo.clusterReadOnlyEndpoint] = new ProxyInfo( - proxy, - environment.databaseInfo.clusterReadOnlyEndpoint, - proxyControlPort - ); + if (environment.proxyDatabaseInfo.clusterReadOnlyEndpoint !== undefined) { + const client = new Toxiproxy(TestEnvironment.createProxyUrl(environment.proxyDatabaseInfo.clusterReadOnlyEndpoint, proxyControlPort)); + const proxy = await client.get(`${environment.databaseInfo.clusterReadOnlyEndpoint}:${environment.databaseInfo.clusterReadOnlyEndpointPort}`); + + if (proxy !== undefined) { + environment.proxies[environment.databaseInfo.clusterReadOnlyEndpoint] = new ProxyInfo( + proxy, + environment.databaseInfo.clusterReadOnlyEndpoint, + proxyControlPort + ); + } } } } diff --git a/tests/integration/host/build.gradle.kts b/tests/integration/host/build.gradle.kts index 6cda14906..fb93ca9c4 100644 --- a/tests/integration/host/build.gradle.kts +++ b/tests/integration/host/build.gradle.kts @@ -108,6 +108,34 @@ tasks.register("test-aurora-mysql") { } } +tasks.register("test-all-aurora-performance") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + systemProperty("exclude-docker", "true") + } +} + +tasks.register("test-aurora-pg-performance") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + systemProperty("exclude-docker", "true") + systemProperty("exclude-mysql-driver", "true") + systemProperty("exclude-mysql-engine", "true") + } +} + +tasks.register("test-aurora-mysql-performance") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.runTests") + doFirst { + systemProperty("exclude-docker", "true") + systemProperty("exclude-pg-driver", "true") + systemProperty("exclude-pg-engine", "true") + } +} + // Debug tasks.register("debug-all-environments") { @@ -157,3 +185,24 @@ tasks.register("debug-aurora-mysql") { systemProperty("exclude-pg-engine", "true") } } + +tasks.register("debug-aurora-pg-performance") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.debugTests") + doFirst { + systemProperty("exclude-docker", "true") + systemProperty("exclude-mysql-driver", "true") + systemProperty("exclude-mysql-engine", "true") + } +} + +tasks.register("debug-aurora-mysql-performance") { + group = "verification" + filter.includeTestsMatching("integration.host.TestRunner.debugTests") + doFirst { + systemProperty("exclude-docker", "true") + systemProperty("exclude-pg-driver", "true") + systemProperty("exclude-pg-engine", "true") + } +} + diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java index 490d4a210..aebd8d1c4 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentProvider.java @@ -59,8 +59,6 @@ public Stream provideTestTemplateInvocationContex final boolean excludeIam = Boolean.parseBoolean(System.getProperty("exclude-iam", "false")); final boolean excludeSecretsManager = Boolean.parseBoolean(System.getProperty("exclude-secrets-manager", "false")); - final boolean excludePython38 = Boolean.parseBoolean(System.getProperty("exclude-python-38", "false")); - final boolean excludePython311 = Boolean.parseBoolean(System.getProperty("exclude-python-311", "false")); final boolean testAutoscalingOnly = Boolean.parseBoolean(System.getProperty("test-autoscaling", "false")); if (!excludeDocker) { diff --git a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java index 3519685fb..b135eaf8b 100644 --- a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java +++ b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java @@ -171,6 +171,7 @@ public T withFixedExposedPort(int hostPort, int containerPort) { .withFileSystemBind("../../../package-lock.json", "/app/package-lock.json", BindMode.READ_ONLY) .withFileSystemBind("../../../tests/integration/container", "/app/tests/integration/container", BindMode.READ_WRITE) + .withFileSystemBind("../../../tests/integration/container/reports", "/app/build/reports/tests", BindMode.READ_WRITE) // some tests may write some files here .withPrivilegedMode(true); // Required to control Linux core settings like TcpKeepAlive } diff --git a/tests/unit/monitor_connection_context.test.ts b/tests/unit/monitor_connection_context.test.ts index 3152391be..3466dc885 100644 --- a/tests/unit/monitor_connection_context.test.ts +++ b/tests/unit/monitor_connection_context.test.ts @@ -17,7 +17,9 @@ import { instance, mock, spy, verify } from "ts-mockito"; import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; +import { PluginService } from "../../common/lib/plugin_service"; +const mockPluginService: PluginService = mock(PluginService); const mockMonitor = mock(MonitorImpl); const mockTargetClient = { end() { @@ -39,7 +41,8 @@ describe("monitor connection context test", () => { null, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT + FAILURE_DETECTION_COUNT, + instance(mockPluginService) ); }); @@ -57,19 +60,6 @@ describe("monitor connection context test", () => { expect(context.failureCount).toBe(1); }); - it("isHostUnhealthy exceeds failure detection count - return true", async () => { - const expectedFailureCount = FAILURE_DETECTION_COUNT + 1; - context.failureCount = FAILURE_DETECTION_COUNT; - context.resetInvalidHostStartTimeNano(); - - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", false, currentTimeNano, currentTimeNano); - - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(expectedFailureCount); - expect(context.isInvalidHostStartTimeDefined()).toBe(true); - }); - it("isHostUnhealthy exceeds failure detection count", async () => { let currentTimeNano = Date.now(); context.failureCount = 0; @@ -80,7 +70,11 @@ describe("monitor connection context test", () => { const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - expect(context.isHostUnhealthy).toBe(false); + if (i >= FAILURE_DETECTION_COUNT - 1) { + expect(context.isHostUnhealthy).toBe(true); + } else { + expect(context.isHostUnhealthy).toBe(false); + } currentTimeNano += VALIDATION_INTERVAL_MILLIS * 1_000_000; } @@ -118,7 +112,8 @@ describe("monitor connection context test", () => { mockTargetClient, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT + FAILURE_DETECTION_COUNT, + instance(mockPluginService) ); await context.abortConnection();