Skip to content

Commit

Permalink
debug
Browse files Browse the repository at this point in the history
  • Loading branch information
@sophia-bq
sophia-bq committed Feb 5, 2025
1 parent 26c1e25 commit dd541e0
Show file tree
Hide file tree
Showing 6 changed files with 36 additions and 141 deletions.
5 changes: 5 additions & 0 deletions common/lib/plugins/failover/reader_failover_handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,12 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler
// Ensure new connection is to a reader host
await this.pluginService.refreshHostList();
try {
// debugging logging
const role = await this.pluginService.getHostRole(result.client);
logger.debug(`DBG failover1 host role failoverReader: ${role}`);
// end debug logging
if ((await this.pluginService.getHostRole(result.client)) !== HostRole.READER) {
logger.debug(`DBG treating as a reader??: ${role}`);
return result;
}
} catch (error) {
Expand Down
8 changes: 8 additions & 0 deletions common/lib/plugins/failover/writer_failover_handler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,10 @@ class ReconnectToWriterHandlerTask {
props.set(WrapperProperties.HOST.name, this.originalWriterHost.host);
this.currentClient = await this.pluginService.forceConnect(this.originalWriterHost, props);
await this.pluginService.forceRefreshHostList(this.currentClient);
// debugging logging
const role = await this.pluginService.getHostRole(this.currentClient);
logger.debug(`DBG failover1 host role failoverWriter call: ${role}`);
// end debug logging
latestTopology = this.pluginService.getHosts();
} catch (error) {
// Propagate errors that are not caused by network errors.
Expand Down Expand Up @@ -439,6 +443,10 @@ class WaitForNewWriterHandlerTask {
await this.callCloseClient(this.currentReaderTargetClient);
await this.callCloseClient(this.currentClient);
this.currentClient = targetClient;
// debugging logging
const role = await this.pluginService.getHostRole(writerCandidate);
logger.debug(`DBG failover1 host role failoverWriter connectToWriter: ${role}`);
// end debug logging
return true;
} catch (error) {
this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE);
Expand Down
11 changes: 11 additions & 0 deletions common/lib/plugins/failover2/failover2_plugin.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,9 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele
try {
const candidateClient: ClientWrapper = await this.createConnectionForHost(readerCandidate);
const role: HostRole = await this.pluginService.getHostRole(candidateClient);
// debugging logging
logger.debug(`DBG failover2 host role getReaderFailoverConnection remaining: ${role}`);
// end debug logging
if (role === HostRole.READER || this.failoverMode !== FailoverMode.STRICT_READER) {
if (role !== readerCandidate.role) {
// Update readerCandidate to reflect correct role.
Expand Down Expand Up @@ -348,6 +351,9 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele
try {
const candidateClient: ClientWrapper = await this.createConnectionForHost(originalWriter);
const role: HostRole = await this.pluginService.getHostRole(candidateClient);
// debugging logging
logger.debug(`DBG failover2 host role getReaderFailoverConnection writer: ${role}`);
// end debug logging
if (role === HostRole.READER || this.failoverMode != FailoverMode.STRICT_READER) {
const updatedHostInfo: HostInfo = this.pluginService.getHostInfoBuilder().copyFrom(originalWriter).withRole(role).build();
return new ReaderFailoverResult(candidateClient, updatedHostInfo, true);
Expand Down Expand Up @@ -400,6 +406,11 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele
this.logAndThrowError("Failover.unableToConnectToWriter");
}

// debugging logging
const role = await this.pluginService.getHostRole(writerCandidateClient);
logger.debug(`DBG failover2 host role failoverWriter: ${role}`);
// end debug logging

if ((await this.pluginService.getHostRole(writerCandidateClient)) !== HostRole.WRITER) {
try {
await writerCandidateClient.end();
Expand Down
4 changes: 4 additions & 0 deletions mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ export class RdsMultiAZMySQLDatabaseDialect extends MySQLDatabaseDialect impleme
}

async getHostRole(client: ClientWrapper): Promise<HostRole> {
// debugging logging
const query = await this.executeTopologyRelatedQuery(client, RdsMultiAZMySQLDatabaseDialect.IS_READER_QUERY);
logger.debug(`MySQL getHostRole query: ${query}`);
// end debug logging
return (await this.executeTopologyRelatedQuery(client, RdsMultiAZMySQLDatabaseDialect.IS_READER_QUERY)) ? HostRole.WRITER : HostRole.READER;
}

Expand Down
4 changes: 4 additions & 0 deletions pg/lib/dialect/rds_multi_az_pg_database_dialect.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,10 @@ export class RdsMultiAZPgDatabaseDialect extends PgDatabaseDialect implements To
}

async getHostRole(client: ClientWrapper): Promise<HostRole> {
// debugging logging
const query = await this.executeTopologyRelatedQuery(client, RdsMultiAZPgDatabaseDialect.IS_READER_QUERY);
logger.debug(`PG getHostRole query: ${query}`);
// end debug logging
return (await this.executeTopologyRelatedQuery(client, RdsMultiAZPgDatabaseDialect.IS_READER_QUERY)) ? HostRole.WRITER : HostRole.READER;
}

Expand Down
145 changes: 4 additions & 141 deletions tests/integration/container/tests/aurora_failover2.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,15 @@ import { TestEnvironment } from "./utils/test_environment";
import { DriverHelper } from "./utils/driver_helper";
import { AuroraTestUtility } from "./utils/aurora_test_utility";
import { FailoverSuccessError, TransactionResolutionUnknownError } from "../../../../common/lib/utils/errors";
import { DatabaseEngine } from "./utils/database_engine";
import { QueryResult } from "pg";
import { ProxyHelper } from "./utils/proxy_helper";
import { RdsUtils } from "../../../../common/lib/utils/rds_utils";
import { logger } from "../../../../common/logutils";
import { features, instanceCount } from "./config";
import { TestEnvironmentFeatures } from "./utils/test_environment_features";
import { PluginManager } from "../../../../common/lib";
import { DatabaseEngine } from "./utils/database_engine";
import { TransactionIsolationLevel } from "../../../../common/lib/utils/transaction_isolation_level";
import { RdsUtils } from "../../../../common/lib/utils/rds_utils";
import { QueryResult } from "pg";

const itIf =
features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) &&
Expand All @@ -36,7 +36,6 @@ const itIf =
? it
: it.skip;
const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip;
const itIfMinThreeInstance = instanceCount >= 3 ? itIf : it.skip;

let env: TestEnvironment;
let driver;
Expand Down Expand Up @@ -66,27 +65,6 @@ async function initDefaultConfig(host: string, port: number, connectToProxy: boo
return config;
}

async function initConfigWithRWSplitting(host: string, port: number, connectToProxy: boolean): Promise<any> {
let config: any = {
user: env.databaseInfo.username,
host: host,
database: env.databaseInfo.defaultDbName,
password: env.databaseInfo.password,
port: port,
plugins: "readWriteSplitting,failover2",
failoverTimeoutMs: 400000,
enableTelemetry: true,
telemetryTracesBackend: "OTLP",
telemetryMetricsBackend: "OTLP"
};

if (connectToProxy) {
config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix;
}
config = DriverHelper.addDriverSpecificConfiguration(config, env.engine);
return config;
}

describe("aurora failover2", () => {
beforeEach(async () => {
logger.info(`Test started: ${expect.getState().currentTestName}`);
Expand All @@ -97,8 +75,6 @@ describe("aurora failover2", () => {
initClientFunc = DriverHelper.getClient(driver);
await ProxyHelper.enableAllConnectivity();
await TestEnvironment.verifyClusterStatus();
await TestEnvironment.verifyAllInstancesHasRightState("available");
await TestEnvironment.verifyAllInstancesUp();

client = null;
secondaryClient = null;
Expand All @@ -112,6 +88,7 @@ describe("aurora failover2", () => {
// pass
}
}

if (secondaryClient !== null) {
try {
await secondaryClient.end();
Expand Down Expand Up @@ -281,118 +258,4 @@ describe("aurora failover2", () => {
},
1320000
);

itIfMinThreeInstance(
"test failover to new writer set read only true false",
async () => {
// Connect to writer instance
const writerConfig = await initConfigWithRWSplitting(
env.proxyDatabaseInfo.writerInstanceEndpoint,
env.proxyDatabaseInfo.instanceEndpointPort,
true
);
client = initClientFunc(writerConfig);
await client.connect();

const initialWriterId = await auroraTestUtility.queryInstanceId(client);
expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true);

// Kill all reader instances
for (const host of env.proxyDatabaseInfo.instances) {
if (host.instanceId && host.instanceId !== initialWriterId) {
await ProxyHelper.disableConnectivity(env.engine, host.instanceId);
}
}

// Force internal reader connection to the writer instance
await client.setReadOnly(true);
const currentId0 = await auroraTestUtility.queryInstanceId(client);

expect(currentId0).toStrictEqual(initialWriterId);
await client.setReadOnly(false);

await ProxyHelper.enableAllConnectivity();
// Crash instance 1 and nominate a new writer
await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged();
await TestEnvironment.verifyClusterStatus();

await expect(async () => {
await auroraTestUtility.queryInstanceId(client);
}).rejects.toThrow(FailoverSuccessError);
const newWriterId = await auroraTestUtility.queryInstanceId(client);
expect(await auroraTestUtility.isDbInstanceWriter(newWriterId)).toStrictEqual(true);
expect(newWriterId).not.toBe(initialWriterId);

await client.setReadOnly(true);
const currentReaderId = await auroraTestUtility.queryInstanceId(client);
expect(currentReaderId).not.toBe(newWriterId);

await client.setReadOnly(false);
const currentId = await auroraTestUtility.queryInstanceId(client);
expect(currentId).toStrictEqual(newWriterId);
},
1320000
);

itIfMinThreeInstance(
"test failover to new reader set read only false true",
async () => {
// Connect to writer instance
const writerConfig = await initConfigWithRWSplitting(
env.proxyDatabaseInfo.writerInstanceEndpoint,
env.proxyDatabaseInfo.instanceEndpointPort,
true
);
writerConfig["failoverMode"] = "reader-or-writer";
client = initClientFunc(writerConfig);

await client.connect();
const initialWriterId = await auroraTestUtility.queryInstanceId(client);
expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true);
await client.setReadOnly(true);

const readerConnectionId = await auroraTestUtility.queryInstanceId(client);
expect(readerConnectionId).not.toBe(initialWriterId);

// Get a reader instance
let otherReaderId;
for (const host of env.proxyDatabaseInfo.instances) {
if (host.instanceId && host.instanceId !== readerConnectionId && host.instanceId !== initialWriterId) {
otherReaderId = host.instanceId;
break;
}
}

if (!otherReaderId) {
throw new Error("Could not find a reader instance");
}
// Kill all instances except one other reader
for (const host of env.proxyDatabaseInfo.instances) {
if (host.instanceId && host.instanceId !== otherReaderId) {
await ProxyHelper.disableConnectivity(env.engine, host.instanceId);
}
}

await expect(async () => {
await auroraTestUtility.queryInstanceId(client);
}).rejects.toThrow(FailoverSuccessError);

const currentReaderId0 = await auroraTestUtility.queryInstanceId(client);

expect(currentReaderId0).toStrictEqual(otherReaderId);
expect(currentReaderId0).not.toBe(readerConnectionId);

await ProxyHelper.enableAllConnectivity();
await client.setReadOnly(false);

const currentId = await auroraTestUtility.queryInstanceId(client);
expect(currentId).toStrictEqual(initialWriterId);

await client.setReadOnly(true);

const currentReaderId2 = await auroraTestUtility.queryInstanceId(client);
expect(currentReaderId2).toStrictEqual(otherReaderId);
},
1320000
);
});

0 comments on commit dd541e0

Please sign in to comment.