Skip to content

Commit

Permalink
fix failover
Browse files Browse the repository at this point in the history
  • Loading branch information
joyc-bq committed Jan 6, 2025
1 parent f0aba96 commit 63a5924
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 27 deletions.
2 changes: 1 addition & 1 deletion common/lib/host_list_provider/rds_host_list_provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ export class RdsHostListProvider implements DynamicHostListProvider {
if (client) {
return await dialect.getHostRole(client);
} else {
throw new AwsWrapperError(Messages.get("AwsClient targetClient not defined."));
throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined"));
}
}

Expand Down
19 changes: 17 additions & 2 deletions common/lib/plugins/failover/failover_plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin {
await this.pluginService.setCurrentClient(result.client, result.newHost);
this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases));
await this.updateTopology(true);
this.failoverReaderSuccessCounter.inc();
await this.throwFailoverSuccessError();
} catch (error: any) {
this.failoverReaderFailedCounter.inc();
throw error;
Expand All @@ -406,6 +406,21 @@ export class FailoverPlugin extends AbstractConnectionPlugin {
}
}

async throwFailoverSuccessError() {
if (this._isInTransaction || this.pluginService.isInTransaction()) {
this.pluginService.setInTransaction(false);

// "Transaction resolution unknown. Please re-configure session state if required and try
// restarting transaction."
logger.debug(Messages.get("Failover.transactionResolutionUnknownError"));
throw new TransactionResolutionUnknownError(Messages.get("Failover.transactionResolutionUnknownError"));
} else {
// "The active SQL connection has changed due to a connection failure. Please re-configure
// session state if required."
throw new FailoverSuccessError(Messages.get("Failover.connectionChangedError"));
}
}

async failoverWriter() {
logger.debug(Messages.get("Failover.startWriterFailover"));

Expand Down Expand Up @@ -440,7 +455,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin {
await this.pluginService.setCurrentClient(result.client, writerHostInfo);
logger.debug(Messages.get("Failover.establishedConnection", this.pluginService.getCurrentHostInfo()?.host ?? ""));
await this.pluginService.refreshHostList();
this.failoverWriterSuccessCounter.inc();
await this.throwFailoverSuccessError();
} catch (error: any) {
this.failoverWriterFailedCounter.inc();
throw error;
Expand Down
32 changes: 10 additions & 22 deletions common/lib/plugins/failover/reader_failover_handler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,23 +113,14 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler
// ensure new connection is to a reader host
await this.pluginService.refreshHostList();
const topology = this.pluginService.getHosts();

for (let i = 0; i < topology.length; i++) {
const host = topology[i];
if (host.host === result.newHost.host) {
// found new connection host in the latest topology
if (host.role === HostRole.READER) {
return result;
}
try {
if ((await this.pluginService.getHostRole(result.client)) !== HostRole.READER) {
return result;
}
} catch (error) {
logger.debug(Messages.get("ClusterAwareReaderFailoverHandler.errorGettingHostRole", error.message));
}

// New host is not found in the latest topology. There are few possible reasons for that.
// - Host is not yet presented in the topology due to failover process in progress
// - Host is in the topology but its role isn't a
// READER (that is not acceptable option due to this.strictReader setting)
// Need to continue this loop and to make another try to connect to a reader.

try {
await this.pluginService.abortTargetClient(result.client);
} catch (error) {
Expand Down Expand Up @@ -276,7 +267,9 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler

const hostsByPriority: HostInfo[] = [...activeReaders];
const numReaders: number = activeReaders.length + downHostList.length;
if (writerHost && (!this.enableFailoverStrictReader || numReaders === 0)) {
// Since the writer instance may change during failover, the original writer is likely now a reader. We will include
// it and then verify the role once connected if using "strict-reader".
if (writerHost || numReaders === 0) {
hostsByPriority.push(writerHost);
}
hostsByPriority.push(...downHostList);
Expand Down Expand Up @@ -325,13 +318,8 @@ class ConnectionAttemptTask {

// ensure that new connection is a connection to a reader node
try {
// eslint-disable-next-line no-constant-condition
while (1) {
await this.pluginService.forceRefreshHostList();
if ((await this.pluginService.getHostRole(this.targetClient)) === HostRole.READER) {
break;
}
await sleep(1000);
if ((await this.pluginService.getHostRole(this.targetClient)) === HostRole.READER) {
return this.targetClient;
}
} catch (error: any) {
logger.debug(Messages.get("ClusterAwareReaderFailoverHandler.errorGettingHostRole", error.message));
Expand Down
4 changes: 2 additions & 2 deletions common/lib/utils/locales/en.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
"ClusterAwareWriterFailoverHandler.standaloneHost": "[TaskB] Host %s is not yet connected to a cluster. The cluster is still being reconfigured.",
"ClusterAwareWriterFailoverHandler.taskBAttemptConnectionToNewWriter": "[TaskB] Trying to connect to a new writer: '%s'",
"ClusterAwareWriterFailoverHandler.alreadyWriter": "Current reader connection is actually a new writer connection.",
"ClusterAwareReaderFailoverHandler.errorGettingHostRole": "An error occurred while trying to determine the role of the reader candidate: ",
"ClusterAwareReaderFailoverHandler.errorGettingHostRole": "An error occurred while trying to determine the role of the reader candidate: %s.",
"Failover.TransactionResolutionUnknownError": "Transaction resolution unknown. Please re-configure session state if required and try restarting the transaction.",
"Failover.connectionChangedError": "The active SQL connection has changed due to a connection failure. Please re-configure session state if required.",
"Failover.parameterValue": "%s = %s",
Expand Down Expand Up @@ -195,5 +195,5 @@
"ConfigurationProfileBuilder.profileNameRequired": "Profile name is required.",
"ConfigurationProfileBuilder.canNotUpdateKnownPreset": "Can't add or update a built-in preset configuration profile '%s'.",
"AwsClient.configurationProfileNotFound": "Configuration profile '%s' not found.",
"AwsClient targetClient not defined.": "AwsClient targetClient not defined."
"AwsClient.targetClientNotDefined": "AwsClient targetClient not defined."
}

0 comments on commit 63a5924

Please sign in to comment.