From def178bda26c58cc71dd31917c03331d6701ec5a Mon Sep 17 00:00:00 2001 From: Karen Chen <64801825+karenc-bq@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:47:01 -0800 Subject: [PATCH 01/20] chore: run workflows on dev/v3 branch --- .github/workflows/integration_tests.yml | 1 + .github/workflows/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index db65f2f6..4b99a950 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - dev/v3 paths-ignore: - '**/*.md' - '**/*.jpg' diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 02c8389b..69a7d034 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - dev/v3 pull_request: branches: - "*" From 00203f8c3b832f4228c71d573ca2f8187c452c84 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Thu, 19 Feb 2026 17:20:05 -0800 Subject: [PATCH 02/20] chore: move read write splitting files into a read_write_splitting directory (#609) --- common/lib/connection_plugin_chain_builder.ts | 2 +- .../read_write_splitting_plugin.ts | 62 ++++++++++--------- .../read_write_splitting_plugin_factory.ts | 36 +++++++++++ .../read_write_splitting_plugin_factory.ts | 36 ----------- .../profile/driver_configuration_profiles.ts | 2 +- index.ts | 2 +- tests/unit/read_write_splitting.test.ts | 2 +- 7 files changed, 73 insertions(+), 69 deletions(-) rename common/lib/plugins/{ => read_write_splitting}/read_write_splitting_plugin.ts (90%) create mode 100644 common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts delete mode 100644 common/lib/plugins/read_write_splitting_plugin_factory.ts diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 9cc01baa..9c399dd7 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -29,7 +29,7 @@ import { FailoverPluginFactory } from "./plugins/failover/failover_plugin_factor import { Failover2PluginFactory } from "./plugins/failover2/failover2_plugin_factory"; import { StaleDnsPluginFactory } from "./plugins/stale_dns/stale_dns_plugin_factory"; import { FederatedAuthPluginFactory } from "./plugins/federated_auth/federated_auth_plugin_factory"; -import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting_plugin_factory"; +import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { OktaAuthPluginFactory } from "./plugins/federated_auth/okta_auth_plugin_factory"; import { HostMonitoringPluginFactory } from "./plugins/efm/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "./plugins/aurora_initial_connection_strategy_plugin_factory"; diff --git a/common/lib/plugins/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts similarity index 90% rename from common/lib/plugins/read_write_splitting_plugin.ts rename to common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts index bb30b232..c48fd009 100644 --- a/common/lib/plugins/read_write_splitting_plugin.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts @@ -1,33 +1,37 @@ /* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { AbstractConnectionPlugin } from "../abstract_connection_plugin"; -import { HostInfo, FailoverError, HostRole } from "../"; -import { PluginService } from "../plugin_service"; -import { HostListProviderService } from "../host_list_provider_service"; -import { OldConnectionSuggestionAction } from "../old_connection_suggestion_action"; -import { HostChangeOptions } from "../host_change_options"; -import { WrapperProperties } from "../wrapper_property"; -import { Messages } from "../utils/messages"; -import { logger } from "../../logutils"; -import { SqlMethodUtils } from "../utils/sql_method_utils"; -import { ClientWrapper } from "../client_wrapper"; -import { getWriter, logAndThrowError } from "../utils/utils"; -import { CanReleaseResources } from "../can_release_resources"; -import { PoolClientWrapper } from "../pool_client_wrapper"; + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; +import { + HostInfo, + FailoverError, + HostRole +} from "../../index"; +import { PluginService } from "../../plugin_service"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; +import { HostChangeOptions } from "../../host_change_options"; +import { WrapperProperties } from "../../wrapper_property"; +import { Messages } from "../../utils/messages"; +import { logger } from "../../../logutils"; +import { SqlMethodUtils } from "../../utils/sql_method_utils"; +import { ClientWrapper } from "../../client_wrapper"; +import { getWriter, logAndThrowError } from "../../utils/utils"; +import { CanReleaseResources } from "../../can_release_resources"; +import { PoolClientWrapper } from "../../pool_client_wrapper"; export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implements CanReleaseResources { private static readonly subscribedMethods: Set = new Set(["initHostProvider", "connect", "notifyConnectionChanged", "query"]); diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts new file mode 100644 index 00000000..881b59db --- /dev/null +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { PluginService } from "../../plugin_service"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; + +export class ReadWriteSplittingPluginFactory extends ConnectionPluginFactory { + private static readWriteSplittingPlugin: any; + + async getInstance(pluginService: PluginService, properties: Map): Promise { + try { + if (!ReadWriteSplittingPluginFactory.readWriteSplittingPlugin) { + ReadWriteSplittingPluginFactory.readWriteSplittingPlugin = await import("./read_write_splitting_plugin"); + } + return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(pluginService, properties); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "readWriteSplittingPlugin")); + } + } +} diff --git a/common/lib/plugins/read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting_plugin_factory.ts deleted file mode 100644 index 18fd230b..00000000 --- a/common/lib/plugins/read_write_splitting_plugin_factory.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; -import { ConnectionPlugin } from "../connection_plugin"; -import { AwsWrapperError } from "../utils/errors"; -import { Messages } from "../utils/messages"; - -export class ReadWriteSplittingPluginFactory extends ConnectionPluginFactory { - private static readWriteSplittingPlugin: any; - - async getInstance(pluginService: PluginService, properties: Map): Promise { - try { - if (!ReadWriteSplittingPluginFactory.readWriteSplittingPlugin) { - ReadWriteSplittingPluginFactory.readWriteSplittingPlugin = await import("./read_write_splitting_plugin"); - } - return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(pluginService, properties); - } catch (error: any) { - throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "readWriteSplittingPlugin")); - } - } -} diff --git a/common/lib/profile/driver_configuration_profiles.ts b/common/lib/profile/driver_configuration_profiles.ts index ee387b82..958d5dea 100644 --- a/common/lib/profile/driver_configuration_profiles.ts +++ b/common/lib/profile/driver_configuration_profiles.ts @@ -20,7 +20,7 @@ import { WrapperProperties } from "../wrapper_property"; import { HostMonitoringPluginFactory } from "../plugins/efm/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "../plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "../plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; -import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting_plugin_factory"; +import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { FailoverPluginFactory } from "../plugins/failover/failover_plugin_factory"; import { InternalPooledConnectionProvider } from "../internal_pooled_connection_provider"; import { AwsPoolConfig } from "../aws_pool_config"; diff --git a/index.ts b/index.ts index 067cf6e0..eb152e55 100644 --- a/index.ts +++ b/index.ts @@ -27,7 +27,7 @@ export { ConnectTimePlugin } from "./common/lib/plugins/connect_time_plugin"; export { ExecuteTimePlugin } from "./common/lib/plugins/execute_time_plugin"; export { AuroraInitialConnectionStrategyPlugin } from "./common/lib/plugins/aurora_initial_connection_strategy_plugin"; export { DefaultPlugin } from "./common/lib/plugins/default_plugin"; -export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting_plugin"; +export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; export { FailoverPlugin } from "./common/lib/plugins/failover/failover_plugin"; export { Failover2Plugin } from "./common/lib/plugins/failover2/failover2_plugin"; export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/host_monitoring_connection_plugin"; diff --git a/tests/unit/read_write_splitting.test.ts b/tests/unit/read_write_splitting.test.ts index 2c4f1388..aba156d7 100644 --- a/tests/unit/read_write_splitting.test.ts +++ b/tests/unit/read_write_splitting.test.ts @@ -29,7 +29,7 @@ import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { AwsMySQLClient } from "../../mysql/lib"; import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; import { HostListProviderService } from "../../common/lib/host_list_provider_service"; -import { ReadWriteSplittingPlugin } from "../../common/lib/plugins/read_write_splitting_plugin"; +import { ReadWriteSplittingPlugin } from "../../common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { MySQLDatabaseDialect } from "../../mysql/lib/dialect/mysql_database_dialect"; import { HostChangeOptions } from "../../common/lib/host_change_options"; From 1fa8805b2106454434a2d078fe08b7ebfb3c68a4 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Tue, 24 Feb 2026 17:32:16 -0800 Subject: [PATCH 03/20] chore: storage service refactoring (#611) --- .github/workflows/main.yml | 2 +- common/lib/aws_client.ts | 5 + .../monitoring/cluster_topology_monitor.ts | 25 +- .../monitoring_host_list_provider.ts | 1 - .../rds_host_list_provider.ts | 38 ++- common/lib/host_list_provider/topology.ts | 29 ++ common/lib/types.ts | 33 +++ common/lib/utils/core_services_container.ts | 48 ++++ common/lib/utils/storage/expiration_cache.ts | 160 +++++++++++ common/lib/utils/storage/storage_service.ts | 264 ++++++++++++++++++ tests/unit/rds_host_list_provider.test.ts | 44 +-- tests/unit/storage_service.test.ts | 57 ++++ 12 files changed, 665 insertions(+), 41 deletions(-) create mode 100644 common/lib/host_list_provider/topology.ts create mode 100644 common/lib/types.ts create mode 100644 common/lib/utils/core_services_container.ts create mode 100644 common/lib/utils/storage/expiration_cache.ts create mode 100644 common/lib/utils/storage/storage_service.ts create mode 100644 tests/unit/storage_service.test.ts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69a7d034..ed517c71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,10 +4,10 @@ on: push: branches: - main - - dev/v3 pull_request: branches: - "*" + - dev/v3 permissions: contents: read diff --git a/common/lib/aws_client.ts b/common/lib/aws_client.ts index b1e3b125..0140c847 100644 --- a/common/lib/aws_client.ts +++ b/common/lib/aws_client.ts @@ -35,11 +35,14 @@ import { Messages } from "./utils/messages"; import { HostListProviderService } from "./host_list_provider_service"; import { SessionStateClient } from "./session_state_client"; import { DriverConnectionProvider } from "./driver_connection_provider"; +import { StorageService } from "./utils/storage/storage_service"; +import { CoreServicesContainer } from "./utils/core_services_container"; const { EventEmitter } = pkgStream; export abstract class AwsClient extends EventEmitter implements SessionStateClient { private _defaultPort: number = -1; + private readonly storageService: StorageService; protected telemetryFactory: TelemetryFactory; protected pluginManager: PluginManager; protected pluginService: PluginService; @@ -64,6 +67,8 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie this.properties = new Map(Object.entries(config)); + this.storageService = CoreServicesContainer.getInstance().getStorageService(); + const profileName = WrapperProperties.PROFILE_NAME.get(this.properties); if (profileName && profileName.length > 0) { this._configurationProfile = DriverConfigurationProfiles.getProfileConfiguration(profileName); diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index 7e7263d6..a5a3e2f0 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -25,6 +25,9 @@ import { ClientWrapper } from "../../client_wrapper"; import { AwsWrapperError } from "../../utils/errors"; import { MonitoringRdsHostListProvider } from "./monitoring_host_list_provider"; import { Messages } from "../../utils/messages"; +import { CoreServicesContainer } from "../../utils/core_services_container"; +import { Topology } from "../topology"; +import { StorageService, StorageServiceImpl } from "../../utils/storage/storage_service"; export interface ClusterTopologyMonitor { forceRefresh(client: ClientWrapper, timeoutMs: number): Promise; @@ -45,8 +48,8 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { private readonly _hostListProvider: MonitoringRdsHostListProvider; private readonly refreshRateMs: number; private readonly highRefreshRateMs: number; + private readonly storageService: StorageService; - private topologyMap: CacheMap; private writerHostInfo: HostInfo = null; private isVerifiedWriterConnection: boolean = false; private monitoringClient: ClientWrapper = null; @@ -72,7 +75,6 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { constructor( clusterId: string, - topologyMap: CacheMap, initialHostInfo: HostInfo, props: Map, pluginService: PluginService, @@ -81,7 +83,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { highRefreshRateMs: number ) { this.clusterId = clusterId; - this.topologyMap = topologyMap; + this.storageService = CoreServicesContainer.getInstance().getStorageService(); // TODO: store serviceContainer instead this.initialHostInfo = initialHostInfo; this._pluginService = pluginService; this._hostListProvider = hostListProvider; @@ -125,7 +127,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { if (Date.now() < this.ignoreNewTopologyRequestsEndTimeMs) { // Previous failover has just completed, use results without triggering new update. - const currentHosts = this.topologyMap.get(this.clusterId); + const currentHosts = this.getStoredHosts(); if (currentHosts !== null) { logger.info(Messages.get("ClusterTopologyMonitoring.ignoringNewTopologyRequest")); return currentHosts; @@ -158,7 +160,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { // Signal to any monitor that might be in delay, that topology should be updated. this.requestToUpdateTopology = true; - const currentHosts: HostInfo[] = this.topologyMap.get(this.clusterId); + const currentHosts: HostInfo[] = this.getStoredHosts(); if (timeoutMs === 0) { logger.info(logTopology(currentHosts, Messages.get("ClusterTopologyMonitoring.timeoutSetToZero"))); @@ -168,7 +170,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { const endTime = Date.now() + timeoutMs; let latestHosts: HostInfo[]; - while ((latestHosts = this.topologyMap.get(this.clusterId)) === currentHosts && Date.now() < endTime) { + while ((latestHosts = this.getStoredHosts()) === currentHosts && Date.now() < endTime) { await sleep(1000); } @@ -244,7 +246,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } updateTopologyCache(hosts: HostInfo[]): void { - this.topologyMap.put(this.clusterId, hosts, ClusterTopologyMonitorImpl.TOPOLOGY_CACHE_EXPIRATION_NANOS); + this.storageService.set(this.clusterId, new Topology(hosts)); this.requestToUpdateTopology = false; } @@ -296,7 +298,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { this.hostMonitorsLatestTopology = []; // Use any client to gather topology information. - let hosts: HostInfo[] = this.topologyMap.get(this.clusterId); + let hosts: HostInfo[] = this.getStoredHosts(); if (!hosts) { hosts = await this.openAnyClientAndUpdateTopology(); } @@ -392,6 +394,11 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } } + private getStoredHosts(): HostInfo[] | null { + const topology = this.storageService.get(Topology, this.clusterId); + return topology == null ? null : topology.hosts; + } + private async delay(useHighRefreshRate: boolean) { if (Date.now() < this.highRefreshRateEndTimeMs) { useHighRefreshRate = true; @@ -404,7 +411,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } logTopology(msgPrefix: string) { - const hosts: HostInfo[] = this.topologyMap.get(this.clusterId); + const hosts: HostInfo[] = this.getStoredHosts(); if (hosts && hosts.length !== 0) { logger.debug(logTopology(hosts, msgPrefix)); } diff --git a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts index 49c5ae8f..335baa6c 100644 --- a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts +++ b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts @@ -89,7 +89,6 @@ export class MonitoringRdsHostListProvider extends RdsHostListProvider implement () => new ClusterTopologyMonitorImpl( this.clusterId, - MonitoringRdsHostListProvider.topologyCache, this.initialHost, this.properties, this.pluginService, diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index fd7353bb..655f827d 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -30,10 +30,15 @@ import { CacheMap } from "../utils/cache_map"; import { isDialectTopologyAware, logTopology } from "../utils/utils"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; +import { CoreServicesContainer } from "../utils/core_services_container"; +import { StorageService } from "../utils/storage/storage_service"; +import { Topology } from "./topology"; +import { ExpirationCache } from "../utils/storage/expiration_cache"; export class RdsHostListProvider implements DynamicHostListProvider { private readonly originalUrl: string; private readonly rdsHelper: RdsUtils; + private readonly storageService: StorageService; protected readonly properties: Map; private rdsUrlType: RdsUrlType; private initialHostList: HostInfo[]; @@ -45,9 +50,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { protected readonly hostListProviderService: HostListProviderService; public static readonly suggestedPrimaryClusterIdCache: CacheMap = new CacheMap(); - public static readonly primaryClusterIdCache: CacheMap = new CacheMap(); - public static readonly topologyCache: CacheMap = new CacheMap(); public clusterId: string = Date.now().toString(); public isInitialized: boolean = false; public isPrimaryClusterId?: boolean; @@ -59,6 +62,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.connectionUrlParser = hostListProviderService.getConnectionUrlParser(); this.originalUrl = originalUrl; this.properties = properties; + this.storageService = CoreServicesContainer.getInstance().getStorageService(); // TODO: store the service container instead. let port = WrapperProperties.PORT.get(properties); if (port == null) { @@ -197,7 +201,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.isPrimaryClusterId = true; } - const cachedHosts: HostInfo[] | null = RdsHostListProvider.topologyCache.get(this.clusterId); + const cachedHosts: HostInfo[] | null = this.getStoredTopology(); // This clusterId is a primary one and is about to create a new entry in the cache. // When a primary entry is created it needs to be suggested for other (non-primary) entries. @@ -211,7 +215,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { const hosts = await this.queryForTopology(targetClient, this.hostListProviderService.getDialect()); if (hosts && hosts.length > 0) { - RdsHostListProvider.topologyCache.put(this.clusterId, hosts, this.refreshRateNano); + this.storageService.set(this.clusterId, new Topology(hosts)); if (needToSuggest) { this.suggestPrimaryCluster(hosts); } @@ -227,14 +231,18 @@ export class RdsHostListProvider implements DynamicHostListProvider { } private getSuggestedClusterId(hostAndPort: string): ClusterSuggestedResult | null { - for (const [key, hosts] of RdsHostListProvider.topologyCache.getEntries()) { + const cache: ExpirationCache = this.storageService.getAll(Topology) as ExpirationCache; + if (!cache) { + return null; + } + for (const [key, hosts] of cache.getEntries()) { const isPrimaryCluster: boolean = RdsHostListProvider.primaryClusterIdCache.get(key, false, this.suggestedClusterIdRefreshRateNano) ?? false; if (key === hostAndPort) { return new ClusterSuggestedResult(hostAndPort, isPrimaryCluster); } if (hosts) { - for (const hostInfo of hosts) { + for (const hostInfo of hosts.hosts) { if (hostInfo.hostAndPort === hostAndPort) { logger.debug(Messages.get("RdsHostListProvider.suggestedClusterId", key, hostAndPort)); return new ClusterSuggestedResult(key, isPrimaryCluster); @@ -255,7 +263,11 @@ export class RdsHostListProvider implements DynamicHostListProvider { primaryClusterHostUrls.add(hostInfo.url); }); - for (const [clusterId, clusterHosts] of RdsHostListProvider.topologyCache.getEntries()) { + const cache: ExpirationCache = this.storageService.getAll(Topology) as ExpirationCache; + if (!cache) { + return; + } + for (const [clusterId, clusterHosts] of cache.getEntries()) { const isPrimaryCluster: boolean | null = RdsHostListProvider.primaryClusterIdCache.get( clusterId, false, @@ -266,7 +278,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { continue; } - for (const clusterHost of clusterHosts) { + for (const clusterHost of clusterHosts.hosts) { if (primaryClusterHostUrls.has(clusterHost.url)) { RdsHostListProvider.suggestedPrimaryClusterIdCache.put(clusterId, this.clusterId, this.suggestedClusterIdRefreshRateNano); break; @@ -343,22 +355,24 @@ export class RdsHostListProvider implements DynamicHostListProvider { return host.replace("?", hostName); } - getCachedTopology(): HostInfo[] | null { + getStoredTopology(): HostInfo[] | null { if (!this.clusterId) { return null; } - return RdsHostListProvider.topologyCache.get(this.clusterId) ?? null; + + const topology: Topology = this.storageService.get(Topology, this.clusterId); + + return topology == null ? null : topology.hosts; } static clearAll(): void { - RdsHostListProvider.topologyCache.clear(); RdsHostListProvider.primaryClusterIdCache.clear(); RdsHostListProvider.suggestedPrimaryClusterIdCache.clear(); } clear(): void { if (this.clusterId) { - RdsHostListProvider.topologyCache.delete(this.clusterId); + CoreServicesContainer.getInstance().getStorageService().remove(Topology, this.clusterId); } } diff --git a/common/lib/host_list_provider/topology.ts b/common/lib/host_list_provider/topology.ts new file mode 100644 index 00000000..35a53460 --- /dev/null +++ b/common/lib/host_list_provider/topology.ts @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HostInfo } from "../host_info"; + +export class Topology { + private readonly _hosts: HostInfo[]; + + constructor(hosts: HostInfo[]) { + this._hosts = hosts; + } + + get hosts(): HostInfo[] { + return this._hosts; + } +} diff --git a/common/lib/types.ts b/common/lib/types.ts new file mode 100644 index 00000000..3332b0c9 --- /dev/null +++ b/common/lib/types.ts @@ -0,0 +1,33 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Type representing a constructor for any class. + */ +export type Constructor = new (...args: unknown[]) => T; + +/** + * Function type that determines whether an item should be disposed when expired. + * @param item The item to check + * @returns true if the item should be disposed, false otherwise + */ +export type ShouldDisposeFunc = (item: V) => boolean; + +/** + * Function type that defines how to dispose of an item when it is removed. + * @param item The item to dispose + */ +export type ItemDisposalFunc = (item: V) => void; diff --git a/common/lib/utils/core_services_container.ts b/common/lib/utils/core_services_container.ts new file mode 100644 index 00000000..c9ab89fe --- /dev/null +++ b/common/lib/utils/core_services_container.ts @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { StorageService, StorageServiceImpl } from "./storage/storage_service"; + +/** + * A singleton container object used to instantiate and access core universal services. This class should be used + * instead of directly instantiating core services so that only one instance of each service is instantiated. + * + * @see FullServicesContainer for a container that holds both connection-specific services and core universal + * services. + */ +export class CoreServicesContainer { + private static readonly INSTANCE = new CoreServicesContainer(); + + // private readonly monitorService: MonitorService; // TODO: implement monitor service + private readonly storageService: StorageService; + + private constructor() { + this.storageService = new StorageServiceImpl(); + // this.monitorService = new MonitorServiceImpl(); + } + + static getInstance(): CoreServicesContainer { + return CoreServicesContainer.INSTANCE; + } + + getStorageService(): StorageService { + return this.storageService; + } + + // getMonitorService(): MonitorService { + // return this.monitorService; + // } +} diff --git a/common/lib/utils/storage/expiration_cache.ts b/common/lib/utils/storage/expiration_cache.ts new file mode 100644 index 00000000..5afd57e9 --- /dev/null +++ b/common/lib/utils/storage/expiration_cache.ts @@ -0,0 +1,160 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { getTimeInNanos } from "../utils"; +import { ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; + +export class CacheEntry { + readonly value: V; + private _expirationTimeNanos: bigint; + + constructor(value: V, expirationTimeNanos: bigint) { + this.value = value; + this._expirationTimeNanos = expirationTimeNanos; + } + + get expirationTimeNanos(): bigint { + return this._expirationTimeNanos; + } + + renewExpiration(timeToLiveNanos: bigint): void { + this._expirationTimeNanos = getTimeInNanos() + timeToLiveNanos; + } + + isExpired(): boolean { + return getTimeInNanos() > this._expirationTimeNanos; + } +} + +export class ExpirationCache { + private readonly map: Map> = new Map(); + private readonly isRenewableExpiration: boolean; + private readonly timeToLiveNanos: bigint; + private readonly shouldDisposeFunc?: ShouldDisposeFunc | null; + private readonly itemDisposalFunc?: ItemDisposalFunc | null; + + constructor( + isRenewableExpiration: boolean = true, + timeToLiveNanos: bigint = BigInt(300_000_000_000), // 5 minutes default + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ) { + this.isRenewableExpiration = isRenewableExpiration; + this.timeToLiveNanos = timeToLiveNanos; + this.shouldDisposeFunc = shouldDisposeFunc; + this.itemDisposalFunc = itemDisposalFunc; + } + + put(key: K, value: V): void { + const expirationTime = getTimeInNanos() + this.timeToLiveNanos; + this.map.set(key, new CacheEntry(value, expirationTime)); + } + + get(key: K): V | null | undefined { + const entry = this.map.get(key); + if (!entry) { + return null; + } + + if (entry.isExpired()) { + if (this.shouldDispose(entry.value)) { + this.removeEntry(key, entry); + return null; + } + } + + if (this.isRenewableExpiration) { + entry.renewExpiration(this.timeToLiveNanos); + } + + return entry.value; + } + + exists(key: K): boolean { + const entry = this.map.get(key); + if (!entry) { + return false; + } + + if (entry.isExpired() && this.shouldDispose(entry.value)) { + this.removeEntry(key, entry); + return false; + } + + return true; + } + + remove(key: K): void { + const entry = this.map.get(key); + if (entry) { + this.removeEntry(key, entry); + } + } + + clear(): void { + for (const entry of this.map.values()) { + this.disposeItem(entry.value); + } + this.map.clear(); + } + + size(): number { + return this.map.size; + } + + getEntries(): Map { + const result = new Map(); + for (const [key, entry] of this.map.entries()) { + result.set(key, entry.value); + } + return result; + } + + removeExpiredEntries(): void { + const keysToRemove: K[] = []; + + for (const [key, entry] of this.map.entries()) { + if (entry.isExpired() && this.shouldDispose(entry.value)) { + keysToRemove.push(key); + } + } + + for (const key of keysToRemove) { + const entry = this.map.get(key); + if (entry) { + this.removeEntry(key, entry); + } + } + } + + private shouldDispose(value: V): boolean { + if (this.shouldDisposeFunc) { + return this.shouldDisposeFunc(value); + } + return true; + } + + private removeEntry(key: K, entry: CacheEntry): void { + this.map.delete(key); + this.disposeItem(entry.value); + } + + private disposeItem(value: V): void { + if (this.itemDisposalFunc) { + this.itemDisposalFunc(value); + } + } +} diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts new file mode 100644 index 00000000..d13bc052 --- /dev/null +++ b/common/lib/utils/storage/storage_service.ts @@ -0,0 +1,264 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Constructor, ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; +import { logger } from "../../../logutils"; +import { ExpirationCache } from "./expiration_cache"; +import { Topology } from "../../host_list_provider/topology"; + +const DEFAULT_CLEANUP_INTERVAL_NANOS = 5 * 60 * 1_000_000_000; // 5 minutes + +/** + * Interface for a storage service that manages items with expiration and disposal logic. + * Extends StateSnapshotProvider for state management capabilities. + */ +export interface StorageService { + /** + * Registers a new item class with the storage service. This method needs to be called before adding new classes of + * items to the service, so that the service knows when and how to dispose of the item. Expected item classes will be + * added automatically during driver initialization, but this method can be called to add new classes of items. + * + * @param itemClass The constructor/class of the item that will be stored + * @param isRenewableExpiration Controls whether the item's expiration should be renewed if the item is fetched, + * regardless of whether it is already expired or not + * @param timeToLiveNanos How long an item should be stored before being considered expired, in nanoseconds + * @param shouldDisposeFunc A function defining whether an item should be disposed if expired. If null/undefined, + * the item will always be disposed if expired + * @param itemDisposalFunc A function defining how to dispose of an item when it is removed. If null/undefined, + * the item will be removed without performing any additional operations + */ + registerItemClassIfAbsent( + itemClass: Constructor, + isRenewableExpiration: boolean, + timeToLiveNanos: bigint, + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ): void; + + /** + * Stores an item in the storage service under the given item class. + * + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @param item The item to store + */ + set(key: unknown, item: V): void; + + // TODO: temporary method to return all storage services for a specific item class. + // Should be removed along with the cluster id refactoring. + getAll(itemClass: Constructor): ExpirationCache | null; + + /** + * Gets an item stored in the storage service. + * + * @param itemClass The expected constructor/class of the item being retrieved + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @returns The item stored at the given key for the given item class, or null/undefined if not found + */ + get(itemClass: Constructor, key?: unknown): V | null; + + /** + * Indicates whether an item exists under the given item class and key. + * + * @param itemClass The constructor/class of the item + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @returns true if the item exists under the given item class and key, otherwise returns false + */ + exists(itemClass: Constructor, key: unknown): boolean; + + /** + * Removes an item stored under the given item class. + * + * @param itemClass The constructor/class of the item + * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + */ + remove(itemClass: Constructor, key: unknown): void; + + /** + * Clears all items of the given item class. For example, storageService.clear(AllowedAndBlockedHosts) will + * remove all AllowedAndBlockedHost items from the storage service. + * + * @param itemClass The constructor/class of the items to clear + */ + clear(itemClass: Constructor): void; + + /** + * Clears all items from the storage service. + */ + clearAll(): void; + + /** + * Returns the number of items stored for the given item class. + * + * @param itemClass The constructor/class of the items to count + * @returns The number of items stored for the given item class + */ + size(itemClass: Constructor): number; +} + +type CacheSupplier = () => ExpirationCache; + +export class StorageServiceImpl implements StorageService { + private static readonly defaultCacheSuppliers: Map = new Map([[Topology, () => new ExpirationCache()]]); + + protected readonly caches: Map> = new Map(); + protected cleanupIntervalHandle?: NodeJS.Timeout; + + constructor(cleanupIntervalNanos: number = DEFAULT_CLEANUP_INTERVAL_NANOS) { + this.initCleanupThread(cleanupIntervalNanos); + } + + protected initCleanupThread(cleanupIntervalNanos: number): void { + const intervalMs = cleanupIntervalNanos / 1_000_000; + this.cleanupIntervalHandle = setInterval(() => { + this.removeExpiredItems(); + }, intervalMs); + + // Allow Node.js to exit even if this timer is active + if (this.cleanupIntervalHandle.unref) { + this.cleanupIntervalHandle.unref(); + } + } + + protected removeExpiredItems(): void { + logger.debug("StorageServiceImpl: Removing expired items"); + for (const cache of this.caches.values()) { + cache.removeExpiredEntries(); + } + } + + registerItemClassIfAbsent( + itemClass: Constructor, + isRenewableExpiration: boolean, + timeToLiveNanos: bigint, + shouldDisposeFunc?: ShouldDisposeFunc | null, + itemDisposalFunc?: ItemDisposalFunc | null + ): void { + if (!this.caches.has(itemClass)) { + const cache = new ExpirationCache( + isRenewableExpiration, + timeToLiveNanos, + shouldDisposeFunc ?? undefined, + itemDisposalFunc ?? undefined + ); + this.caches.set(itemClass, cache); + } + } + + set(key: unknown, item: V): void { + const itemClass = item.constructor as Constructor; + let cache = this.caches.get(itemClass); + + if (!cache) { + const supplier = StorageServiceImpl.defaultCacheSuppliers.get(itemClass); + if (!supplier) { + throw new Error(`StorageServiceImpl: Item class not registered: ${itemClass.name}`); + } + cache = supplier(); + this.caches.set(itemClass, cache); + } + + try { + cache.put(key, item); + } catch (error) { + throw new Error(`StorageServiceImpl: Unexpected value mismatch for ${itemClass.name}: ${error}`); + } + } + + getAll(itemClass: Constructor): ExpirationCache | null { + const cache = this.caches.get(itemClass); + if (!cache) { + return null; + } + return cache; + } + + get(itemClass: Constructor, key?: unknown): V | null { + const cache = this.caches.get(itemClass); + if (!cache) { + return null; + } + + const value = cache.get(key); + if (value === null || value === undefined) { + return null; + } + + if (value instanceof itemClass) { + return value as V; + } + + logger.debug(`StorageServiceImpl: Item class mismatch for key ${String(key)}: ` + `expected ${itemClass.name}, got ${value.constructor.name}`); + return null; + } + + exists(itemClass: Constructor, key: unknown): boolean { + const cache = this.caches.get(itemClass); + if (!cache) { + return false; + } + return cache.exists(key); + } + + remove(itemClass: Constructor, key: unknown): void { + const cache = this.caches.get(itemClass); + if (cache) { + cache.remove(key); + } + } + + clear(itemClass: Constructor): void { + const cache = this.caches.get(itemClass); + if (cache) { + cache.clear(); + } + } + + clearAll(): void { + for (const cache of this.caches.values()) { + cache.clear(); + } + + this.caches.clear(); + } + + size(itemClass: Constructor): number { + const cache = this.caches.get(itemClass); + if (!cache) { + return 0; + } + return cache.size(); + } + + /** + * Registers a default cache supplier for a specific item class. + * This allows automatic cache creation when items of this class are stored. + */ + static registerDefaultCacheSupplier(itemClass: Constructor, supplier: CacheSupplier): void { + StorageServiceImpl.defaultCacheSuppliers.set(itemClass, supplier); + } + + /** + * Cleanup method to stop the cleanup interval timer. + * Should be called when the service is no longer needed. + */ + destroy(): void { + if (this.cleanupIntervalHandle) { + clearInterval(this.cleanupIntervalHandle); + this.cleanupIntervalHandle = undefined; + } + this.clearAll(); + } +} diff --git a/tests/unit/rds_host_list_provider.test.ts b/tests/unit/rds_host_list_provider.test.ts index df3b34c8..1801169c 100644 --- a/tests/unit/rds_host_list_provider.test.ts +++ b/tests/unit/rds_host_list_provider.test.ts @@ -22,11 +22,14 @@ import { AwsWrapperError, HostInfo, HostInfoBuilder, HostRole } from "../../comm import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { ConnectionUrlParser } from "../../common/lib/utils/connection_url_parser"; import { AwsPGClient } from "../../pg/lib"; -import { sleep } from "../../common/lib/utils/utils"; +import { convertMsToNanos, sleep } from "../../common/lib/utils/utils"; import { AuroraPgDatabaseDialect } from "../../pg/lib/dialect/aurora_pg_database_dialect"; import { ClientWrapper } from "../../common/lib/client_wrapper"; import { PgConnectionUrlParser } from "../../pg/lib/pg_connection_url_parser"; import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { Topology } from "../../common/lib/host_list_provider/topology"; const mockClient: AwsClient = mock(AwsPGClient); const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); @@ -54,6 +57,8 @@ const clientWrapper: ClientWrapper = new PgClientWrapper(undefined, currentHostI const mockClientWrapper: ClientWrapper = mock(clientWrapper); +const storageService: StorageService = CoreServicesContainer.getInstance().getStorageService(); + const defaultRefreshRateNano: number = 5 * 1_000_000_000; function createHost(config: any): HostInfo { @@ -80,6 +85,7 @@ describe("testRdsHostListProvider", () => { afterEach(() => { RdsHostListProvider.clearAll(); + CoreServicesContainer.getInstance().getStorageService().clearAll(); reset(mockDialect); reset(mockClientWrapper); @@ -92,7 +98,7 @@ describe("testRdsHostListProvider", () => { const spiedProvider = spy(rdsHostListProvider); const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); expect(result.hosts.length).toEqual(2); @@ -108,7 +114,7 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.isClientValid(anything())).thenResolve(true); - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, hosts, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(hosts)); const newHosts: HostInfo[] = [ createHost({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), @@ -133,7 +139,7 @@ describe("testRdsHostListProvider", () => { spiedProvider.isInitialized = true; const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); @@ -175,22 +181,24 @@ describe("testRdsHostListProvider", () => { const rdsHostListProvider = getRdsHostListProvider("foo"); const expected: HostInfo[] = hosts; - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, expected, defaultRefreshRateNano); + storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); - const result = rdsHostListProvider.getCachedTopology(); + const result = rdsHostListProvider.getStoredTopology(); expect(result).toEqual(expected); }); it("testGetCachedTopology_returnNull", async () => { let rdsHostListProvider = getRdsHostListProvider("foo"); - expect(rdsHostListProvider.getCachedTopology()).toBeNull(); + expect(rdsHostListProvider.getStoredTopology()).toBeNull(); rdsHostListProvider.clear(); rdsHostListProvider = getRdsHostListProvider("foo"); - RdsHostListProvider.topologyCache.put(rdsHostListProvider.clusterId, hosts, 1_000_000); + + storageService.registerItemClassIfAbsent(Topology, true, convertMsToNanos(1)); + storageService.set(rdsHostListProvider.clusterId, new Topology(hosts)); await sleep(2); - expect(rdsHostListProvider.getCachedTopology()).toBeNull(); + expect(rdsHostListProvider.getStoredTopology()).toBeNull(); }); it("testTopologyCache_noSuggestedClusterId", async () => { @@ -220,7 +228,7 @@ describe("testRdsHostListProvider", () => { ]; when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); + expect(storageService.getAll(Topology)).toBeNull(); const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); expect(topologyProvider1).toEqual(topologyClusterA); @@ -248,7 +256,7 @@ describe("testRdsHostListProvider", () => { when(spiedProvider2.queryForTopology(instance(mockClientWrapper), anything())).thenReturn(Promise.resolve(topologyClusterB)); expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterB); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); + expect(storageService.getAll(Topology).size()).toEqual(2); }); it("testTopologyCache_suggestedClusterIdForRds", async () => { @@ -278,7 +286,7 @@ describe("testRdsHostListProvider", () => { const spiedProvider1 = spy(provider1); when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); + expect(storageService.getAll(Topology)).toBeNull(); const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); expect(topologyProvider1).toEqual(topologyClusterA); @@ -290,7 +298,7 @@ describe("testRdsHostListProvider", () => { expect(provider2.isPrimaryClusterId).toBeTruthy(); expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(1); + expect(storageService.getAll(Topology).size()).toEqual(1); }); it("testTopologyCache_suggestedClusterIdForInstance", async () => { @@ -320,7 +328,7 @@ describe("testRdsHostListProvider", () => { const spiedProvider1 = spy(provider1); when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); + expect(storageService.getAll(Topology)).toBeNull(); const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); expect(topologyProvider1).toEqual(topologyClusterA); @@ -332,7 +340,7 @@ describe("testRdsHostListProvider", () => { expect(provider2.isPrimaryClusterId).toBeTruthy(); expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(1); + expect(storageService.getAll(Topology).size()).toEqual(1); }); it("testTopologyCache_acceptSuggestion", async () => { @@ -362,7 +370,7 @@ describe("testRdsHostListProvider", () => { const spiedProvider1 = spy(provider1); when(spiedProvider1.queryForTopology(anything(), anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(RdsHostListProvider.topologyCache.size()).toEqual(0); + expect(storageService.getAll(Topology)).toBeNull(); const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); expect(topologyProvider1).toEqual(topologyClusterA); @@ -376,12 +384,12 @@ describe("testRdsHostListProvider", () => { expect(provider2.isPrimaryClusterId).toBeTruthy(); expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterA); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); + expect(storageService.getAll(Topology).size()).toEqual(2); expect(RdsHostListProvider.suggestedPrimaryClusterIdCache.get(provider1.clusterId)).toEqual("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); expect(await provider1.forceRefresh(instance(mockClientWrapper))).toEqual(topologyClusterA); expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(RdsHostListProvider.topologyCache.size()).toEqual(2); + expect(storageService.getAll(Topology).size()).toEqual(2); expect(provider1.isPrimaryClusterId).toBeTruthy(); expect(provider2.isPrimaryClusterId).toBeTruthy(); }); diff --git a/tests/unit/storage_service.test.ts b/tests/unit/storage_service.test.ts new file mode 100644 index 00000000..25d32e48 --- /dev/null +++ b/tests/unit/storage_service.test.ts @@ -0,0 +1,57 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { Topology } from "../../common/lib/host_list_provider/topology"; +import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; +import { HostInfoBuilder } from "../../common/lib"; +import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; + +describe("test_storage_service", () => { + let storageService: StorageService; + + beforeEach(() => { + storageService = CoreServicesContainer.getInstance().getStorageService(); + }); + + afterEach(() => { + storageService.clearAll(); + }); + + it("should correctly identify Topology class from constructor", () => { + const topology = new Topology([]); + const secondTopology = new Topology([ + new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).withHost("foo").build() + ]); + const key = "test-key"; + + // Store the topology instance + storageService.set(key, topology); + + let retrieved = storageService.get(Topology, key); + expect(retrieved).toBeDefined(); + expect(retrieved).toBeInstanceOf(Topology); + expect(retrieved).toBe(topology); + + storageService.set(key, secondTopology); + + // Ensure the topology cache has been overwritten. + retrieved = storageService.get(Topology, key); + expect(retrieved).toBeDefined(); + expect(retrieved).toBeInstanceOf(Topology); + expect(retrieved).toBe(secondTopology); + }); +}); From f808fc6029e5fbded7c6b81b69692dbc831382bd Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:27:47 -0800 Subject: [PATCH 04/20] refactor: remove duplicated getHostAndPort method (#613) --- common/lib/host_info.ts | 4 ---- .../monitoring/cluster_topology_monitor.ts | 4 ++-- common/lib/plugins/bluegreen/blue_green_interim_status.ts | 4 ++-- common/lib/plugins/bluegreen/blue_green_status_provider.ts | 2 +- .../lib/plugins/bluegreen/routing/base_connect_routing.ts | 2 +- .../lib/plugins/bluegreen/routing/base_execute_routing.ts | 2 +- .../plugins/bluegreen/routing/substitute_connect_routing.ts | 6 +++--- .../plugins/connection_tracker/opened_connection_tracker.ts | 2 +- common/lib/plugins/failover2/failover2_plugin.ts | 2 +- 9 files changed, 12 insertions(+), 16 deletions(-) diff --git a/common/lib/host_info.ts b/common/lib/host_info.ts index 566f5cfa..3ac57751 100644 --- a/common/lib/host_info.ts +++ b/common/lib/host_info.ts @@ -65,10 +65,6 @@ export class HostInfo { return this.port != HostInfo.NO_PORT; } - getHostAndPort(): string { - return this.isPortSpecified() ? this.host + ":" + this.port : this.host; - } - addAlias(...alias: string[]) { if (!alias || alias.length < 1) { return; diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index a5a3e2f0..958f4046 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -528,9 +528,9 @@ export class HostMonitor { } const latestWriterHostInfo: HostInfo = hosts.find((x) => x.role === HostRole.WRITER); - if (latestWriterHostInfo && writerHostInfo && latestWriterHostInfo.getHostAndPort() !== writerHostInfo.getHostAndPort()) { + if (latestWriterHostInfo && writerHostInfo && latestWriterHostInfo.hostAndPort !== writerHostInfo.hostAndPort) { this.writerChanged = true; - logger.debug(Messages.get("HostMonitor.writerHostChanged", writerHostInfo.getHostAndPort(), latestWriterHostInfo.getHostAndPort())); + logger.debug(Messages.get("HostMonitor.writerHostChanged", writerHostInfo.hostAndPort, latestWriterHostInfo.hostAndPort)); this.monitor.updateTopologyCache(hosts); logger.debug(logTopology(hosts, `[hostMonitor ${this.hostInfo.hostId}] `)); } diff --git a/common/lib/plugins/bluegreen/blue_green_interim_status.ts b/common/lib/plugins/bluegreen/blue_green_interim_status.ts index c2c97931..4cf7bb3f 100644 --- a/common/lib/plugins/bluegreen/blue_green_interim_status.ts +++ b/common/lib/plugins/bluegreen/blue_green_interim_status.ts @@ -104,7 +104,7 @@ export class BlueGreenInterimStatus { this.startTopology == null ? "" : this.startTopology - .map((x) => x.getHostAndPort() + x.role) + .map((x) => x.hostAndPort + x.role) .sort() .join(",") ); @@ -114,7 +114,7 @@ export class BlueGreenInterimStatus { this.currentTopology == null ? "" : this.currentTopology - .map((x) => x.getHostAndPort() + x.role) + .map((x) => x.hostAndPort + x.role) .sort() .join(",") ); diff --git a/common/lib/plugins/bluegreen/blue_green_status_provider.ts b/common/lib/plugins/bluegreen/blue_green_status_provider.ts index bf899e11..00f88db0 100644 --- a/common/lib/plugins/bluegreen/blue_green_status_provider.ts +++ b/common/lib/plugins/bluegreen/blue_green_status_provider.ts @@ -872,7 +872,7 @@ export class BlueGreenStatusProvider { logger.debug( "Corresponding hosts:\n" + Array.from(this.correspondingHosts.entries()) - .map(([key, value]) => ` ${key} -> ${value.right == null ? "" : value.right.getHostAndPort()}`) + .map(([key, value]) => ` ${key} -> ${value.right == null ? "" : value.right.hostAndPort}`) .join("\n") ); diff --git a/common/lib/plugins/bluegreen/routing/base_connect_routing.ts b/common/lib/plugins/bluegreen/routing/base_connect_routing.ts index 5dfdd25c..9a0deecc 100644 --- a/common/lib/plugins/bluegreen/routing/base_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/base_connect_routing.ts @@ -25,7 +25,7 @@ import { PluginService } from "../../../plugin_service"; export abstract class BaseConnectRouting extends BaseRouting implements ConnectRouting { isMatch(hostInfo: HostInfo, hostRole: BlueGreenRole): boolean { return ( - (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.getHostAndPort().toLowerCase())) && + (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.hostAndPort.toLowerCase())) && (this.role === null || this.role === hostRole) ); } diff --git a/common/lib/plugins/bluegreen/routing/base_execute_routing.ts b/common/lib/plugins/bluegreen/routing/base_execute_routing.ts index 80d7d50e..e631442b 100644 --- a/common/lib/plugins/bluegreen/routing/base_execute_routing.ts +++ b/common/lib/plugins/bluegreen/routing/base_execute_routing.ts @@ -24,7 +24,7 @@ import { ConnectionPlugin } from "../../../connection_plugin"; export abstract class BaseExecuteRouting extends BaseRouting implements ExecuteRouting { isMatch(hostInfo: HostInfo, hostRole: BlueGreenRole): boolean { return ( - (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.getHostAndPort().toLowerCase())) && + (this.hostAndPort === null || this.hostAndPort === (hostInfo ?? hostInfo.hostAndPort.toLowerCase())) && (this.role === null || this.role === hostRole) ); } diff --git a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts index 1d453e46..94f35677 100644 --- a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts @@ -99,12 +99,12 @@ export class SubstituteConnectRouting extends BaseConnectRouting { // try with another IAM host } } - throw new AwsWrapperError(Messages.get("Bgd.inProgressCantOpenConnection", this.substituteHost.getHostAndPort())); + throw new AwsWrapperError(Messages.get("Bgd.inProgressCantOpenConnection", this.substituteHost.hostAndPort)); } toString(): string { - return `${this.constructor.name} [${this.hostAndPort ?? ""}, ${this.role?.name ?? ""}, substitute: ${this.substituteHost?.getHostAndPort() ?? ""}, iamHosts: ${ - this.iamHosts?.map((host) => host.getHostAndPort()).join(", ") ?? "" + return `${this.constructor.name} [${this.hostAndPort ?? ""}, ${this.role?.name ?? ""}, substitute: ${this.substituteHost?.hostAndPort ?? ""}, iamHosts: ${ + this.iamHosts?.map((host) => host.hostAndPort).join(", ") ?? "" }]`; } } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 364a123b..6c8b6a58 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -36,7 +36,7 @@ export class OpenedConnectionTracker { // Check if the connection was established using an instance endpoint if (OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host)) { - this.trackConnection(hostInfo.getHostAndPort(), client); + this.trackConnection(hostInfo.hostAndPort, client); return; } diff --git a/common/lib/plugins/failover2/failover2_plugin.ts b/common/lib/plugins/failover2/failover2_plugin.ts index 5d7060d0..3b4a7f04 100644 --- a/common/lib/plugins/failover2/failover2_plugin.ts +++ b/common/lib/plugins/failover2/failover2_plugin.ts @@ -138,7 +138,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele return await this._staleDnsHelper.getVerifiedConnection(hostInfo.host, isInitialConnection, this.hostListProviderService!, props, connectFunc); } - const hostInfoWithAvailability: HostInfo = this.pluginService.getHosts().find((x) => x.getHostAndPort() === hostInfo.getHostAndPort()); + const hostInfoWithAvailability: HostInfo = this.pluginService.getHosts().find((x) => x.hostAndPort === hostInfo.hostAndPort); let client: ClientWrapper = null; if (!hostInfoWithAvailability || hostInfoWithAvailability.getAvailability() != HostAvailability.NOT_AVAILABLE) { From 7aaf5582715d1fd399ce4316a3b179c3c52a8e5d Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 25 Feb 2026 14:51:16 -0800 Subject: [PATCH 05/20] refactor: remove suggested cluster id (#612) --- .../rds_host_list_provider.ts | 111 +--------- common/lib/utils/storage/storage_service.ts | 4 - common/lib/wrapper_property.ts | 4 +- .../using-plugins/UsingTheFailover2Plugin.md | 17 +- tests/unit/rds_host_list_provider.test.ts | 193 ------------------ 5 files changed, 14 insertions(+), 315 deletions(-) diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index 655f827d..30c80a3f 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -44,16 +44,12 @@ export class RdsHostListProvider implements DynamicHostListProvider { private initialHostList: HostInfo[]; protected initialHost: HostInfo; private refreshRateNano: number; - private suggestedClusterIdRefreshRateNano: number = 10 * 60 * 1_000_000_000; // 10 minutes private hostList?: HostInfo[]; protected readonly connectionUrlParser: ConnectionUrlParser; protected readonly hostListProviderService: HostListProviderService; - public static readonly suggestedPrimaryClusterIdCache: CacheMap = new CacheMap(); - public static readonly primaryClusterIdCache: CacheMap = new CacheMap(); public clusterId: string = Date.now().toString(); public isInitialized: boolean = false; - public isPrimaryClusterId?: boolean; public clusterInstanceTemplate?: HostInfo; constructor(properties: Map, originalUrl: string, hostListProviderService: HostListProviderService) { @@ -87,8 +83,6 @@ export class RdsHostListProvider implements DynamicHostListProvider { return; } - this.isPrimaryClusterId = false; - const hostInfoBuilder = this.hostListProviderService.getHostInfoBuilder(); this.clusterInstanceTemplate = hostInfoBuilder @@ -98,29 +92,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.validateHostPatternSetting(this.clusterInstanceTemplate.host); - const clusterIdSetting: string = WrapperProperties.CLUSTER_ID.get(this.properties); - if (clusterIdSetting) { - this.clusterId = clusterIdSetting; - } else if (this.rdsUrlType === RdsUrlType.RDS_PROXY) { - // Each proxy is associated with a single cluster, so it's safe to use RDS Proxy Url as cluster - // identification - this.clusterId = this.initialHost.url; - } else if (this.rdsUrlType.isRds) { - const clusterSuggestedResult: ClusterSuggestedResult | null = this.getSuggestedClusterId(this.initialHost.hostAndPort); - if (clusterSuggestedResult && clusterSuggestedResult.clusterId) { - this.clusterId = clusterSuggestedResult.clusterId; - this.isPrimaryClusterId = clusterSuggestedResult.isPrimaryClusterId; - } else { - const clusterRdsHostUrl: string | null = this.rdsHelper.getRdsClusterHostUrl(this.initialHost.host); - if (clusterRdsHostUrl) { - this.clusterId = this.clusterInstanceTemplate.isPortSpecified() - ? `${clusterRdsHostUrl}:${this.clusterInstanceTemplate.port}` - : clusterRdsHostUrl; - this.isPrimaryClusterId = true; - RdsHostListProvider.primaryClusterIdCache.put(this.clusterId, true, this.suggestedClusterIdRefreshRateNano); - } - } - } + this.clusterId = WrapperProperties.CLUSTER_ID.get(this.properties); this.isInitialized = true; } @@ -195,18 +167,11 @@ export class RdsHostListProvider implements DynamicHostListProvider { throw new AwsWrapperError("no cluster id"); } - const suggestedPrimaryClusterId: string | null = RdsHostListProvider.suggestedPrimaryClusterIdCache.get(this.clusterId); - if (suggestedPrimaryClusterId && this.clusterId !== suggestedPrimaryClusterId) { - this.clusterId = suggestedPrimaryClusterId; - this.isPrimaryClusterId = true; - } - const cachedHosts: HostInfo[] | null = this.getStoredTopology(); // This clusterId is a primary one and is about to create a new entry in the cache. // When a primary entry is created it needs to be suggested for other (non-primary) entries. // Remember a flag to do suggestion after cache is updated. - const needToSuggest: boolean = !cachedHosts && this.isPrimaryClusterId === true; if (!cachedHosts || forceUpdate) { // need to re-fetch the topology. if (!targetClient || !(await this.hostListProviderService.isClientValid(targetClient))) { @@ -216,9 +181,6 @@ export class RdsHostListProvider implements DynamicHostListProvider { const hosts = await this.queryForTopology(targetClient, this.hostListProviderService.getDialect()); if (hosts && hosts.length > 0) { this.storageService.set(this.clusterId, new Topology(hosts)); - if (needToSuggest) { - this.suggestPrimaryCluster(hosts); - } return new FetchTopologyResult(false, hosts); } } @@ -230,63 +192,6 @@ export class RdsHostListProvider implements DynamicHostListProvider { } } - private getSuggestedClusterId(hostAndPort: string): ClusterSuggestedResult | null { - const cache: ExpirationCache = this.storageService.getAll(Topology) as ExpirationCache; - if (!cache) { - return null; - } - for (const [key, hosts] of cache.getEntries()) { - const isPrimaryCluster: boolean = RdsHostListProvider.primaryClusterIdCache.get(key, false, this.suggestedClusterIdRefreshRateNano) ?? false; - if (key === hostAndPort) { - return new ClusterSuggestedResult(hostAndPort, isPrimaryCluster); - } - - if (hosts) { - for (const hostInfo of hosts.hosts) { - if (hostInfo.hostAndPort === hostAndPort) { - logger.debug(Messages.get("RdsHostListProvider.suggestedClusterId", key, hostAndPort)); - return new ClusterSuggestedResult(key, isPrimaryCluster); - } - } - } - } - return null; - } - - suggestPrimaryCluster(primaryClusterHosts: HostInfo[]): void { - if (!primaryClusterHosts) { - return; - } - - const primaryClusterHostUrls: Set = new Set(); - primaryClusterHosts.forEach((hostInfo) => { - primaryClusterHostUrls.add(hostInfo.url); - }); - - const cache: ExpirationCache = this.storageService.getAll(Topology) as ExpirationCache; - if (!cache) { - return; - } - for (const [clusterId, clusterHosts] of cache.getEntries()) { - const isPrimaryCluster: boolean | null = RdsHostListProvider.primaryClusterIdCache.get( - clusterId, - false, - this.suggestedClusterIdRefreshRateNano - ); - const suggestedPrimaryClusterId: string | null = RdsHostListProvider.suggestedPrimaryClusterIdCache.get(clusterId); - if (isPrimaryCluster || suggestedPrimaryClusterId || !clusterHosts) { - continue; - } - - for (const clusterHost of clusterHosts.hosts) { - if (primaryClusterHostUrls.has(clusterHost.url)) { - RdsHostListProvider.suggestedPrimaryClusterIdCache.put(clusterId, this.clusterId, this.suggestedClusterIdRefreshRateNano); - break; - } - } - } - } - async queryForTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { if (!isDialectTopologyAware(dialect)) { throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); @@ -366,8 +271,8 @@ export class RdsHostListProvider implements DynamicHostListProvider { } static clearAll(): void { - RdsHostListProvider.primaryClusterIdCache.clear(); - RdsHostListProvider.suggestedPrimaryClusterIdCache.clear(); + // No-op + // TODO: remove if still not used after full service container refactoring } clear(): void { @@ -420,13 +325,3 @@ export class FetchTopologyResult { this.isCachedData = isCachedData; } } - -class ClusterSuggestedResult { - clusterId: string; - isPrimaryClusterId: boolean; - - constructor(clusterId: string, isPrimaryClusterId: boolean) { - this.clusterId = clusterId; - this.isPrimaryClusterId = isPrimaryClusterId; - } -} diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts index d13bc052..0c32d376 100644 --- a/common/lib/utils/storage/storage_service.ts +++ b/common/lib/utils/storage/storage_service.ts @@ -56,10 +56,6 @@ export interface StorageService { */ set(key: unknown, item: V): void; - // TODO: temporary method to return all storage services for a specific item class. - // Should be removed along with the cluster id refactoring. - getAll(itemClass: Constructor): ExpirationCache | null; - /** * Gets an item stored in the storage service. * diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index c0ee775c..c61e0caa 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -234,8 +234,8 @@ export class WrapperProperties { "clusterId", "A unique identifier for the cluster. " + "Connections with the same cluster id share a cluster topology cache. " + - "If unspecified, a cluster id is automatically created for AWS RDS clusters.", - null + "If unspecified, a cluster id is '1'.", + "1" ); static readonly CLUSTER_INSTANCE_HOST_PATTERN = new WrapperProperty( diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md index e943fe2d..37789e59 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md @@ -62,14 +62,15 @@ Please refer to the [failover configuration guide](../FailoverConfigurationGuide In addition to the parameters that you can configure for the underlying driver, you can pass the following parameters to the AWS Advanced NodeJS Wrapper through the connection URL to specify additional failover behavior. -| Parameter | Value | Required | Description | Default Value | -| ------------------------------------ | :-----: | :------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | -| `failoverMode` | String | No | Defines a mode for failover process. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer instance and connects to a new writer when it changes.
- `reader-or-writer` - During failover, the wrapper tries to connect to any available/accessible reader instance. If no reader is available, the wrapper will connect to a writer instance. This logic mimics the logic of the Aurora read-only cluster endpoint.
- `strict-reader` - During failover, the wrapper tries to connect to any available reader instance. If no reader is available, the wrapper raises an error. Reader failover to a writer instance will only be allowed for single-instance clusters. This logic mimics the logic of the Aurora read-only cluster endpoint. | Default value depends on connection url. For Aurora read-only cluster endpoint, it's set to `reader-or-writer`. Otherwise, it's `strict-writer`. | -| `enableClusterAwareFailover` | Boolean | No | Set to `true` to enable the fast failover behavior offered by the AWS Advanced NodeJS Wrapper. Set to `false` for simple connections that do not require fast failover functionality. | `true` | -| `clusterTopologyRefreshRateMs` | Number | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. | `30000` | -| `clusterTopologyHighRefreshRateMs` | Number | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer is detected. | `100` | -| `failoverTimeoutMs` | Number | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | -| `failoverReaderHostSelectorStrategy` | String | No | The strategy that should be used to select a new reader host while opening a new connection. See: [Reader Selection Strategies](./../ReaderSelectionStrategies.md) for options. | `random` | +| Parameter | Value | Required | Description | Default Value | +| ------------------------------------ | :-----: | :-----------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------ | +| `failoverMode` | String | No | Defines a mode for failover process. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer instance and connects to a new writer when it changes.
- `reader-or-writer` - During failover, the wrapper tries to connect to any available/accessible reader instance. If no reader is available, the wrapper will connect to a writer instance. This logic mimics the logic of the Aurora read-only cluster endpoint.
- `strict-reader` - During failover, the wrapper tries to connect to any available reader instance. If no reader is available, the wrapper raises an error. Reader failover to a writer instance will only be allowed for single-instance clusters. This logic mimics the logic of the Aurora read-only cluster endpoint. | Default value depends on connection url. For Aurora read-only cluster endpoint, it's set to `reader-or-writer`. Otherwise, it's `strict-writer`. | +| `enableClusterAwareFailover` | Boolean | No | Set to `true` to enable the fast failover behavior offered by the AWS Advanced NodeJS Wrapper. Set to `false` for simple connections that do not require fast failover functionality. | `true` | +| `clusterTopologyRefreshRateMs` | Number | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. | `30000` | +| `clusterTopologyHighRefreshRateMs` | Number | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer is detected. | `100` | +| `failoverTimeoutMs` | Number | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | +| `clusterId` | String | If using multiple database clusters, yes; otherwise, no | A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. This parameter is optional and defaults to `1`. When supporting multiple database clusters, this parameter becomes mandatory. Each connection string must include the `clusterId` parameter with a value that can be any number or string. However, all connection strings associated with the same database cluster must use identical `clusterId` values, while connection strings belonging to different database clusters must specify distinct values. Examples of value: `1`, `2`, `1234`, `abc-1`, `abc-2`. | `1` | +| `failoverReaderHostSelectorStrategy` | String | No | The strategy that should be used to select a new reader host while opening a new connection. See: [Reader Selection Strategies](./../ReaderSelectionStrategies.md) for options. | `random` | Please refer to the original [Failover Plugin](./UsingTheFailoverPlugin.md) for more details about error codes, configurations, connection pooling and sample codes. diff --git a/tests/unit/rds_host_list_provider.test.ts b/tests/unit/rds_host_list_provider.test.ts index 1801169c..86f20d00 100644 --- a/tests/unit/rds_host_list_provider.test.ts +++ b/tests/unit/rds_host_list_provider.test.ts @@ -201,199 +201,6 @@ describe("testRdsHostListProvider", () => { expect(rdsHostListProvider.getStoredTopology()).toBeNull(); }); - it("testTopologyCache_noSuggestedClusterId", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const provider1 = getRdsHostListProvider("cluster-a.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(storageService.getAll(Topology)).toBeNull(); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-b.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider2 = spy(provider2); - - const topologyClusterB: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-b-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - when(spiedProvider2.queryForTopology(instance(mockClientWrapper), anything())).thenReturn(Promise.resolve(topologyClusterB)); - - expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterB); - expect(storageService.getAll(Topology).size()).toEqual(2); - }); - - it("testTopologyCache_suggestedClusterIdForRds", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(storageService.getAll(Topology)).toBeNull(); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(storageService.getAll(Topology).size()).toEqual(1); - }); - - it("testTopologyCache_suggestedClusterIdForInstance", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(storageService.getAll(Topology)).toBeNull(); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("instance-a-3.xyz.us-east-2.rds.amazonaws.com"); - - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(mockClientWrapper)).toEqual(topologyClusterA); - expect(storageService.getAll(Topology).size()).toEqual(1); - }); - - it("testTopologyCache_acceptSuggestion", async () => { - RdsHostListProvider.clearAll(); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - - const topologyClusterA: HostInfo[] = [ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-1.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.WRITER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-2.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-a-3.xyz.us-east-2.rds.amazonaws.com", - role: HostRole.READER - }) - ]; - - const provider1 = getRdsHostListProvider("instance-a-2.xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider1 = spy(provider1); - - when(spiedProvider1.queryForTopology(anything(), anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(storageService.getAll(Topology)).toBeNull(); - - const topologyProvider1: HostInfo[] = await provider1.refresh(mockClientWrapper); - expect(topologyProvider1).toEqual(topologyClusterA); - - const provider2 = getRdsHostListProvider("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - const spiedProvider2 = spy(provider2); - - when(spiedProvider2.queryForTopology(anything(), anything())).thenReturn(Promise.resolve(topologyClusterA)); - expect(provider2.clusterId).not.toEqual(provider1.clusterId); - expect(provider1.isPrimaryClusterId).toBeFalsy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - - expect(await provider2.refresh(instance(mockClientWrapper))).toEqual(topologyClusterA); - expect(storageService.getAll(Topology).size()).toEqual(2); - expect(RdsHostListProvider.suggestedPrimaryClusterIdCache.get(provider1.clusterId)).toEqual("cluster-a.cluster-xyz.us-east-2.rds.amazonaws.com"); - - expect(await provider1.forceRefresh(instance(mockClientWrapper))).toEqual(topologyClusterA); - expect(provider2.clusterId).toEqual(provider1.clusterId); - expect(storageService.getAll(Topology).size()).toEqual(2); - expect(provider1.isPrimaryClusterId).toBeTruthy(); - expect(provider2.isPrimaryClusterId).toBeTruthy(); - }); - it("testIdentifyConnectionWithInvalidHostIdQuery", async () => { when(mockDialect.identifyConnection(anything())).thenThrow(new AwsWrapperError("bad things")); From 5df709cd14e8a5387fe63289eaf8a1c766b7f7d9 Mon Sep 17 00:00:00 2001 From: Karen Chen <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 25 Feb 2026 18:06:53 -0800 Subject: [PATCH 06/20] chore: consolidate cache items to one location --- .github/workflows/integration_tests.yml | 1 - common/lib/utils/cache_map.ts | 45 +++++++++++++------- common/lib/utils/core_services_container.ts | 4 ++ common/lib/utils/sliding_expiration_cache.ts | 42 ++++++------------ common/lib/utils/storage/storage_service.ts | 13 +++--- common/lib/wrapper_property.ts | 4 +- tests/unit/sliding_expiration_cache.test.ts | 16 +++---- 7 files changed, 62 insertions(+), 63 deletions(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4b99a950..db65f2f6 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,7 +5,6 @@ on: push: branches: - main - - dev/v3 paths-ignore: - '**/*.md' - '**/*.jpg' diff --git a/common/lib/utils/cache_map.ts b/common/lib/utils/cache_map.ts index 8728dcab..19485421 100644 --- a/common/lib/utils/cache_map.ts +++ b/common/lib/utils/cache_map.ts @@ -14,17 +14,40 @@ limitations under the License. */ -class CacheItem { - readonly item: V; - private readonly expirationTimeNanos: bigint; +import { getTimeInNanos } from "./utils"; - constructor(item: V, expirationTimeNanos: bigint) { +export class CacheItem { + private readonly item: V; + private _expirationTimeNs: bigint; + + constructor(item: V, expirationTime: bigint) { this.item = item; - this.expirationTimeNanos = expirationTimeNanos; + this._expirationTimeNs = expirationTime; } isExpired(): boolean { - return process.hrtime.bigint() > this.expirationTimeNanos; + if (this._expirationTimeNs <= 0) { + // No expiration time. + return false; + } + return getTimeInNanos() > this._expirationTimeNs; + } + + get(returnExpired: boolean = false): V | null { + return this.isExpired() && !returnExpired ? null : this.item; + } + + updateExpiration(expirationIntervalNanos: bigint): CacheItem { + this._expirationTimeNs = getTimeInNanos() + expirationIntervalNanos; + return this; + } + + get expirationTimeNs(): bigint { + return this._expirationTimeNs; + } + + toString(): string { + return `CacheItem [item=${this.item}, expirationTime=${this._expirationTimeNs}]`; } } @@ -38,7 +61,7 @@ export class CacheMap { get(key: K, defaultItemValue?: any, itemExpirationNano?: any): V | null { const cacheItem: CacheItem | undefined = this.cache.get(key); if (cacheItem && !cacheItem.isExpired()) { - return cacheItem.item; + return cacheItem.get(); } if (!defaultItemValue || !itemExpirationNano) { @@ -74,14 +97,6 @@ export class CacheMap { this.cache.clear(); } - getEntries(): Map { - const entries: Map = new Map(); - this.cache.forEach((v, k) => { - entries.set(k, v.item); - }); - return entries; - } - protected cleanUp() { if (this.cleanupTimeNanos < process.hrtime.bigint()) { this.cleanupTimeNanos = process.hrtime.bigint() + this.cleanupIntervalNanos; diff --git a/common/lib/utils/core_services_container.ts b/common/lib/utils/core_services_container.ts index c9ab89fe..8ae9f27f 100644 --- a/common/lib/utils/core_services_container.ts +++ b/common/lib/utils/core_services_container.ts @@ -45,4 +45,8 @@ export class CoreServicesContainer { // getMonitorService(): MonitorService { // return this.monitorService; // } + + static releaseResources(): void { + CoreServicesContainer.INSTANCE.storageService.releaseResources(); + } } diff --git a/common/lib/utils/sliding_expiration_cache.ts b/common/lib/utils/sliding_expiration_cache.ts index b2d27abb..904d9dc3 100644 --- a/common/lib/utils/sliding_expiration_cache.ts +++ b/common/lib/utils/sliding_expiration_cache.ts @@ -16,25 +16,7 @@ import { getTimeInNanos } from "./utils"; import { MapUtils } from "./map_utils"; - -class CacheItem { - readonly item: V; - private _expirationTimeNanos: bigint; - - constructor(item: V, expirationTimeNanos: bigint) { - this.item = item; - this._expirationTimeNanos = expirationTimeNanos; - } - - get expirationTimeNs(): bigint { - return this._expirationTimeNanos; - } - - updateExpiration(expirationIntervalNanos: bigint): CacheItem { - this._expirationTimeNanos = getTimeInNanos() + expirationIntervalNanos; - return this; - } -} +import { CacheItem } from "./cache_map"; export class SlidingExpirationCache { protected _cleanupIntervalNanos: bigint = BigInt(10 * 60_000_000_000); // 10 minutes @@ -69,29 +51,29 @@ export class SlidingExpirationCache { computeIfAbsent(key: K, mappingFunc: (key: K) => V, itemExpirationNanos: bigint): V | null { this.cleanUp(); const cacheItem = MapUtils.computeIfAbsent(this.map, key, (k) => new CacheItem(mappingFunc(k), getTimeInNanos() + itemExpirationNanos)); - return cacheItem?.updateExpiration(itemExpirationNanos).item ?? null; + return cacheItem?.updateExpiration(itemExpirationNanos).get() ?? null; } putIfAbsent(key: K, value: V, itemExpirationNanos: bigint): V | null { const cacheItem = MapUtils.putIfAbsent(this.map, key, new CacheItem(value, getTimeInNanos() + itemExpirationNanos)); - return cacheItem?.item ?? null; + return cacheItem?.get(true) ?? null; } get(key: K, itemExpirationNanos?: bigint): V | undefined { this.cleanUp(); const cacheItem = this.map.get(key); - if (cacheItem?.item && itemExpirationNanos) { + if (cacheItem?.get(true) && itemExpirationNanos) { cacheItem.updateExpiration(itemExpirationNanos); - return cacheItem.item; + return cacheItem.get(); } - return cacheItem?.item; + return cacheItem?.get(true); } put(key: K, value: V, itemExpirationNanos: bigint): V | null { this.cleanUp(); const cacheItem = new CacheItem(value, getTimeInNanos() + itemExpirationNanos); this.map.set(key, cacheItem); - return cacheItem.item; + return cacheItem.get(true); } remove(key: K): void { @@ -102,7 +84,7 @@ export class SlidingExpirationCache { removeAndDispose(key: K): void { const cacheItem = MapUtils.remove(this.map, key); if (cacheItem != null && this._itemDisposalFunc != null) { - this._itemDisposalFunc(cacheItem.item); + this._itemDisposalFunc(cacheItem.get(true)); } } @@ -110,7 +92,7 @@ export class SlidingExpirationCache { let item; MapUtils.computeIfPresent(this.map, key, (key, cacheItem) => { if (this.shouldCleanupItem(cacheItem)) { - item = cacheItem.item; + item = cacheItem.get(true); return null; } return cacheItem; @@ -123,15 +105,15 @@ export class SlidingExpirationCache { shouldCleanupItem(cacheItem: CacheItem): boolean { if (this._shouldDisposeFunc != null) { - return getTimeInNanos() > cacheItem.expirationTimeNs && this._shouldDisposeFunc(cacheItem.item); + return cacheItem.isExpired() && this._shouldDisposeFunc(cacheItem.get(true)); } - return getTimeInNanos() > cacheItem.expirationTimeNs; + return cacheItem.isExpired(); } clear(): void { for (const [key, val] of this.map.entries()) { if (val !== undefined && this._itemDisposalFunc !== undefined) { - this._itemDisposalFunc(val.item); + this._itemDisposalFunc(val.get()); } } this.map.clear(); diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts index 0c32d376..87706130 100644 --- a/common/lib/utils/storage/storage_service.ts +++ b/common/lib/utils/storage/storage_service.ts @@ -102,6 +102,12 @@ export interface StorageService { * @returns The number of items stored for the given item class */ size(itemClass: Constructor): number; + + /** + * Cleanup method to stop the cleanup interval timer. + * Should be called when the service is no longer needed. + */ + releaseResources(): void; } type CacheSupplier = () => ExpirationCache; @@ -129,7 +135,6 @@ export class StorageServiceImpl implements StorageService { } protected removeExpiredItems(): void { - logger.debug("StorageServiceImpl: Removing expired items"); for (const cache of this.caches.values()) { cache.removeExpiredEntries(); } @@ -246,11 +251,7 @@ export class StorageServiceImpl implements StorageService { StorageServiceImpl.defaultCacheSuppliers.set(itemClass, supplier); } - /** - * Cleanup method to stop the cleanup interval timer. - * Should be called when the service is no longer needed. - */ - destroy(): void { + releaseResources(): void { if (this.cleanupIntervalHandle) { clearInterval(this.cleanupIntervalHandle); this.cleanupIntervalHandle = undefined; diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index c61e0caa..16aff1dc 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -232,9 +232,7 @@ export class WrapperProperties { static readonly CLUSTER_ID = new WrapperProperty( "clusterId", - "A unique identifier for the cluster. " + - "Connections with the same cluster id share a cluster topology cache. " + - "If unspecified, a cluster id is '1'.", + "A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. If unspecified, cluster id will be '1'.", "1" ); diff --git a/tests/unit/sliding_expiration_cache.test.ts b/tests/unit/sliding_expiration_cache.test.ts index 490e772f..181924f9 100644 --- a/tests/unit/sliding_expiration_cache.test.ts +++ b/tests/unit/sliding_expiration_cache.test.ts @@ -15,7 +15,7 @@ */ import { SlidingExpirationCache } from "../../common/lib/utils/sliding_expiration_cache"; -import { convertNanosToMs, sleep } from "../../common/lib/utils/utils"; +import { convertMsToNanos, convertNanosToMs, sleep } from "../../common/lib/utils/utils"; import { SlidingExpirationCacheWithCleanupTask } from "../../common/lib/utils/sliding_expiration_cache_with_cleanup_task"; class DisposableItem { @@ -48,9 +48,9 @@ class AsyncDisposableItem { describe("test_sliding_expiration_cache", () => { it("test compute if absent", async () => { const target = new SlidingExpirationCache(BigInt(50_000_000)); - const result1 = target.computeIfAbsent(1, () => "a", BigInt(1)); + const result1 = target.computeIfAbsent(1, () => "a", convertMsToNanos(100)); const originalItemExpiration = target.map.get(1)!.expirationTimeNs; - const result2 = target.computeIfAbsent(1, () => "b", BigInt(5)); + const result2 = target.computeIfAbsent(1, () => "b", convertMsToNanos(500)); const updatedItemExpiration = target.map.get(1)?.expirationTimeNs; expect(updatedItemExpiration).toBeGreaterThan(originalItemExpiration); @@ -59,7 +59,7 @@ describe("test_sliding_expiration_cache", () => { expect(target.get(1)).toEqual("a"); await sleep(700); - const result3 = target.computeIfAbsent(1, () => "b", BigInt(5)); + const result3 = target.computeIfAbsent(1, () => "b", convertMsToNanos(500)); expect(result3).toEqual("b"); expect(target.get(1)).toEqual("b"); }); @@ -71,19 +71,19 @@ describe("test_sliding_expiration_cache", () => { (item) => item.dispose() ); const itemToRemove = new DisposableItem(true); - let result = target.computeIfAbsent("itemToRemove", () => itemToRemove, BigInt(15_000_000_000)); + let result = target.computeIfAbsent("itemToRemove", () => itemToRemove, convertMsToNanos(15_000)); expect(itemToRemove).toEqual(result); const itemToCleanup = new DisposableItem(true); - result = target.computeIfAbsent("itemToCleanup", () => itemToCleanup, BigInt(1)); + result = target.computeIfAbsent("itemToCleanup", () => itemToCleanup, convertMsToNanos(100)); expect(itemToCleanup).toEqual(result); const nonDisposableItem = new DisposableItem(false); - result = target.computeIfAbsent("nonDisposableItem", () => nonDisposableItem, BigInt(1)); + result = target.computeIfAbsent("nonDisposableItem", () => nonDisposableItem, convertMsToNanos(100)); expect(nonDisposableItem).toEqual(result); const nonExpiredItem = new DisposableItem(true); - result = target.computeIfAbsent("nonExpiredItem", () => nonExpiredItem, BigInt(15_000_000_000)); + result = target.computeIfAbsent("nonExpiredItem", () => nonExpiredItem, convertMsToNanos(15_000)); expect(nonExpiredItem).toEqual(result); await sleep(700); From de41b23ec22bb00507bfa0391ff86a35d0d9300f Mon Sep 17 00:00:00 2001 From: Karen Chen <64801825+karenc-bq@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:28:24 -0800 Subject: [PATCH 07/20] chore: move log messages and fix iam integration tests for multi-az environments --- common/lib/utils/messages.ts | 4 +++- common/lib/utils/storage/storage_service.ts | 16 ++++------------ .../integration/host/util/AuroraTestUtility.java | 1 + 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 5c0af393..b0469231 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -374,7 +374,9 @@ const MESSAGES: Record = { "Blue/Green Deployment switchover is still in progress and a corresponding host for '%s' is not found after %s ms. Try to connect again later.", "Bgd.correspondingHostFoundContinueWithConnect": "A corresponding host for '%s' is found. Continue with connect call. The call was suspended for %s ms.", - "Bgd.completedContinueWithConnect": "Blue/Green Deployment status is completed. Continue with 'connect' call. The call was suspended for %s ms." + "Bgd.completedContinueWithConnect": "Blue/Green Deployment status is completed. Continue with 'connect' call. The call was suspended for %s ms.", + "StorageService.itemClassNotRegistered": "StorageServiceImpl: Item class not registered: %s", + "StorageService.unexpectedValueMismatch": "StorageServiceImpl: Unexpected value mismatch for %s: %s" }; export class Messages { diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts index 87706130..06f6ca00 100644 --- a/common/lib/utils/storage/storage_service.ts +++ b/common/lib/utils/storage/storage_service.ts @@ -15,9 +15,10 @@ */ import { Constructor, ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; -import { logger } from "../../../logutils"; import { ExpirationCache } from "./expiration_cache"; import { Topology } from "../../host_list_provider/topology"; +import { AwsWrapperError } from "../errors"; +import { Messages } from "../messages"; const DEFAULT_CLEANUP_INTERVAL_NANOS = 5 * 60 * 1_000_000_000; // 5 minutes @@ -165,7 +166,7 @@ export class StorageServiceImpl implements StorageService { if (!cache) { const supplier = StorageServiceImpl.defaultCacheSuppliers.get(itemClass); if (!supplier) { - throw new Error(`StorageServiceImpl: Item class not registered: ${itemClass.name}`); + throw new AwsWrapperError(Messages.get("StorageService.itemClassNotRegistered", itemClass.name)); } cache = supplier(); this.caches.set(itemClass, cache); @@ -174,18 +175,10 @@ export class StorageServiceImpl implements StorageService { try { cache.put(key, item); } catch (error) { - throw new Error(`StorageServiceImpl: Unexpected value mismatch for ${itemClass.name}: ${error}`); + throw new AwsWrapperError(Messages.get("StorageService.unexpectedValueMismatch", itemClass.name, String(error))); } } - getAll(itemClass: Constructor): ExpirationCache | null { - const cache = this.caches.get(itemClass); - if (!cache) { - return null; - } - return cache; - } - get(itemClass: Constructor, key?: unknown): V | null { const cache = this.caches.get(itemClass); if (!cache) { @@ -201,7 +194,6 @@ export class StorageServiceImpl implements StorageService { return value as V; } - logger.debug(`StorageServiceImpl: Item class mismatch for key ${String(key)}: ` + `expected ${itemClass.name}, got ${value.constructor.name}`); return null; } diff --git a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java index 54f19846..cb1bd3cd 100644 --- a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java +++ b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java @@ -1117,6 +1117,7 @@ public void addAuroraAwsIamUser( } else { stmt.execute("GRANT ALL PRIVILEGES ON `%`.* TO '" + dbUser + "'@'%';"); } + stmt.execute("GRANT REPLICATION CLIENT ON *.* TO '" + dbUser + "'@'%';"); // BG switchover status needs it. stmt.execute("GRANT SELECT ON mysql.* TO '" + dbUser + "'@'%';"); From ddb89fc3c74f6ede40f5296d15ed5c6adb967421 Mon Sep 17 00:00:00 2001 From: Karen Chen <64801825+karenc-bq@users.noreply.github.com> Date: Thu, 26 Feb 2026 22:42:44 -0800 Subject: [PATCH 08/20] chore: disable multi-az integration tests temporarily --- .github/workflows/integration_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index db65f2f6..ecd6a876 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -60,7 +60,7 @@ jobs: fail-fast: false matrix: versions: [ "default", "latest" ] - dbEngine: ["aurora-mysql", "aurora-postgres", "multi-az-mysql", "multi-az-postgres" ] + dbEngine: ["aurora-mysql", "aurora-postgres" ] steps: - name: 'Clone repository' From 11714cb7dda4a59580e2638bb48760a138297390 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Fri, 27 Feb 2026 16:57:58 -0800 Subject: [PATCH 09/20] feat: GDB database dialects (#616) --- .../topology_aware_database_dialect.ts | 35 ++++++ common/lib/topology_aware_database_dialect.ts | 31 ----- common/lib/utils/rds_url_type.ts | 20 ++-- common/lib/utils/rds_utils.ts | 14 ++- common/lib/utils/utils.ts | 2 +- .../DatabaseDialects.md | 2 +- .../dialect/aurora_mysql_database_dialect.ts | 4 +- .../global_aurora_mysql_database_dialect.ts | 99 ++++++++++++++++ .../rds_multi_az_mysql_database_dialect.ts | 2 +- pg/lib/dialect/aurora_pg_database_dialect.ts | 2 +- .../global_aurora_pg_database_dialect.ts | 109 ++++++++++++++++++ .../rds_multi_az_pg_database_dialect.ts | 2 +- 12 files changed, 274 insertions(+), 48 deletions(-) create mode 100644 common/lib/database_dialect/topology_aware_database_dialect.ts delete mode 100644 common/lib/topology_aware_database_dialect.ts create mode 100644 mysql/lib/dialect/global_aurora_mysql_database_dialect.ts create mode 100644 pg/lib/dialect/global_aurora_pg_database_dialect.ts diff --git a/common/lib/database_dialect/topology_aware_database_dialect.ts b/common/lib/database_dialect/topology_aware_database_dialect.ts new file mode 100644 index 00000000..432a3583 --- /dev/null +++ b/common/lib/database_dialect/topology_aware_database_dialect.ts @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HostInfo } from "../host_info"; +import { HostListProvider } from "../host_list_provider/host_list_provider"; +import { HostRole } from "../host_role"; +import { ClientWrapper } from "../client_wrapper"; + +export interface TopologyAwareDatabaseDialect { + queryForTopology(client: ClientWrapper, hostListProvider: HostListProvider): Promise; + + identifyConnection(targetClient: ClientWrapper): Promise; + + getHostRole(client: ClientWrapper): Promise; + + // Returns the host id of the targetClient if it is connected to a writer, null otherwise. + getWriterId(targetClient: ClientWrapper): Promise; +} + +export interface GlobalAuroraTopologyDialect extends TopologyAwareDatabaseDialect { + getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise; +} diff --git a/common/lib/topology_aware_database_dialect.ts b/common/lib/topology_aware_database_dialect.ts deleted file mode 100644 index e5d6bc1c..00000000 --- a/common/lib/topology_aware_database_dialect.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { HostInfo } from "./host_info"; -import { HostListProvider } from "./host_list_provider/host_list_provider"; -import { HostRole } from "./host_role"; -import { ClientWrapper } from "./client_wrapper"; - -export interface TopologyAwareDatabaseDialect { - queryForTopology(client: ClientWrapper, hostListProvider: HostListProvider): Promise; - - identifyConnection(targetClient: ClientWrapper): Promise; - - getHostRole(client: ClientWrapper): Promise; - - // Returns the host id of the targetClient if it is connected to a writer, null otherwise. - getWriterId(targetClient: ClientWrapper): Promise; -} diff --git a/common/lib/utils/rds_url_type.ts b/common/lib/utils/rds_url_type.ts index ebcac9b8..46300354 100644 --- a/common/lib/utils/rds_url_type.ts +++ b/common/lib/utils/rds_url_type.ts @@ -15,17 +15,19 @@ */ export class RdsUrlType { - public static readonly IP_ADDRESS = new RdsUrlType(false, false); - public static readonly RDS_WRITER_CLUSTER = new RdsUrlType(true, true); - public static readonly RDS_READER_CLUSTER = new RdsUrlType(true, true); - public static readonly RDS_CUSTOM_CLUSTER = new RdsUrlType(true, false); - public static readonly RDS_PROXY = new RdsUrlType(true, false); - public static readonly RDS_INSTANCE = new RdsUrlType(true, false); - public static readonly RDS_AURORA_LIMITLESS_DB_SHARD_GROUP = new RdsUrlType(true, false); - public static readonly OTHER = new RdsUrlType(false, false); + public static readonly IP_ADDRESS = new RdsUrlType(false, false, false); + public static readonly RDS_WRITER_CLUSTER = new RdsUrlType(true, true, true); + public static readonly RDS_READER_CLUSTER = new RdsUrlType(true, true, true); + public static readonly RDS_CUSTOM_CLUSTER = new RdsUrlType(true, false, true); + public static readonly RDS_PROXY = new RdsUrlType(true, false, true); + public static readonly RDS_INSTANCE = new RdsUrlType(true, false, true); + public static readonly RDS_AURORA_LIMITLESS_DB_SHARD_GROUP = new RdsUrlType(true, false, true); + public static readonly RDS_GLOBAL_WRITER_CLUSTER = new RdsUrlType(true, true, false); + public static readonly OTHER = new RdsUrlType(false, false, false); private constructor( public readonly isRds: boolean, - public readonly isRdsCluster: boolean + public readonly isRdsCluster: boolean, + public readonly hasRegion: boolean ) {} } diff --git a/common/lib/utils/rds_utils.ts b/common/lib/utils/rds_utils.ts index 866dfaf9..66543349 100644 --- a/common/lib/utils/rds_utils.ts +++ b/common/lib/utils/rds_utils.ts @@ -60,6 +60,10 @@ export class RdsUtils { // https://aws.amazon.com/compliance/fips/#FIPS_Endpoints_by_Service // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html + // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Concepts.Aurora_Fea_Regions_DB-eng.Feature.GlobalDatabase.html + private static readonly AURORA_GLOBAL_WRITER_DNS_PATTERN = + /^(?.+)\.(?global-)?(?[a-zA-Z0-9]+\.global\.rds\.amazonaws\.com\.?)$/i; + private static readonly AURORA_DNS_PATTERN = /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; private static readonly AURORA_INSTANCE_PATTERN = /^(?.+)\.(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; @@ -231,6 +235,11 @@ export class RdsUtils { return null; } + public isGlobalDbWriterClusterDns(host: string): boolean { + const dnsGroup = this.getDnsGroup(host); + return equalsIgnoreCase(dnsGroup, "global-"); + } + public isWriterClusterDns(host: string): boolean { const dnsGroup = this.getDnsGroup(host); return equalsIgnoreCase(dnsGroup, "cluster-"); @@ -300,6 +309,8 @@ export class RdsUtils { if (this.isIPv4(host) || this.isIPv6(host)) { return RdsUrlType.IP_ADDRESS; + } else if (this.isGlobalDbWriterClusterDns(host)) { + return RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER; } else if (this.isWriterClusterDns(host)) { return RdsUrlType.RDS_WRITER_CLUSTER; } else if (this.isReaderClusterDns(host)) { @@ -382,7 +393,8 @@ export class RdsUtils { RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, - RdsUtils.AURORA_GOV_DNS_PATTERN + RdsUtils.AURORA_GOV_DNS_PATTERN, + RdsUtils.AURORA_GLOBAL_WRITER_DNS_PATTERN ); return this.getRegexGroup(matcher, RdsUtils.DNS_GROUP); } diff --git a/common/lib/utils/utils.ts b/common/lib/utils/utils.ts index 32dd0a47..ef1e062b 100644 --- a/common/lib/utils/utils.ts +++ b/common/lib/utils/utils.ts @@ -20,7 +20,7 @@ import { WrapperProperties } from "../wrapper_property"; import { HostRole } from "../host_role"; import { logger } from "../../logutils"; import { AwsWrapperError, InternalQueryTimeoutError } from "./errors"; -import { TopologyAwareDatabaseDialect } from "../topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; export function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); diff --git a/docs/using-the-nodejs-wrapper/DatabaseDialects.md b/docs/using-the-nodejs-wrapper/DatabaseDialects.md index 04aa92f5..d6834ec3 100644 --- a/docs/using-the-nodejs-wrapper/DatabaseDialects.md +++ b/docs/using-the-nodejs-wrapper/DatabaseDialects.md @@ -33,7 +33,7 @@ Dialect codes specify what kind of database any connections will be made to. If you are interested in using the AWS Advanced NodeJS Wrapper but your desired database type is not currently supported, it is possible to create a custom dialect. -To create a custom dialect, implement the [`DatabaseDialect`](../../common/lib/database_dialect/database_dialect.ts) interface. For databases clusters that are aware of their topology, the [`TopologyAwareDatabaseDialect`](../../common/lib/topology_aware_database_dialect.ts) interface should also be implemented. For database clusters that use an [Aurora Limitless Database](../../docs/using-the-nodejs-wrapper/using-plugins/UsingTheLimitlessConnectionPlugin.md#what-is-amazon-aurora-limitless-database) then [`LimitlessDatabaseDialect`](../../common/lib/database_dialect/limitless_database_dialect.ts) should be implemented. +To create a custom dialect, implement the [`DatabaseDialect`](../../common/lib/database_dialect/database_dialect.ts) interface. For databases clusters that are aware of their topology, the [`TopologyAwareDatabaseDialect`](../../common/lib/database_dialect/topology_aware_database_dialect.ts) interface should also be implemented. For database clusters that use an [Aurora Limitless Database](../../docs/using-the-nodejs-wrapper/using-plugins/UsingTheLimitlessConnectionPlugin.md#what-is-amazon-aurora-limitless-database) then [`LimitlessDatabaseDialect`](../../common/lib/database_dialect/limitless_database_dialect.ts) should be implemented. See the following classes for examples: diff --git a/mysql/lib/dialect/aurora_mysql_database_dialect.ts b/mysql/lib/dialect/aurora_mysql_database_dialect.ts index c2843edc..08976c43 100644 --- a/mysql/lib/dialect/aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/aurora_mysql_database_dialect.ts @@ -19,7 +19,7 @@ import { HostListProviderService } from "../../../common/lib/host_list_provider_ import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { HostInfo } from "../../../common/lib/host_info"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { HostRole } from "../../../common/lib/host_role"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; @@ -87,7 +87,7 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements try { const writerId: string = res[0][0]["server_id"]; return writerId ? writerId : null; - } catch (e) { + } catch (e: any) { if (e.message.includes("Cannot read properties of undefined")) { // Query returned no result, targetClient is not connected to a writer. return null; diff --git a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts new file mode 100644 index 00000000..d05d41db --- /dev/null +++ b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts @@ -0,0 +1,99 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { AuroraMySQLDatabaseDialect } from "./aurora_mysql_database_dialect"; +import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; +import { ClientWrapper } from "../../../common/lib/client_wrapper"; +import { HostInfo } from "../../../common/lib"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; + +export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect implements GlobalAuroraTopologyDialect { + private static readonly GLOBAL_STATUS_TABLE_EXISTS_QUERY = + "SELECT 1 AS tmp FROM information_schema.tables WHERE" + + " upper(table_schema) = 'INFORMATION_SCHEMA' AND upper(table_name) = 'AURORA_GLOBAL_DB_STATUS'"; + + private static readonly GLOBAL_INSTANCE_STATUS_EXISTS_QUERY = + "SELECT 1 AS tmp FROM information_schema.tables WHERE" + + " upper(table_schema) = 'INFORMATION_SCHEMA' AND upper(table_name) = 'AURORA_GLOBAL_DB_INSTANCE_STATUS'"; + + private static readonly GLOBAL_TOPOLOGY_QUERY = + "SELECT SERVER_ID, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS IS_WRITER, " + + "VISIBILITY_LAG_IN_MSEC, AWS_REGION " + + "FROM information_schema.aurora_global_db_instance_status"; + + private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM information_schema.aurora_global_db_status"; + + private static readonly REGION_BY_INSTANCE_ID_QUERY = + "SELECT AWS_REGION FROM information_schema.aurora_global_db_instance_status WHERE SERVER_ID = ?"; + + async isDialect(targetClient: ClientWrapper): Promise { + try { + // Check if both global status tables exist + const [statusRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_STATUS_TABLE_EXISTS_QUERY); + if (!statusRows?.[0]) { + return false; + } + + const [instanceStatusRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_INSTANCE_STATUS_EXISTS_QUERY); + if (!instanceStatusRows?.[0]) { + return false; + } + + // Check if there are multiple regions + const [regionCountRows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.REGION_COUNT_QUERY); + if (!regionCountRows?.[0]) { + return false; + } + + const awsRegionCount = regionCountRows[0]["count(1)"]; + return awsRegionCount > 1; + } catch { + return false; + } + } + + getDialectUpdateCandidates(): string[] { + return []; + } + + async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + const res = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); + const hosts: HostInfo[] = []; + const rows: any[] = res[0]; + rows.forEach((row) => { + // According to the topology query the result set + // should contain 4 columns: node ID, 1/0 (writer/reader), CPU utilization, node lag in time. + const hostName: string = row["server_id"]; + const isWriter: boolean = row["is_writer"]; + const hostLag: number = row["visibility_lag_in_msec"] ?? 0; // visibility_lag_in_msec is nullable. + const awsRegion: string = row["aws_region"]; // TODO: update this after topologyUtils PR is merged. + const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100, Date.now() /* TODO: update this after topologyUtils PR is merged */); + hosts.push(host); + }); + return hosts; + } + + async getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise { + try { + const [rows] = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.REGION_BY_INSTANCE_ID_QUERY, [instanceId]); + if (!rows?.[0]) { + return null; + } + return rows[0]["AWS_REGION"] ?? null; + } catch { + return null; + } + } +} diff --git a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts index da6d26a7..52660758 100644 --- a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts +++ b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts @@ -23,7 +23,7 @@ import { HostRole } from "../../../common/lib/host_role"; import { Messages } from "../../../common/lib/utils/messages"; import { logger } from "../../../common/logutils"; import { AwsWrapperError } from "../../../common/lib/utils/errors"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { FailoverRestriction } from "../../../common/lib/plugins/failover/failover_restriction"; import { WrapperProperties } from "../../../common/lib/wrapper_property"; diff --git a/pg/lib/dialect/aurora_pg_database_dialect.ts b/pg/lib/dialect/aurora_pg_database_dialect.ts index 9009ddbb..250b39ee 100644 --- a/pg/lib/dialect/aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/aurora_pg_database_dialect.ts @@ -18,7 +18,7 @@ import { PgDatabaseDialect } from "./pg_database_dialect"; import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { HostInfo, HostRole } from "../../../common/lib"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; diff --git a/pg/lib/dialect/global_aurora_pg_database_dialect.ts b/pg/lib/dialect/global_aurora_pg_database_dialect.ts new file mode 100644 index 00000000..7061b09f --- /dev/null +++ b/pg/lib/dialect/global_aurora_pg_database_dialect.ts @@ -0,0 +1,109 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AuroraPgDatabaseDialect } from "./aurora_pg_database_dialect"; +import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; +import { ClientWrapper } from "../../../common/lib/client_wrapper"; +import { HostInfo } from "../../../common/lib"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; + +export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect implements GlobalAuroraTopologyDialect { + private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_status'::regproc"; + + private static readonly GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_instance_status'::regproc"; + + private static readonly GLOBAL_TOPOLOGY_QUERY = + "SELECT SERVER_ID, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS IS_WRITER, " + + "VISIBILITY_LAG_IN_MSEC, AWS_REGION " + + "FROM aurora_global_db_instance_status()"; + + private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM aurora_global_db_status()"; + + private static readonly REGION_BY_INSTANCE_ID_QUERY = "SELECT AWS_REGION FROM aurora_global_db_instance_status() WHERE SERVER_ID = $1"; + + async isDialect(targetClient: ClientWrapper): Promise { + try { + // First check if aurora_stat_utils extension is available + const extensionsResult = await targetClient.query( + "SELECT (setting LIKE '%aurora_stat_utils%') AS aurora_stat_utils FROM pg_catalog.pg_settings WHERE name OPERATOR(pg_catalog.=) 'rds.extensions'" + ); + + if (!extensionsResult.rows?.[0]) { + return false; + } + + const auroraUtils = extensionsResult.rows[0]["aurora_stat_utils"]; + if (!auroraUtils) { + return false; + } + + // Check if both global status functions exist + const statusResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_STATUS_FUNC_EXISTS_QUERY); + if (!statusResult.rows?.[0]) { + return false; + } + + const instanceStatusResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY); + if (!instanceStatusResult.rows?.[0]) { + return false; + } + + // Check if there are multiple regions + const regionCountResult = await targetClient.query(GlobalAuroraPgDatabaseDialect.REGION_COUNT_QUERY); + if (!regionCountResult.rows?.[0]) { + return false; + } + + const awsRegionCount = regionCountResult.rows[0]["count"]; + return awsRegionCount > 1; + } catch { + return false; + } + } + + getDialectUpdateCandidates(): string[] { + return []; + } + + async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + const res = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); + const hosts: HostInfo[] = []; + const rows: any[] = res.rows; + rows.forEach((row) => { + // According to the topology query the result set + // should contain 4 columns: node ID, 1/0 (writer/reader), CPU utilization, node lag in time. + const hostName: string = row["server_id"]; + const isWriter: boolean = row["is_writer"]; + const hostLag: number = row["visibility_lag_in_msec"] ?? 0; // visibility_lag_in_msec is nullable. + const awsRegion: string = row["aws_region"]; // TODO: update this after topologyUtils PR is merged. + const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100, Date.now() /* TODO: update this after topologyUtils PR is merged */); + hosts.push(host); + }); + return hosts; + } + + async getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise { + try { + const result = await targetClient.query(GlobalAuroraPgDatabaseDialect.REGION_BY_INSTANCE_ID_QUERY, [instanceId]); + if (!result.rows?.[0]) { + return null; + } + return result.rows[0]["aws_region"] ?? null; + } catch { + return null; + } + } +} diff --git a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts index ee517c13..92526149 100644 --- a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts +++ b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts @@ -20,7 +20,7 @@ import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { AwsWrapperError, HostInfo, HostRole } from "../../../common/lib"; import { Messages } from "../../../common/lib/utils/messages"; import { logger } from "../../../common/logutils"; -import { TopologyAwareDatabaseDialect } from "../../../common/lib/topology_aware_database_dialect"; +import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { PgDatabaseDialect } from "./pg_database_dialect"; import { ErrorHandler } from "../../../common/lib/error_handler"; From 2c2cc9fd10a49d8da92d13b1c0d0bf368dde0495 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Tue, 3 Mar 2026 14:00:19 -0800 Subject: [PATCH 10/20] chore: topology utils refactoring (#615) --- .../topology_aware_database_dialect.ts | 7 +- .../global_topology_utils.ts | 88 ++++ .../host_list_provider/host_list_provider.ts | 4 +- .../monitoring/cluster_topology_monitor.ts | 35 +- .../monitoring_host_list_provider.ts | 21 +- .../rds_host_list_provider.ts | 123 ++---- .../lib/host_list_provider/topology_utils.ts | 273 ++++++++++++ common/lib/plugin_service.ts | 2 +- common/lib/utils/messages.ts | 10 +- .../dialect/aurora_mysql_database_dialect.ts | 48 ++- .../global_aurora_mysql_database_dialect.ts | 28 +- .../rds_multi_az_mysql_database_dialect.ts | 71 ++-- pg/lib/dialect/aurora_pg_database_dialect.ts | 48 ++- .../global_aurora_pg_database_dialect.ts | 22 +- .../rds_multi_az_pg_database_dialect.ts | 74 ++-- tests/unit/rds_host_list_provider.test.ts | 93 +--- tests/unit/topology_utils.test.ts | 396 ++++++++++++++++++ 17 files changed, 1057 insertions(+), 286 deletions(-) create mode 100644 common/lib/host_list_provider/global_topology_utils.ts create mode 100644 common/lib/host_list_provider/topology_utils.ts create mode 100644 tests/unit/topology_utils.test.ts diff --git a/common/lib/database_dialect/topology_aware_database_dialect.ts b/common/lib/database_dialect/topology_aware_database_dialect.ts index 432a3583..cdd1929c 100644 --- a/common/lib/database_dialect/topology_aware_database_dialect.ts +++ b/common/lib/database_dialect/topology_aware_database_dialect.ts @@ -14,13 +14,12 @@ * limitations under the License. */ -import { HostInfo } from "../host_info"; -import { HostListProvider } from "../host_list_provider/host_list_provider"; import { HostRole } from "../host_role"; import { ClientWrapper } from "../client_wrapper"; +import { TopologyQueryResult } from "../host_list_provider/topology_utils"; export interface TopologyAwareDatabaseDialect { - queryForTopology(client: ClientWrapper, hostListProvider: HostListProvider): Promise; + queryForTopology(client: ClientWrapper): Promise; identifyConnection(targetClient: ClientWrapper): Promise; @@ -28,6 +27,8 @@ export interface TopologyAwareDatabaseDialect { // Returns the host id of the targetClient if it is connected to a writer, null otherwise. getWriterId(targetClient: ClientWrapper): Promise; + + getInstanceId(targetClient: ClientWrapper): Promise<[string, string]>; } export interface GlobalAuroraTopologyDialect extends TopologyAwareDatabaseDialect { diff --git a/common/lib/host_list_provider/global_topology_utils.ts b/common/lib/host_list_provider/global_topology_utils.ts new file mode 100644 index 00000000..7e9131db --- /dev/null +++ b/common/lib/host_list_provider/global_topology_utils.ts @@ -0,0 +1,88 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { TopologyQueryResult, TopologyUtils } from "./topology_utils"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { isDialectTopologyAware } from "../utils/utils"; +import { Messages } from "../utils/messages"; +import { AwsWrapperError } from "../utils/errors"; + +export class GlobalTopologyUtils extends TopologyUtils { + async queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + clusterInstanceTemplate: HostInfo + ): Promise { + throw new AwsWrapperError("Not implemented"); + } + + async queryForTopologyWithRegion( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + instanceTemplateByRegion: Map + ): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + return await dialect + .queryForTopology(targetClient) + .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHostsWithTemplateMap(res, initialHost, instanceTemplateByRegion))); + } + + private createHostsWithTemplateMap( + topologyQueryResults: TopologyQueryResult[], + initialHost: HostInfo, + instanceTemplateByRegion: Map + ): HostInfo[] { + const hostsMap = new Map(); + topologyQueryResults.forEach((row) => { + if (!row.awsRegion) { + throw new AwsWrapperError(Messages.get("GlobalTopologyUtils.missingRegion", row.host)); + } + const clusterInstanceTemplate = instanceTemplateByRegion.get(row.awsRegion); + + if (!clusterInstanceTemplate) { + throw new AwsWrapperError(Messages.get("GlobalTopologyUtils.missingTemplateForRegion", row.awsRegion, row.host)); + } + + const lastUpdateTime = row.lastUpdateTime ?? Date.now(); + + const host = this.createHost( + row.id, + row.host, + row.isWriter, + row.weight, + lastUpdateTime, + initialHost, + clusterInstanceTemplate, + row.endpoint, + row.port + ); + + const existing = hostsMap.get(host.host); + if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { + hostsMap.set(host.host, host); + } + }); + + return Array.from(hostsMap.values()); + } +} diff --git a/common/lib/host_list_provider/host_list_provider.ts b/common/lib/host_list_provider/host_list_provider.ts index 2c2c24ec..a8ea22f4 100644 --- a/common/lib/host_list_provider/host_list_provider.ts +++ b/common/lib/host_list_provider/host_list_provider.ts @@ -40,9 +40,7 @@ export interface HostListProvider { getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise; - identifyConnection(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise; - - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo; + identifyConnection(targetClient: ClientWrapper): Promise; getHostProviderType(): string; diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index 958f4046..2808c55a 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -15,7 +15,6 @@ */ import { HostInfo } from "../../host_info"; -import { CacheMap } from "../../utils/cache_map"; import { PluginService } from "../../plugin_service"; import { HostAvailability } from "../../host_availability/host_availability"; import { logTopology, sleep } from "../../utils/utils"; @@ -27,7 +26,9 @@ import { MonitoringRdsHostListProvider } from "./monitoring_host_list_provider"; import { Messages } from "../../utils/messages"; import { CoreServicesContainer } from "../../utils/core_services_container"; import { Topology } from "../topology"; -import { StorageService, StorageServiceImpl } from "../../utils/storage/storage_service"; +import { StorageService } from "../../utils/storage/storage_service"; +import { TopologyUtils } from "../topology_utils"; +import { RdsUtils } from "../../utils/rds_utils"; export interface ClusterTopologyMonitor { forceRefresh(client: ClientWrapper, timeoutMs: number): Promise; @@ -49,6 +50,9 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { private readonly refreshRateMs: number; private readonly highRefreshRateMs: number; private readonly storageService: StorageService; + private readonly topologyUtils: TopologyUtils; + private readonly rdsUtils: RdsUtils = new RdsUtils(); + private readonly instanceTemplate: HostInfo; private writerHostInfo: HostInfo = null; private isVerifiedWriterConnection: boolean = false; @@ -74,17 +78,21 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { private requestToUpdateTopology: boolean = false; constructor( + topologyUtils: TopologyUtils, clusterId: string, initialHostInfo: HostInfo, props: Map, + instanceTemplate: HostInfo, pluginService: PluginService, hostListProvider: MonitoringRdsHostListProvider, refreshRateMs: number, highRefreshRateMs: number ) { + this.topologyUtils = topologyUtils; this.clusterId = clusterId; this.storageService = CoreServicesContainer.getInstance().getStorageService(); // TODO: store serviceContainer instead this.initialHostInfo = initialHostInfo; + this.instanceTemplate = instanceTemplate; this._pluginService = pluginService; this._hostListProvider = hostListProvider; this.refreshRateMs = refreshRateMs; @@ -212,12 +220,19 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { this.monitoringClient = client; logger.debug(Messages.get("ClusterTopologyMonitor.openedMonitoringConnection", this.initialHostInfo.host)); try { - const writerId = await this.getWriterHostIdIfConnected(this.monitoringClient, this.initialHostInfo.hostId); - if (writerId) { + if (await this.topologyUtils.isWriterInstance(this.monitoringClient)) { this.isVerifiedWriterConnection = true; - this.writerHostInfo = this.initialHostInfo; - logger.info(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.initialHostInfo.host)); - writerVerifiedByThisTask = true; + + if (this.rdsUtils.isRdsInstance(this.initialHostInfo.host)) { + this.writerHostInfo = this.initialHostInfo; + logger.info(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.writerHostInfo.host)); + writerVerifiedByThisTask = true; + } else { + const pair: [string, string] = await this.topologyUtils.getInstanceId(this.monitoringClient); + const instanceTemplate: HostInfo = await this.getInstanceTemplate(pair[1], this.monitoringClient); + this.writerHostInfo = this.topologyUtils.createHost(pair[0], pair[1], true, 0, Date.now(), this.initialHostInfo, instanceTemplate); + logger.debug(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.writerHostInfo.host)); + } } } catch (error) { // Do nothing. @@ -245,6 +260,10 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { return hosts; } + protected getInstanceTemplate(hostId: string, targetClient: ClientWrapper): Promise { + return Promise.resolve(this.instanceTemplate); + } + updateTopologyCache(hosts: HostInfo[]): void { this.storageService.set(this.clusterId, new Topology(hosts)); this.requestToUpdateTopology = false; @@ -513,7 +532,7 @@ export class HostMonitor { let hosts: HostInfo[]; try { hosts = await this.monitor.hostListProvider.sqlQueryForTopology(client); - if (hosts === null || hosts.length === 0) { + if (hosts === null) { return; } this.monitor.hostMonitorsLatestTopology = hosts; diff --git a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts index 335baa6c..c3f9b87d 100644 --- a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts +++ b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts @@ -27,6 +27,7 @@ import { BlockingHostListProvider } from "../host_list_provider"; import { logger } from "../../../logutils"; import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; import { isDialectTopologyAware } from "../../utils/utils"; +import { TopologyUtils } from "../topology_utils"; export class MonitoringRdsHostListProvider extends RdsHostListProvider implements BlockingHostListProvider { static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(60_000_000_000); // 1 minute. @@ -48,8 +49,14 @@ export class MonitoringRdsHostListProvider extends RdsHostListProvider implement private readonly pluginService: PluginService; - constructor(properties: Map, originalUrl: string, hostListProviderService: HostListProviderService, pluginService: PluginService) { - super(properties, originalUrl, hostListProviderService); + constructor( + properties: Map, + originalUrl: string, + topologyUtils: TopologyUtils, + hostListProviderService: HostListProviderService, + pluginService: PluginService + ) { + super(properties, originalUrl, topologyUtils, hostListProviderService); this.pluginService = pluginService; } @@ -58,7 +65,7 @@ export class MonitoringRdsHostListProvider extends RdsHostListProvider implement await MonitoringRdsHostListProvider.monitors.clear(); } - async queryForTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { const monitor: ClusterTopologyMonitor = this.initMonitor(); try { @@ -70,11 +77,7 @@ export class MonitoringRdsHostListProvider extends RdsHostListProvider implement } async sqlQueryForTopology(targetClient: ClientWrapper): Promise { - const dialect: DatabaseDialect = this.hostListProviderService.getDialect(); - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } - return await dialect.queryForTopology(targetClient, this).then((res: any) => this.processQueryResults(res)); + return await this.topologyUtils.queryForTopology(targetClient, this.pluginService.getDialect(), this.initialHost, this.clusterInstanceTemplate); } async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { @@ -88,9 +91,11 @@ export class MonitoringRdsHostListProvider extends RdsHostListProvider implement this.clusterId, () => new ClusterTopologyMonitorImpl( + this.topologyUtils, this.clusterId, this.initialHost, this.properties, + this.clusterInstanceTemplate, this.pluginService, this, WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties), diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index 30c80a3f..aae69e4f 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -25,20 +25,19 @@ import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; import { WrapperProperties } from "../wrapper_property"; import { logger } from "../../logutils"; -import { HostAvailability } from "../host_availability/host_availability"; -import { CacheMap } from "../utils/cache_map"; import { isDialectTopologyAware, logTopology } from "../utils/utils"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; import { CoreServicesContainer } from "../utils/core_services_container"; import { StorageService } from "../utils/storage/storage_service"; import { Topology } from "./topology"; -import { ExpirationCache } from "../utils/storage/expiration_cache"; +import { TopologyUtils } from "./topology_utils"; export class RdsHostListProvider implements DynamicHostListProvider { private readonly originalUrl: string; private readonly rdsHelper: RdsUtils; private readonly storageService: StorageService; + protected readonly topologyUtils: TopologyUtils; protected readonly properties: Map; private rdsUrlType: RdsUrlType; private initialHostList: HostInfo[]; @@ -52,8 +51,9 @@ export class RdsHostListProvider implements DynamicHostListProvider { public isInitialized: boolean = false; public clusterInstanceTemplate?: HostInfo; - constructor(properties: Map, originalUrl: string, hostListProviderService: HostListProviderService) { + constructor(properties: Map, originalUrl: string, topologyUtils: TopologyUtils, hostListProviderService: HostListProviderService) { this.rdsHelper = new RdsUtils(); + this.topologyUtils = topologyUtils; this.hostListProviderService = hostListProviderService; this.connectionUrlParser = hostListProviderService.getConnectionUrlParser(); this.originalUrl = originalUrl; @@ -112,15 +112,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { } async getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } - - if (client) { - return await dialect.getHostRole(client); - } else { - throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); - } + return this.topologyUtils.getHostRole(client); } async getWriterId(client: ClientWrapper): Promise { @@ -131,21 +123,42 @@ export class RdsHostListProvider implements DynamicHostListProvider { if (client) { return await dialect.getWriterId(client); - } else { - throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); } + + throw new AwsWrapperError(Messages.get("AwsClient.targetClientNotDefined")); } - async identifyConnection(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + async identifyConnection(targetClient: ClientWrapper): Promise { + const instanceIds: [string, string] = await this.topologyUtils.getInstanceId(targetClient); + if (instanceIds.some((id) => !id)) { + return null; + } + + let topology = await this.refresh(targetClient); + let isForcedRefresh = false; + + if (!topology) { + topology = await this.forceRefresh(); + isForcedRefresh = true; + } + + if (!topology) { + return null; } - const instanceName = await dialect.identifyConnection(targetClient); - return this.refresh(targetClient).then((topology) => { - const matches = topology.filter((host) => host.hostId === instanceName); - return matches.length === 0 ? null : matches[0]; - }); + const instanceName = instanceIds[1]; + let matches = topology.filter((host) => host.hostId === instanceName); + const foundHost = matches.length === 0 ? null : matches[0]; + + if (!foundHost && !isForcedRefresh) { + topology = await this.forceRefresh(); + if (!topology) { + return null; + } + } + + matches = topology.filter((host) => host.hostId === instanceName); + return matches.length === 0 ? null : matches[0]; } async refresh(): Promise; @@ -164,7 +177,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.init(); if (!this.clusterId) { - throw new AwsWrapperError("no cluster id"); + throw new AwsWrapperError(Messages.get("RdsHostListProvider.noClusterId")); } const cachedHosts: HostInfo[] | null = this.getStoredTopology(); @@ -178,7 +191,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { return new FetchTopologyResult(false, this.initialHostList); } - const hosts = await this.queryForTopology(targetClient, this.hostListProviderService.getDialect()); + const hosts = await this.getCurrentTopology(targetClient, this.hostListProviderService.getDialect()); if (hosts && hosts.length > 0) { this.storageService.set(this.clusterId, new Topology(hosts)); return new FetchTopologyResult(false, hosts); @@ -192,64 +205,8 @@ export class RdsHostListProvider implements DynamicHostListProvider { } } - async queryForTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } - - return await dialect.queryForTopology(targetClient, this).then((res: any) => this.processQueryResults(res)); - } - - protected async processQueryResults(result: HostInfo[]): Promise { - const hostMap: Map = new Map(); - - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - result.forEach((host) => { - hostMap.set(host.host, host); - }); - - hostMap.forEach((host) => { - if (host.role !== HostRole.WRITER) { - hosts.push(host); - } else { - writers.push(host); - } - }); - - const writerCount: number = writers.length; - if (writerCount === 0) { - hosts = []; - } else if (writerCount === 1) { - hosts.push(writers[0]); - } else { - const sortedWriters: HostInfo[] = writers.sort((a, b) => { - return b.lastUpdateTime - a.lastUpdateTime; // reverse order - }); - - hosts.push(sortedWriters[0]); - } - - return hosts; - } - - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo { - host = !host ? "?" : host; - const endpoint: string | null = this.getHostEndpoint(host); - if (!port) { - port = this.clusterInstanceTemplate?.isPortSpecified() ? this.clusterInstanceTemplate?.port : this.initialHost?.port; - } - - return this.hostListProviderService - .getHostInfoBuilder() - .withHost(endpoint ?? "") - .withPort(port ?? -1) - .withRole(isWriter ? HostRole.WRITER : HostRole.READER) - .withAvailability(HostAvailability.AVAILABLE) - .withWeight(weight) - .withLastUpdateTime(lastUpdateTime) - .withHostId(host) - .build(); + async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.clusterInstanceTemplate); } private getHostEndpoint(hostName: string): string | null { diff --git a/common/lib/host_list_provider/topology_utils.ts b/common/lib/host_list_provider/topology_utils.ts new file mode 100644 index 00000000..2542c63f --- /dev/null +++ b/common/lib/host_list_provider/topology_utils.ts @@ -0,0 +1,273 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { isDialectTopologyAware } from "../utils/utils"; +import { Messages } from "../utils/messages"; +import { HostRole } from "../host_role"; +import { HostAvailability } from "../host_availability/host_availability"; +import { HostInfoBuilder } from "../host_info_builder"; +import { AwsWrapperError } from "../utils/errors"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; + +/** + * Options for creating a TopologyQueryResult instance. + */ +export interface TopologyQueryResultOptions { + host: string; + isWriter: boolean; + weight: number; + lastUpdateTime?: number; + port?: number; + id?: string; + endpoint?: string; + awsRegion?: string; +} + +/** + * Represents the result of a topology query for a single database instance. + * Contains information about the instance's role, weight, and connection details. + */ +export class TopologyQueryResult { + host: string; + isWriter: boolean; + weight: number; + lastUpdateTime?: number; + port?: number; + id?: string; + endpoint?: string; + awsRegion?: string; + + constructor(options: TopologyQueryResultOptions) { + this.host = options.host; + this.isWriter = options.isWriter; + this.weight = options.weight; + this.lastUpdateTime = options.lastUpdateTime; + this.port = options.port; + this.id = options.id; + this.endpoint = options.endpoint; + this.awsRegion = options.awsRegion; + } +} + +/** + * A class defining utility methods that can be used to retrieve and process a variety of database topology + * information. This class can be overridden to define logic specific to various database engine deployments + * (e.g. Aurora, Multi-AZ, Global Aurora etc.). + */ +export class TopologyUtils { + protected readonly dialect: TopologyAwareDatabaseDialect; + protected readonly hostInfoBuilder: HostInfoBuilder; + + constructor(dialect: TopologyAwareDatabaseDialect, hostInfoBuilder: HostInfoBuilder) { + this.dialect = dialect; + this.hostInfoBuilder = hostInfoBuilder; + } + + /** + * Query the database for information for each instance in the database topology. + * + * @param targetClient the client wrapper to use to query the database. + * @param dialect the database dialect to use for the topology query. + * @param clusterInstanceTemplate the template {@link HostInfo} to use when constructing new {@link HostInfo} objects from + * the data returned by the topology query. + * @returns a list of {@link HostInfo} objects representing the results of the topology query. + * @throws TypeError if the dialect is not topology-aware. + */ + async queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + clusterInstanceTemplate: HostInfo + ): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + return await dialect + .queryForTopology(targetClient) + .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHosts(res, initialHost, clusterInstanceTemplate))); + } + + public createHost( + instanceId: string | undefined, + instanceName: string | undefined, + isWriter: boolean, + weight: number, + lastUpdateTime: number, + initialHost: HostInfo, + instanceTemplate: HostInfo, + endpoint?: string, + port?: number + ): HostInfo { + const hostname = !instanceName ? "?" : instanceName; + const finalInstanceId = instanceId ?? hostname; + + if (!finalInstanceId) { + throw new AwsWrapperError(Messages.get("TopologyUtils.instanceIdRequired")); + } + + const finalEndpoint = endpoint ?? this.getHostEndpoint(hostname, instanceTemplate) ?? ""; + + const finalPort = port ?? (instanceTemplate?.isPortSpecified() ? instanceTemplate?.port : initialHost?.port); + + const host: HostInfo = this.hostInfoBuilder + .withHost(finalEndpoint) + .withPort(finalPort ?? HostInfo.NO_PORT) + .withRole(isWriter ? HostRole.WRITER : HostRole.READER) + .withAvailability(HostAvailability.AVAILABLE) + .withWeight(weight) + .withLastUpdateTime(lastUpdateTime) + .withHostId(finalInstanceId) + .build(); + host.addAlias(finalEndpoint); + return host; + } + + /** + * Creates {@link HostInfo} objects from the given topology query results. + * + * @param topologyQueryResults the result set returned by the topology query describing the cluster topology + * @param initialHost the {@link HostInfo} describing the initial connection. + * @param clusterInstanceTemplate the template used to construct the new {@link HostInfo} objects. + * @returns a list of {@link HostInfo} objects representing the topology. + */ + public createHosts(topologyQueryResults: TopologyQueryResult[], initialHost: HostInfo, clusterInstanceTemplate: HostInfo): HostInfo[] { + const hostsMap = new Map(); + topologyQueryResults.forEach((row) => { + const lastUpdateTime = row.lastUpdateTime ?? Date.now(); + + const host = this.createHost( + row.id, + row.host, + row.isWriter, + row.weight, + lastUpdateTime, + initialHost, + clusterInstanceTemplate, + row.endpoint, + row.port + ); + + const existing = hostsMap.get(host.host); + if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { + hostsMap.set(host.host, host); + } + }); + + return Array.from(hostsMap.values()); + } + + /** + * Gets the host endpoint by replacing the placeholder in the cluster instance template. + * + * @param hostName the host name to use in the endpoint. + * @param clusterInstanceTemplate the template containing the endpoint pattern. + * @returns the constructed endpoint, or null if the template is invalid. + */ + protected getHostEndpoint(hostName: string, clusterInstanceTemplate: HostInfo): string | null { + if (!clusterInstanceTemplate || !clusterInstanceTemplate.host) { + return null; + } + const host = clusterInstanceTemplate.host; + return host.replace("?", hostName); + } + + /** + * Verifies that the topology contains exactly one writer instance. + * If multiple writers are found, selects the most recently updated one. + * + * @param allHosts the list of all hosts from the topology query. + * @returns the verified list of hosts with exactly one writer, or null if no writer is found. + */ + protected async verifyWriter(allHosts: HostInfo[]): Promise { + if (allHosts === null || allHosts.length === 0) { + return null; + } + + const hosts: HostInfo[] = []; + const writers: HostInfo[] = []; + + for (const host of allHosts) { + if (host.role === HostRole.WRITER) { + writers.push(host); + } else { + hosts.push(host); + } + } + + const writerCount = writers.length; + if (writerCount === 0) { + return null; + } else if (writerCount === 1) { + hosts.push(writers[0]); + } else { + // Assume the latest updated writer instance is the current writer. Other potential writers will be ignored. + const sortedWriters: HostInfo[] = writers.sort((a, b) => { + return b.lastUpdateTime - a.lastUpdateTime; // reverse order + }); + hosts.push(sortedWriters[0]); + } + + return hosts; + } + + /** + * Identifies instances across different database types using instanceId and instanceName values. + * + * Database types handle these identifiers differently: + * - Aurora: Uses the instance name as both instanceId and instanceName + * Example: "test-instance-1" for both values + * - RDS Cluster: Uses distinct values for instanceId and instanceName + * Example: + * instanceId: "db-WQFQKBTL2LQUPIEFIFBGENS4ZQ" + * instanceName: "test-multiaz-instance-1" + * + * @param client the client wrapper to query. + * @returns a tuple of [instanceId, instanceName], or null if the query fails. + */ + getInstanceId(client: ClientWrapper): Promise<[string, string]> { + return this.dialect.getInstanceId(client); + } + + /** + * Evaluate whether the given connection is to a writer instance. + * + * @param client the client wrapper to evaluate. + * @returns true if the connection is to a writer instance, false otherwise. + * @throws Error if an exception occurs when querying the database or processing the database response. + */ + async isWriterInstance(client: ClientWrapper): Promise { + return (await this.dialect.getWriterId(client)) != null; + } + + /** + * Evaluate the database role of the given connection, either {@link HostRole.WRITER} or {@link HostRole.READER}. + * + * @param client the client wrapper to evaluate. + * @returns the database role of the given connection. + * @throws Error if an exception occurs when querying the database or processing the database response. + */ + async getHostRole(client: ClientWrapper): Promise { + try { + return await this.dialect.getHostRole(client); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("TopologyUtils.errorGettingHostRole")); + } + } +} diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index e2aae506..5421b8bb 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -513,7 +513,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (!provider) { return Promise.reject(); } - return provider.identifyConnection(targetClient, this.dialect); + return provider.identifyConnection(targetClient); } connect(hostInfo: HostInfo, props: Map): Promise; diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index b0469231..abc4db8d 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -56,7 +56,7 @@ const MESSAGES: Record = { "ClusterAwareReaderFailoverHandler.selectedTaskChosen": "Selected task has already been chosen. Abort client for host: %s", "Utils.topology": "Topology: %s", "RdsHostListProvider.incorrectDialect": "Dialect needs to be a topology aware dialect.", - "RdsHostListProvider.suggestedClusterId": "ClusterId '%s' is suggested for url '%s'.", + "RdsHostListProvider.noClusterId": "No clusterId found. Please ensure clusterId parameter is set to a non-empty string.", "ConnectionStringHostListProvider.parsedListEmpty": "Can't parse connection string: '%s'.", "ConnectionStringHostListProvider.errorIdentifyConnection": "An error occurred while obtaining the connection's host ID.", "ExecuteTimePlugin.executeTime": "Executed method '%s' in %s milliseconds.", @@ -375,8 +375,12 @@ const MESSAGES: Record = { "Bgd.correspondingHostFoundContinueWithConnect": "A corresponding host for '%s' is found. Continue with connect call. The call was suspended for %s ms.", "Bgd.completedContinueWithConnect": "Blue/Green Deployment status is completed. Continue with 'connect' call. The call was suspended for %s ms.", - "StorageService.itemClassNotRegistered": "StorageServiceImpl: Item class not registered: %s", - "StorageService.unexpectedValueMismatch": "StorageServiceImpl: Unexpected value mismatch for %s: %s" + "StorageService.itemClassNotRegistered": "[StorageService] Item class not registered: %s", + "StorageService.unexpectedValueMismatch": "[StorageService] Unexpected value mismatch for %s: %s", + "TopologyUtils.instanceIdRequired": "InstanceId must not be en empty string.", + "TopologyUtils.errorGettingHostRole": "An error occurred while trying to get the host role.", + "GlobalTopologyUtils.missingRegion": "Host '%s' is missing region information in the topology query result.", + "GlobalTopologyUtils.missingTemplateForRegion": "No cluster instance template found for region '%s' when processing host '%s'." }; export class Messages { diff --git a/mysql/lib/dialect/aurora_mysql_database_dialect.ts b/mysql/lib/dialect/aurora_mysql_database_dialect.ts index 08976c43..3ebfa224 100644 --- a/mysql/lib/dialect/aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/aurora_mysql_database_dialect.ts @@ -18,15 +18,17 @@ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; -import { HostInfo } from "../../../common/lib/host_info"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { HostRole } from "../../../common/lib/host_role"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; +import { + MonitoringRdsHostListProvider +} from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; +import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect, BlueGreenDialect { private static readonly TOPOLOGY_QUERY: string = @@ -41,6 +43,7 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements "SELECT server_id " + "FROM information_schema.replica_host_status " + "WHERE SESSION_ID = 'MASTER_SESSION_ID' AND SERVER_ID = @@aurora_server_id"; + protected static readonly INSTANCE_ID_QUERY: string = "SELECT @@aurora_server_id as instance_id, @@aurora_server_id as instance_name"; private static readonly AURORA_VERSION_QUERY = "SHOW VARIABLES LIKE 'aurora_version'"; private static readonly BG_STATUS_QUERY: string = "SELECT * FROM mysql.rds_topology"; @@ -48,15 +51,22 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements "SELECT 1 AS tmp FROM information_schema.tables WHERE table_schema = 'mysql' AND table_name = 'rds_topology'"; getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { + const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); + return new MonitoringRdsHostListProvider( + props, + originalUrl, + topologyUtils, + hostListProviderService, + (hostListProviderService) + ); } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(AuroraMySQLDatabaseDialect.TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + const results: TopologyQueryResult[] = []; const rows: any[] = res[0]; rows.forEach((row) => { // According to the topology query the result set @@ -66,10 +76,15 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements const cpuUtilization: number = row["cpu"]; const hostLag: number = row["lag"]; const lastUpdateTime: number = row["last_update_timestamp"] ? Date.parse(row["last_update_timestamp"]) : Date.now(); - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100 + Math.round(cpuUtilization), lastUpdateTime); - hosts.push(host); + const result: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100 + Math.round(cpuUtilization), + lastUpdateTime: lastUpdateTime + }); + results.push(result); }); - return hosts; + return results; } async identifyConnection(targetClient: ClientWrapper): Promise { @@ -96,6 +111,21 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + const res = await targetClient.query(AuroraMySQLDatabaseDialect.INSTANCE_ID_QUERY); + try { + const instance_id: string = res[0][0]["instance_id"]; + const instance_name: string = res[0][0]["instance_name"]; + return [instance_id, instance_name]; + } catch (e: any) { + if (e.message.includes("Cannot read properties of undefined")) { + // Query returned no result, targetClient is not connected to a writer. + return ["", ""]; + } + throw e; + } + } + async isDialect(targetClient: ClientWrapper): Promise { return targetClient .query(AuroraMySQLDatabaseDialect.AURORA_VERSION_QUERY) diff --git a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts index d05d41db..1b51b4e1 100644 --- a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import { AuroraMySQLDatabaseDialect } from "./aurora_mysql_database_dialect"; import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { HostInfo } from "../../../common/lib"; -import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect implements GlobalAuroraTopologyDialect { private static readonly GLOBAL_STATUS_TABLE_EXISTS_QUERY = @@ -68,21 +68,27 @@ export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect return []; } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + // TODO: implement GetHostListProvider once GDBHostListProvider is implemented + + async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + const results: TopologyQueryResult[] = []; const rows: any[] = res[0]; rows.forEach((row) => { - // According to the topology query the result set - // should contain 4 columns: node ID, 1/0 (writer/reader), CPU utilization, node lag in time. const hostName: string = row["server_id"]; const isWriter: boolean = row["is_writer"]; const hostLag: number = row["visibility_lag_in_msec"] ?? 0; // visibility_lag_in_msec is nullable. - const awsRegion: string = row["aws_region"]; // TODO: update this after topologyUtils PR is merged. - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100, Date.now() /* TODO: update this after topologyUtils PR is merged */); - hosts.push(host); + const awsRegion: string = row["aws_region"]; + + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100, + awsRegion: awsRegion + }); + results.push(host); }); - return hosts; + return results; } async getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise { @@ -91,7 +97,7 @@ export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect if (!rows?.[0]) { return null; } - return rows[0]["AWS_REGION"] ?? null; + return rows[0]["aws_region"] ?? null; } catch { return null; } diff --git a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts index 52660758..f070e8a1 100644 --- a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts +++ b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts @@ -18,10 +18,8 @@ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { HostInfo } from "../../../common/lib/host_info"; import { HostRole } from "../../../common/lib/host_role"; import { Messages } from "../../../common/lib/utils/messages"; -import { logger } from "../../../common/logutils"; import { AwsWrapperError } from "../../../common/lib/utils/errors"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; @@ -29,6 +27,7 @@ import { FailoverRestriction } from "../../../common/lib/plugins/failover/failov import { WrapperProperties } from "../../../common/lib/wrapper_property"; import { PluginService } from "../../../common/lib/plugin_service"; import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; +import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect { private static readonly TOPOLOGY_QUERY: string = "SELECT id, endpoint, port FROM mysql.rds_topology"; @@ -41,6 +40,8 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect private static readonly HOST_ID_QUERY_COLUMN_NAME: string = "host"; private static readonly IS_READER_QUERY: string = "SELECT @@read_only AS is_reader"; private static readonly IS_READER_QUERY_COLUMN_NAME: string = "is_reader"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT id as instance_id, SUBSTRING_INDEX(endpoint, '.', 1) as instance_name FROM mysql.rds_topology WHERE id = @@server_id"; async isDialect(targetClient: ClientWrapper): Promise { let res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.TOPOLOGY_TABLE_EXIST_QUERY).catch(() => false); @@ -71,13 +72,20 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect } getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { + const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); + return new MonitoringRdsHostListProvider( + props, + originalUrl, + topologyUtils, + hostListProviderService, + (hostListProviderService) + ); } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { try { let writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -90,7 +98,7 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect const res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.TOPOLOGY_QUERY); const rows: any[] = res[0]; - return this.processTopologyQueryResults(hostListProvider, writerHostId, rows); + return this.processTopologyQueryResults(writerHostId, rows); } catch (error: any) { throw new AwsWrapperError(Messages.get("RdsMultiAZMySQLDatabaseDialect.invalidQuery", error.message)); } @@ -105,8 +113,8 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect return ""; } - private async processTopologyQueryResults(hostListProvider: HostListProvider, writerHostId: string, rows: any[]): Promise { - const hostMap: Map = new Map(); + private async processTopologyQueryResults(writerHostId: string, rows: any[]): Promise { + const hosts: TopologyQueryResult[] = []; rows.forEach((row) => { // According to the topology query the result set // should contain 3 columns: endpoint, id, and port @@ -114,39 +122,23 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect const id: string = row["id"]; const port: number = row["port"]; const isWriter: boolean = id === writerHostId; - - const host: HostInfo = hostListProvider.createHost(endpoint.substring(0, endpoint.indexOf(".")), isWriter, 0, Date.now(), port); - host.hostId = id; - host.addAlias(endpoint); - hostMap.set(host.host, host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: endpoint.substring(0, endpoint.indexOf(".")), + isWriter: isWriter, + weight: 0, + lastUpdateTime: Date.now(), + port: port, + id: id + }); + hosts.push(host); }); - - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - - for (const [key, info] of hostMap.entries()) { - if (info.role !== HostRole.WRITER) { - hosts.push(info); - } else { - writers.push(info); - } - } - - const writerCount: number = writers.length; - if (writerCount === 0) { - logger.error(Messages.get("RdsMultiAzDatabaseDialect.invalidTopology", this.dialectName)); - hosts = []; - } else { - hosts.push(writers[0]); - } - return hosts; } async getHostRole(client: ClientWrapper): Promise { return (await this.executeTopologyRelatedQuery( client, - RdsMultiAZClusterMySQLDatabaseDialect.IS_READER_QUERY, + RdsMultiAZClusterMySQLDatabaseDialect.INSTANCE_ID_QUERY, RdsMultiAZClusterMySQLDatabaseDialect.IS_READER_QUERY_COLUMN_NAME )) == "0" ? HostRole.WRITER @@ -171,6 +163,17 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + try { + const res = await targetClient.query(RdsMultiAZClusterMySQLDatabaseDialect.INSTANCE_ID_QUERY); + const instance_id = res[0][0]["instance_id"]; + const instance_name = res[0][0]["instance_name"]; + return [instance_id, instance_name]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("RdsMultiAZMySQLDatabaseDialect.invalidQuery", error.message)); + } + } + async identifyConnection(client: ClientWrapper): Promise { return await this.executeTopologyRelatedQuery( client, diff --git a/pg/lib/dialect/aurora_pg_database_dialect.ts b/pg/lib/dialect/aurora_pg_database_dialect.ts index 250b39ee..c717c2d6 100644 --- a/pg/lib/dialect/aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/aurora_pg_database_dialect.ts @@ -19,7 +19,7 @@ import { HostListProviderService } from "../../../common/lib/host_list_provider_ import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; -import { HostInfo, HostRole } from "../../../common/lib"; +import { HostRole } from "../../../common/lib"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; import { LimitlessDatabaseDialect } from "../../../common/lib/database_dialect/limitless_database_dialect"; @@ -27,6 +27,7 @@ import { WrapperProperties } from "../../../common/lib/wrapper_property"; import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; +import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect, LimitlessDatabaseDialect, BlueGreenDialect { private static readonly VERSION = process.env.npm_package_version; @@ -40,6 +41,8 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo private static readonly EXTENSIONS_SQL: string = "SELECT (setting LIKE '%aurora_stat_utils%') AS aurora_stat_utils FROM pg_catalog.pg_settings WHERE name OPERATOR(pg_catalog.=) 'rds.extensions'"; private static readonly HOST_ID_QUERY: string = "SELECT pg_catalog.aurora_db_instance_identifier() as host"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT pg_catalog.aurora_db_instance_identifier() as instance_id, pg_catalog.aurora_db_instance_identifier() as instance_name"; private static readonly IS_READER_QUERY: string = "SELECT pg_catalog.pg_is_in_recovery() as is_reader"; private static readonly IS_WRITER_QUERY: string = "SELECT server_id " + @@ -51,15 +54,22 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo private static readonly TOPOLOGY_TABLE_EXIST_QUERY: string = "SELECT pg_catalog.'get_blue_green_fast_switchover_metadata'::regproc"; getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { + const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); + return new MonitoringRdsHostListProvider( + props, + originalUrl, + topologyUtils, + hostListProviderService, + (hostListProviderService) + ); } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(AuroraPgDatabaseDialect.TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + const results: TopologyQueryResult[] = []; const rows: any[] = res.rows; rows.forEach((row) => { // According to the topology query the result set @@ -69,10 +79,15 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo const cpuUtilization: number = row["cpu"]; const hostLag: number = row["lag"]; const lastUpdateTime: number = row["last_update_timestamp"] ? Date.parse(row["last_update_timestamp"]) : Date.now(); - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100 + Math.round(cpuUtilization), lastUpdateTime); - hosts.push(host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter, + weight: Math.round(hostLag) * 100 + Math.round(cpuUtilization), + lastUpdateTime + }); + results.push(host); }); - return hosts; + return results; } async identifyConnection(targetClient: ClientWrapper): Promise { @@ -90,7 +105,7 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo try { const writerId: string = res.rows[0]["server_id"]; return writerId ? writerId : null; - } catch (e) { + } catch (e: any) { if (e.message.includes("Cannot read properties of undefined")) { // Query returned no result, targetClient is not connected to a writer. return null; @@ -99,6 +114,21 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + const res = await targetClient.query(AuroraPgDatabaseDialect.INSTANCE_ID_QUERY); + try { + const instance_id: string = res.rows[0]["instance_id"]; + const instance_name: string = res.rows[0]["instance_name"]; + return [instance_id, instance_name]; + } catch (e: any) { + if (e.message.includes("Cannot read properties of undefined")) { + // Query returned no result, targetClient is not connected to a writer. + return ["", ""]; + } + throw e; + } + } + async isDialect(targetClient: ClientWrapper): Promise { if (!(await super.isDialect(targetClient))) { return false; diff --git a/pg/lib/dialect/global_aurora_pg_database_dialect.ts b/pg/lib/dialect/global_aurora_pg_database_dialect.ts index 7061b09f..c2a86454 100644 --- a/pg/lib/dialect/global_aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/global_aurora_pg_database_dialect.ts @@ -17,8 +17,7 @@ import { AuroraPgDatabaseDialect } from "./aurora_pg_database_dialect"; import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { HostInfo } from "../../../common/lib"; -import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect implements GlobalAuroraTopologyDialect { private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_status'::regproc"; @@ -78,18 +77,23 @@ export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect imple return []; } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + // TODO: implement GetHostListProvider once GDBHostListProvider is implemented + + async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); - const hosts: HostInfo[] = []; + const hosts: TopologyQueryResult[] = []; const rows: any[] = res.rows; rows.forEach((row) => { - // According to the topology query the result set - // should contain 4 columns: node ID, 1/0 (writer/reader), CPU utilization, node lag in time. const hostName: string = row["server_id"]; const isWriter: boolean = row["is_writer"]; - const hostLag: number = row["visibility_lag_in_msec"] ?? 0; // visibility_lag_in_msec is nullable. - const awsRegion: string = row["aws_region"]; // TODO: update this after topologyUtils PR is merged. - const host: HostInfo = hostListProvider.createHost(hostName, isWriter, Math.round(hostLag) * 100, Date.now() /* TODO: update this after topologyUtils PR is merged */); + const hostLag: number = row["visibility_lag_in_msec"] ?? 0; + const awsRegion: string = row["aws_region"]; + const host: TopologyQueryResult = new TopologyQueryResult({ + host: hostName, + isWriter: isWriter, + weight: Math.round(hostLag) * 100, + awsRegion: awsRegion + }); hosts.push(host); }); return hosts; diff --git a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts index 92526149..d3e47f94 100644 --- a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts +++ b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts @@ -17,9 +17,8 @@ import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; -import { AwsWrapperError, HostInfo, HostRole } from "../../../common/lib"; +import { AwsWrapperError, HostRole } from "../../../common/lib"; import { Messages } from "../../../common/lib/utils/messages"; -import { logger } from "../../../common/logutils"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { PgDatabaseDialect } from "./pg_database_dialect"; @@ -28,6 +27,7 @@ import { MultiAzPgErrorHandler } from "../multi_az_pg_error_handler"; import { WrapperProperties } from "../../../common/lib/wrapper_property"; import { PluginService } from "../../../common/lib/plugin_service"; import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; +import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect { constructor() { @@ -44,6 +44,10 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem private static readonly HOST_ID_QUERY_COLUMN_NAME: string = "dbi_resource_id"; private static readonly IS_READER_QUERY: string = "SELECT pg_catalog.pg_is_in_recovery() AS is_reader"; private static readonly IS_READER_QUERY_COLUMN_NAME: string = "is_reader"; + protected static readonly INSTANCE_ID_QUERY: string = + "SELECT id as instance_id, SUBSTRING(endpoint FROM 0 FOR POSITION('.' IN endpoint)) as instance_name " + + "FROM rds_tools.show_topology() " + + "WHERE id OPERATOR(pg_catalog.=) rds_tools.dbi_resource_id()"; async isDialect(targetClient: ClientWrapper): Promise { const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.WRITER_HOST_FUNC_EXIST_QUERY).catch(() => false); @@ -61,13 +65,20 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem } getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { + const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider(props, originalUrl, hostListProviderService, (hostListProviderService)); + return new MonitoringRdsHostListProvider( + props, + originalUrl, + topologyUtils, + hostListProviderService, + (hostListProviderService) + ); } - return new RdsHostListProvider(props, originalUrl, hostListProviderService); + return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); } - async queryForTopology(targetClient: ClientWrapper, hostListProvider: HostListProvider): Promise { + async queryForTopology(targetClient: ClientWrapper): Promise { try { let writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -80,7 +91,7 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.TOPOLOGY_QUERY); const rows: any[] = res.rows; - return this.processTopologyQueryResults(hostListProvider, writerHostId, rows); + return this.processTopologyQueryResults(writerHostId, rows); } catch (error: any) { throw new AwsWrapperError(Messages.get("RdsMultiAZPgDatabaseDialect.invalidQuery", error.message)); } @@ -95,8 +106,8 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem return ""; } - private async processTopologyQueryResults(hostListProvider: HostListProvider, writerHostId: string, rows: any[]): Promise { - const hostMap: Map = new Map(); + private async processTopologyQueryResults(writerHostId: string, rows: any[]): Promise { + const hosts: TopologyQueryResult[] = []; rows.forEach((row) => { // According to the topology query the result set // should contain 3 columns: endpoint, id, and port @@ -104,32 +115,16 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem const id: string = row["id"]; const port: number = row["port"]; const isWriter: boolean = id === writerHostId; - - const host: HostInfo = hostListProvider.createHost(endpoint.substring(0, endpoint.indexOf(".")), isWriter, 0, Date.now(), port); - host.hostId = id; - host.addAlias(endpoint); - hostMap.set(host.host, host); + const host: TopologyQueryResult = new TopologyQueryResult({ + host: endpoint.substring(0, endpoint.indexOf(".")), + isWriter: isWriter, + weight: 0, + lastUpdateTime: Date.now(), + port: port, + id: id + }); + hosts.push(host); }); - - let hosts: HostInfo[] = []; - const writers: HostInfo[] = []; - - for (const [key, info] of hostMap.entries()) { - if (info.role !== HostRole.WRITER) { - hosts.push(info); - } else { - writers.push(info); - } - } - - const writerCount: number = writers.length; - if (writerCount === 0) { - logger.error(Messages.get("RdsMultiAzDatabaseDialect.invalidTopology", this.dialectName)); - hosts = []; - } else { - hosts.push(writers[0]); - } - return hosts; } @@ -143,7 +138,7 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem : HostRole.READER; } - async getWriterId(targetClient: ClientWrapper): Promise { + async getWriterId(targetClient: ClientWrapper): Promise { try { const writerHostId: string = await this.executeTopologyRelatedQuery( targetClient, @@ -158,6 +153,17 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem } } + async getInstanceId(targetClient: ClientWrapper): Promise<[string, string]> { + try { + const res = await targetClient.query(RdsMultiAZClusterPgDatabaseDialect.INSTANCE_ID_QUERY); + const instance_id = res.rows[0]["instance_id"]; + const instance_name = res.rows[0]["instance_name"]; + return [instance_id, instance_name]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("RdsMultiAZPgDatabaseDialect.invalidQuery", error.message)); + } + } + getErrorHandler(): ErrorHandler { return new MultiAzPgErrorHandler(); } diff --git a/tests/unit/rds_host_list_provider.test.ts b/tests/unit/rds_host_list_provider.test.ts index 86f20d00..99a91d3d 100644 --- a/tests/unit/rds_host_list_provider.test.ts +++ b/tests/unit/rds_host_list_provider.test.ts @@ -18,7 +18,7 @@ import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_hos import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { AwsClient } from "../../common/lib/aws_client"; -import { AwsWrapperError, HostInfo, HostInfoBuilder, HostRole } from "../../common/lib"; +import { AwsWrapperError, HostInfo, HostInfoBuilder } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { ConnectionUrlParser } from "../../common/lib/utils/connection_url_parser"; import { AwsPGClient } from "../../pg/lib"; @@ -30,11 +30,13 @@ import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; import { StorageService } from "../../common/lib/utils/storage/storage_service"; import { Topology } from "../../common/lib/host_list_provider/topology"; +import { TopologyQueryResult, TopologyUtils } from "../../common/lib/host_list_provider/topology_utils"; const mockClient: AwsClient = mock(AwsPGClient); const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const connectionUrlParser: ConnectionUrlParser = new PgConnectionUrlParser(); +const mockTopologyUtils: TopologyUtils = mock(TopologyUtils); const hosts: HostInfo[] = [ createHost({ @@ -67,7 +69,7 @@ function createHost(config: any): HostInfo { } function getRdsHostListProvider(originalHost: string): RdsHostListProvider { - const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockPluginService)); + const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockTopologyUtils), instance(mockPluginService)); provider.init(); return provider; } @@ -80,6 +82,7 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(currentHostInfo); when(mockPluginService.getConnectionUrlParser()).thenReturn(connectionUrlParser); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); + when(mockClient.targetClient).thenReturn(mockClientWrapper); when(mockPluginService.getHostInfoBuilder()).thenReturn(new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() })); }); @@ -104,7 +107,7 @@ describe("testRdsHostListProvider", () => { expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); - verify(spiedProvider.queryForTopology(anything(), anything())).never(); + verify(spiedProvider.getCurrentTopology(anything(), anything())).never(); }); it("testGetTopology_withForceUpdate_returnsUpdatedTopology", async () => { @@ -123,13 +126,13 @@ describe("testRdsHostListProvider", () => { ]; when(mockClient.isValid()).thenResolve(true); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(newHosts)); + when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(newHosts)); const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); expect(result.hosts.length).toEqual(1); expect(result.hosts).toEqual(newHosts); - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); + verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetTopology_noForceUpdate_queryReturnsEmptyHostList", async () => { @@ -140,12 +143,12 @@ describe("testRdsHostListProvider", () => { const expected: HostInfo[] = hosts; storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); + when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); + verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetTopology_withForceUpdate_returnsInitialHostList", async () => { @@ -160,21 +163,14 @@ describe("testRdsHostListProvider", () => { const spiedProvider = spy(rdsHostListProvider); spiedProvider.clear(); - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); + when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); expect(result.hosts).toBeTruthy(); for (let i = 0; i < result.hosts.length; i++) { expect(result.hosts[i].equals(initialHosts[i])).toBeTruthy(); } - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); - }); - - it("testQueryForTopology_queryResultsInError", async () => { - const rdsHostListProvider = getRdsHostListProvider("someUrl"); - when(mockDialect.queryForTopology(anything(), anything())).thenThrow(new AwsWrapperError("bad things")); - - await expect(rdsHostListProvider.queryForTopology(instance(mockClientWrapper), instance(mockDialect))).rejects.toThrow(AwsWrapperError); + verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetCachedTopology_returnCachedTopology", () => { @@ -202,20 +198,23 @@ describe("testRdsHostListProvider", () => { }); it("testIdentifyConnectionWithInvalidHostIdQuery", async () => { - when(mockDialect.identifyConnection(anything())).thenThrow(new AwsWrapperError("bad things")); + when(mockTopologyUtils.getInstanceId(anything())).thenThrow(new AwsWrapperError("bad things")); const rdsHostListProvider = getRdsHostListProvider("foo"); - await expect(rdsHostListProvider.identifyConnection(instance(mockClientWrapper), instance(mockDialect))).rejects.toThrow(AwsWrapperError); + await expect(rdsHostListProvider.identifyConnection(instance(mockClientWrapper))).rejects.toThrow(AwsWrapperError); }); it("testIdentifyConnectionHostInTopology", async () => { - when(mockDialect.queryForTopology(anything(), anything())).thenResolve([ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: "instance-1" + when(mockDialect.queryForTopology(anything())).thenResolve([ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: HostInfo.DEFAULT_WEIGHT, + lastUpdateTime: Date.now() }) ]); + when(await mockTopologyUtils.getInstanceId(anything())).thenReturn(["instance1", "instance1"]); const rdsHostListProvider = getRdsHostListProvider("foo"); const spiedProvider = spy(rdsHostListProvider); @@ -225,55 +224,7 @@ describe("testRdsHostListProvider", () => { }); when(spiedProvider.refresh()).thenReturn(Promise.resolve([])); - const res = await rdsHostListProvider.identifyConnection(instance(mockClientWrapper), instance(mockDialect)); + const res = await rdsHostListProvider.identifyConnection(instance(mockClientWrapper)); expect(res).toBeNull(); }); - - it("testGetTopology_staleRecord", async () => { - const hostName1: string = "hostName1"; - const hostName2: string = "hostName2"; - const cpuUtilization: number = 11.1; - const lag: number = 0.123; - const firstTimestamp: number = Date.now(); - const secondTimestamp: number = firstTimestamp + 100; - const weight = Math.round(lag) * 100 + Math.round(cpuUtilization); - - const expectedWriter: HostInfo = createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName2, - weight: weight, - lastUpdateTime: secondTimestamp - }); - - when(mockDialect.queryForTopology(anything(), anything())).thenResolve([ - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName1, - role: HostRole.WRITER, - weight: Math.round(lag) * 100 + Math.round(cpuUtilization), - lastUpdateTime: firstTimestamp - }), - createHost({ - hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), - host: hostName2, - role: HostRole.WRITER, - weight: Math.round(lag) * 100 + Math.round(cpuUtilization), - lastUpdateTime: secondTimestamp - }) - ]); - - when(mockPluginService.isClientValid(anything())).thenResolve(true); - when(mockPluginService.getDialect()).thenReturn(instance(mockDialect)); - - const rdsHostListProvider = getRdsHostListProvider("foo"); - const spiedProvider = spy(rdsHostListProvider); - rdsHostListProvider.isInitialized = false; - - when(spiedProvider.queryForTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - - const result = await rdsHostListProvider.getTopology(instance(mockClientWrapper), true); - verify(spiedProvider.queryForTopology(anything(), anything())).atMost(1); - expect(result.hosts.length).toEqual(1); - expect(result.hosts[0].equals(expectedWriter)).toBeTruthy(); - }); }); diff --git a/tests/unit/topology_utils.test.ts b/tests/unit/topology_utils.test.ts new file mode 100644 index 00000000..a757c8a6 --- /dev/null +++ b/tests/unit/topology_utils.test.ts @@ -0,0 +1,396 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TopologyQueryResult, TopologyUtils } from "../../common/lib/host_list_provider/topology_utils"; +import { anything, instance, mock, reset, when } from "ts-mockito"; +import { HostInfo, HostInfoBuilder } from "../../common/lib"; +import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; +import { AuroraPgDatabaseDialect } from "../../pg/lib/dialect/aurora_pg_database_dialect"; +import { ClientWrapper } from "../../common/lib/client_wrapper"; +import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { HostRole } from "../../common/lib/host_role"; +import { HostAvailability } from "../../common/lib/host_availability/host_availability"; +import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; + +const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); +const mockNonTopologyDialect: PgDatabaseDialect = mock(PgDatabaseDialect); + +const currentHostInfo = createHost({ + host: "foo", + port: 1234, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() +}); + +const clientWrapper: ClientWrapper = new PgClientWrapper(undefined, currentHostInfo, new Map()); +const mockClientWrapper: ClientWrapper = mock(clientWrapper); +const hostInfoBuilder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); + +function createHost(config: any): HostInfo { + const info = new HostInfoBuilder(config); + return info.build(); +} + +function getTopologyUtils(): TopologyUtils { + return new TopologyUtils(instance(mockDialect), hostInfoBuilder); +} + +describe("testTopologyUtils", () => { + beforeEach(() => { + reset(mockDialect); + reset(mockClientWrapper); + reset(mockNonTopologyDialect); + }); + + it("testQueryForTopology_withNonTopologyAwareDialect_throwsError", async () => { + const hostInfoBuilder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); + const topologyUtils = new TopologyUtils(instance(mockNonTopologyDialect) as any, hostInfoBuilder); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + await expect( + topologyUtils.queryForTopology(mockClientWrapper, instance(mockNonTopologyDialect), initialHost, clusterInstanceTemplate) + ).rejects.toThrow(TypeError); + }); + + it("testQueryForTopology_returnsTopology", async () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: true, weight: 100, lastUpdateTime: timestamp, port: 5432, id: "id-1" }), + new TopologyQueryResult({ host: "instance-2", isWriter: false, weight: 50, lastUpdateTime: timestamp, port: 5432, id: "id-2" }) + ]; + + when(mockDialect.queryForTopology(anything())).thenResolve(queryResults); + + const result = await topologyUtils.queryForTopology(mockClientWrapper, instance(mockDialect), initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result[1].role).toEqual(HostRole.WRITER); // Writer should be at the bottom. + expect(result[0].role).toEqual(HostRole.READER); + }); + + it("testCreateHost_withAllParameters", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const instanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const result = topologyUtils.createHost("id-1", "instance-1", true, 100, timestamp, initialHost, instanceTemplate); + + expect(result).toBeTruthy(); + expect(result.host).toEqual("instance-1.cluster-endpoint"); + expect(result.port).toEqual(3306); + expect(result.role).toEqual(HostRole.WRITER); + expect(result.availability).toEqual(HostAvailability.AVAILABLE); + expect(result.weight).toEqual(100); + expect(result.lastUpdateTime).toEqual(timestamp); + expect(result.hostId).toEqual("id-1"); + }); + + it("testCreateHost_withMissingInstanceName", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const instanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const result = topologyUtils.createHost("id-1", "", false, 50, timestamp, initialHost, instanceTemplate); + + expect(result).toBeTruthy(); + expect(result.host).toEqual("?.cluster-endpoint"); + expect(result.role).toEqual(HostRole.READER); + }); + + it("testCreateHosts_withMultipleResults", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: true, weight: 100, lastUpdateTime: timestamp, port: 3306, id: "id-1" }), + new TopologyQueryResult({ host: "instance-2", isWriter: false, weight: 50, lastUpdateTime: timestamp, port: 3306, id: "id-2" }), + new TopologyQueryResult({ host: "instance-3", isWriter: false, weight: 75, lastUpdateTime: timestamp, port: 3306, id: "id-3" }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(3); + expect(result[0].host).toEqual("instance-1.cluster-endpoint"); + expect(result[0].role).toEqual(HostRole.WRITER); + expect(result[1].host).toEqual("instance-2.cluster-endpoint"); + expect(result[1].role).toEqual(HostRole.READER); + expect(result[2].host).toEqual("instance-3.cluster-endpoint"); + expect(result[2].role).toEqual(HostRole.READER); + }); + + it("testCreateHosts_withEndpointInQueryResult", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: 100, + lastUpdateTime: timestamp, + port: 3306, + id: "id-1", + endpoint: "custom-endpoint.com" + }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].host).toEqual("custom-endpoint.com"); + }); + + it("testCreateHosts_withMissingPort", () => { + const topologyUtils = getTopologyUtils(); + const timestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ + host: "instance-1", + isWriter: true, + weight: 100, + lastUpdateTime: timestamp, + id: "id-1" + }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].port).toEqual(5432); + }); + + it("testVerifyWriter_withSingleWriter", async () => { + const topologyUtils = getTopologyUtils(); + + const writer = createHost({ + host: "writer-host", + port: 5432, + role: HostRole.WRITER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader = createHost({ + host: "reader-host", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [writer, reader]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result.filter((h) => h.role === HostRole.WRITER).length).toEqual(1); + }); + + it("testVerifyWriter_withMultipleWriters_selectsMostRecent", async () => { + const topologyUtils = getTopologyUtils(); + const oldTimestamp = Date.now() - 10000; + const newTimestamp = Date.now(); + + const oldWriter = createHost({ + host: "old-writer", + port: 5432, + role: HostRole.WRITER, + lastUpdateTime: oldTimestamp, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const newWriter = createHost({ + host: "new-writer", + port: 5432, + role: HostRole.WRITER, + lastUpdateTime: newTimestamp, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader = createHost({ + host: "reader-host", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [oldWriter, newWriter, reader]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(2); + expect(result.filter((h) => h.role === HostRole.WRITER).length).toEqual(1); + expect(result.find((h) => h.role === HostRole.WRITER)?.host).toEqual("new-writer"); + }); + + it("testVerifyWriter_withNoWriter_returnsNull", async () => { + const topologyUtils = getTopologyUtils(); + + const reader1 = createHost({ + host: "reader-1", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const reader2 = createHost({ + host: "reader-2", + port: 5432, + role: HostRole.READER, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const allHosts = [reader1, reader2]; + const result = await topologyUtils["verifyWriter"](allHosts); + + expect(result).toBeNull(); + }); + + it("testIsWriterInstance_returnsTrue", async () => { + const topologyUtils = getTopologyUtils(); + + when(mockDialect.getWriterId(anything())).thenResolve("writer-id"); + + const result = await topologyUtils.isWriterInstance(mockClientWrapper); + + expect(result).toBeTruthy(); + }); + + it("testIsWriterInstance_returnsFalse", async () => { + const topologyUtils = getTopologyUtils(); + + when(mockDialect.getWriterId(anything())).thenResolve(null); + + const result = await topologyUtils.isWriterInstance(mockClientWrapper); + + expect(result).toBeFalsy(); + }); + + it("testCreateHosts_withDuplicateHosts_keepsMostRecent", () => { + const topologyUtils = getTopologyUtils(); + const oldTimestamp = Date.now() - 10000; + const newTimestamp = Date.now(); + + const initialHost = createHost({ + host: "initial-host", + port: 5432, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const clusterInstanceTemplate = createHost({ + host: "?.cluster-endpoint", + port: 3306, + hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() + }); + + const queryResults: TopologyQueryResult[] = [ + new TopologyQueryResult({ host: "instance-1", isWriter: false, weight: 100, lastUpdateTime: oldTimestamp, port: 3306, id: "id-1" }), + new TopologyQueryResult({ host: "instance-1", isWriter: false, weight: 50, lastUpdateTime: newTimestamp, port: 3306, id: "id-1-updated" }) + ]; + + const result = topologyUtils.createHosts(queryResults, initialHost, clusterInstanceTemplate); + + expect(result).toBeTruthy(); + expect(result.length).toEqual(1); + expect(result[0].host).toEqual("instance-1.cluster-endpoint"); + expect(result[0].lastUpdateTime).toEqual(newTimestamp); + expect(result[0].hostId).toEqual("id-1-updated"); + }); +}); From 030df92dfda05ea37d84b09c956fab956a4811af Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Tue, 10 Mar 2026 01:16:43 -0700 Subject: [PATCH 11/20] feat: IAM support for GDB (#618) --- .../aws_secrets_manager_plugin.ts | 2 +- .../iam_authentication_plugin.ts | 35 ++-- common/lib/plugin_manager.ts | 1 + .../custom_endpoint/custom_endpoint_plugin.ts | 5 +- .../credentials_provider_factory.ts | 6 +- .../federated_auth/federated_auth_plugin.ts | 115 +----------- .../federated_auth/okta_auth_plugin.ts | 117 +----------- .../federated_auth/saml_auth_plugin.ts | 167 ++++++++++++++++++ .../saml_credentials_provider_factory.ts | 7 +- common/lib/utils/gdb_region_utils.ts | 110 ++++++++++++ common/lib/utils/iam_auth_utils.ts | 31 ++-- common/lib/utils/messages.ts | 3 + common/lib/utils/region_utils.ts | 15 +- tests/unit/aws_secrets_manager_plugin.test.ts | 14 ++ tests/unit/federated_auth_plugin.test.ts | 164 ----------------- tests/unit/gdb_region_utils.test.ts | 47 +++++ tests/unit/iam_authentication_plugin.test.ts | 39 ++-- tests/unit/okta_auth_plugin.test.ts | 166 ----------------- tests/unit/region_utils.test.ts | 65 +++++++ tests/unit/saml_auth_plugin.test.ts | 161 +++++++++++++++++ 20 files changed, 652 insertions(+), 618 deletions(-) create mode 100644 common/lib/plugins/federated_auth/saml_auth_plugin.ts create mode 100644 common/lib/utils/gdb_region_utils.ts delete mode 100644 tests/unit/federated_auth_plugin.test.ts create mode 100644 tests/unit/gdb_region_utils.test.ts delete mode 100644 tests/unit/okta_auth_plugin.test.ts create mode 100644 tests/unit/region_utils.test.ts create mode 100644 tests/unit/saml_auth_plugin.test.ts diff --git a/common/lib/authentication/aws_secrets_manager_plugin.ts b/common/lib/authentication/aws_secrets_manager_plugin.ts index f8e06e17..c7f2e621 100644 --- a/common/lib/authentication/aws_secrets_manager_plugin.ts +++ b/common/lib/authentication/aws_secrets_manager_plugin.ts @@ -37,7 +37,7 @@ export class AwsSecretsManagerPlugin extends AbstractConnectionPlugin implements private static readonly TELEMETRY_UPDATE_SECRETS = "fetch credentials"; private static readonly TELEMETRY_FETCH_CREDENTIALS_COUNTER = "secretsManager.fetchCredentials.count"; private static SUBSCRIBED_METHODS: Set = new Set(["connect", "forceConnect"]); - private static SECRETS_ARN_PATTERN: RegExp = new RegExp("^arn:aws:secretsmanager:(?[^:\\n]*):[^:\\n]*:([^:/\\n]*[:/])?(.*)$"); + private static SECRETS_ARN_PATTERN: RegExp = new RegExp("^arn:aws[^:]*:secretsmanager:(?[^:\\n]*):[^:\\n]*:([^:/\\n]*[:/])?(.*)$"); private readonly pluginService: PluginService; private readonly fetchCredentialsCounter; private readonly expirationSec: number; diff --git a/common/lib/authentication/iam_authentication_plugin.ts b/common/lib/authentication/iam_authentication_plugin.ts index d3a128a0..95da8a8b 100644 --- a/common/lib/authentication/iam_authentication_plugin.ts +++ b/common/lib/authentication/iam_authentication_plugin.ts @@ -26,19 +26,26 @@ import { IamAuthUtils, TokenInfo } from "../utils/iam_auth_utils"; import { ClientWrapper } from "../client_wrapper"; import { RegionUtils } from "../utils/region_utils"; import { CanReleaseResources } from "../can_release_resources"; +import { RdsUrlType } from "../utils/rds_url_type"; +import { RdsUtils } from "../utils/rds_utils"; +import { GDBRegionUtils } from "../utils/gdb_region_utils"; export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements CanReleaseResources { private static readonly SUBSCRIBED_METHODS = new Set(["connect", "forceConnect"]); protected static readonly tokenCache = new Map(); private readonly telemetryFactory; private readonly fetchTokenCounter; - private pluginService: PluginService; + private readonly pluginService: PluginService; + private readonly rdsUtils: RdsUtils = new RdsUtils(); + protected regionUtils: RegionUtils; + protected readonly iamAuthUtils: IamAuthUtils; - constructor(pluginService: PluginService) { + constructor(pluginService: PluginService, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { super(); this.pluginService = pluginService; this.telemetryFactory = this.pluginService.getTelemetryFactory(); this.fetchTokenCounter = this.telemetryFactory.createCounter("iam.fetchTokenCount"); + this.iamAuthUtils = iamAuthUtils; } getSubscribedMethods(): Set { @@ -74,14 +81,22 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements throw new AwsWrapperError(`${WrapperProperties.USER.name} is null or empty`); } - const host = IamAuthUtils.getIamHost(props, hostInfo); - const region: string = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getCurrentClient().defaultPort); + const host = this.iamAuthUtils.getIamHost(props, hostInfo); + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getCurrentClient().defaultPort); + + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host.host); + this.regionUtils = type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER ? new GDBRegionUtils() : new RegionUtils(); + const region: string | null = await this.regionUtils.getRegion(WrapperProperties.IAM_REGION.name, host, props); + + if (!region) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unableToDetermineRegion", WrapperProperties.IAM_REGION.name)); + } + const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); if (tokenExpirationSec < 0) { throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); } - const cacheKey: string = IamAuthUtils.getCacheKey(port, user, host, region); + const cacheKey: string = this.iamAuthUtils.getCacheKey(port, user, host.host, region); const tokenInfo = IamAuthenticationPlugin.tokenCache.get(cacheKey); const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); @@ -91,8 +106,8 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements WrapperProperties.PASSWORD.set(props, tokenInfo.token); } else { const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const token = await IamAuthUtils.generateAuthenticationToken( - host, + const token = await this.iamAuthUtils.generateAuthenticationToken( + host.host, port, region, user, @@ -118,8 +133,8 @@ export class IamAuthenticationPlugin extends AbstractConnectionPlugin implements // Try to generate a new token and try to connect again const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const token = await IamAuthUtils.generateAuthenticationToken( - host, + const token = await this.iamAuthUtils.generateAuthenticationToken( + host.host, port, region, user, diff --git a/common/lib/plugin_manager.ts b/common/lib/plugin_manager.ts index 49256696..f66e65ba 100644 --- a/common/lib/plugin_manager.ts +++ b/common/lib/plugin_manager.ts @@ -32,6 +32,7 @@ import { TelemetryTraceLevel } from "./utils/telemetry/telemetry_trace_level"; import { ConnectionProvider } from "./connection_provider"; import { ConnectionPluginFactory } from "./plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; +import { BaseSamlAuthPlugin } from "./plugins/federated_auth/saml_auth_plugin"; type PluginFunc = (plugin: ConnectionPlugin, targetFunc: () => Promise) => Promise; diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts index 99e5bfb4..70dee99c 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin.ts @@ -31,11 +31,14 @@ import { sleep } from "../../utils/utils"; import { CustomEndpointMonitor, CustomEndpointMonitorImpl } from "./custom_endpoint_monitor_impl"; import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; import { CanReleaseResources } from "../../can_release_resources"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { GDBRegionUtils } from "../../utils/gdb_region_utils"; export class CustomEndpointPlugin extends AbstractConnectionPlugin implements CanReleaseResources { private static readonly TELEMETRY_WAIT_FOR_INFO_COUNTER = "customEndpoint.waitForInfo.counter"; private static SUBSCRIBED_METHODS: Set = new Set(SubscribedMethodHelper.NETWORK_BOUND_METHODS); private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); + private static readonly regionUtils: RegionUtils = new RegionUtils(); private static readonly rdsUtils = new RdsUtils(); protected static readonly monitors: SlidingExpirationCache = new SlidingExpirationCache( @@ -106,7 +109,7 @@ export class CustomEndpointPlugin extends AbstractConnectionPlugin implements Ca throw new AwsWrapperError(Messages.get("CustomEndpointPlugin.errorParsingEndpointIdentifier", this.customEndpointHostInfo.host)); } - this.region = RegionUtils.getRegion(props.get(WrapperProperties.CUSTOM_ENDPOINT_REGION.name), this.customEndpointHostInfo.host); + this.region = await CustomEndpointPlugin.regionUtils.getRegion(WrapperProperties.CUSTOM_ENDPOINT_REGION.name, this.customEndpointHostInfo, props); if (!this.region) { throw new AwsWrapperError(Messages.get("CustomEndpointPlugin.unableToDetermineRegion", WrapperProperties.CUSTOM_ENDPOINT_REGION.name)); } diff --git a/common/lib/plugins/federated_auth/credentials_provider_factory.ts b/common/lib/plugins/federated_auth/credentials_provider_factory.ts index e879fd8d..edca130f 100644 --- a/common/lib/plugins/federated_auth/credentials_provider_factory.ts +++ b/common/lib/plugins/federated_auth/credentials_provider_factory.ts @@ -17,5 +17,9 @@ import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; export interface CredentialsProviderFactory { - getAwsCredentialsProvider(host: string, region: string, props: Map): Promise; + getAwsCredentialsProvider( + host: string, + region: string | null, + props: Map + ): Promise; } diff --git a/common/lib/plugins/federated_auth/federated_auth_plugin.ts b/common/lib/plugins/federated_auth/federated_auth_plugin.ts index 2a440dd5..40672037 100644 --- a/common/lib/plugins/federated_auth/federated_auth_plugin.ts +++ b/common/lib/plugins/federated_auth/federated_auth_plugin.ts @@ -14,118 +14,13 @@ limitations under the License. */ -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { PluginService } from "../../plugin_service"; -import { RdsUtils } from "../../utils/rds_utils"; -import { HostInfo } from "../../host_info"; -import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; -import { WrapperProperties } from "../../wrapper_property"; -import { logger } from "../../../logutils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; import { CredentialsProviderFactory } from "./credentials_provider_factory"; -import { SamlUtils } from "../../utils/saml_utils"; -import { ClientWrapper } from "../../client_wrapper"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { RegionUtils } from "../../utils/region_utils"; -import { CanReleaseResources } from "../../can_release_resources"; +import { BaseSamlAuthPlugin } from "./saml_auth_plugin"; +import { IamAuthUtils } from "../../utils/iam_auth_utils"; -export class FederatedAuthPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - protected static readonly tokenCache = new Map(); - protected rdsUtils: RdsUtils = new RdsUtils(); - protected pluginService: PluginService; - private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); - private readonly credentialsProviderFactory: CredentialsProviderFactory; - private readonly fetchTokenCounter: TelemetryCounter; - - public getSubscribedMethods(): Set { - return FederatedAuthPlugin.subscribedMethods; - } - - constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory) { - super(); - this.credentialsProviderFactory = credentialsProviderFactory; - this.pluginService = pluginService; - this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter("federatedAuth.fetchToken.count"); - } - - connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - forceConnect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - forceConnectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, forceConnectFunc); - } - - async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { - SamlUtils.checkIdpCredentialsWithFallback(props); - - const host = IamAuthUtils.getIamHost(props, hostInfo); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const region: string = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - - const cacheKey = IamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host, region); - const tokenInfo = FederatedAuthPlugin.tokenCache.get(cacheKey); - - const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); - - if (isCachedToken && tokenInfo) { - logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); - WrapperProperties.PASSWORD.set(props, tokenInfo.token); - } else { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - } - WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); - this.pluginService.updateConfigWithProperties(props); - - try { - return await connectFunc(); - } catch (e) { - if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { - throw e; - } - try { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - return await connectFunc(); - } catch (e: any) { - throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); - } - } - } - - public async updateAuthenticationToken(hostInfo: HostInfo, props: Map, region: string, cacheKey: string, iamHost: string) { - const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); - if (tokenExpirationSec < 0) { - throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); - } - const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const token = await IamAuthUtils.generateAuthenticationToken( - iamHost, - port, - region, - WrapperProperties.DB_USER.get(props), - await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props), - this.pluginService - ); - this.fetchTokenCounter.inc(); - logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); - WrapperProperties.PASSWORD.set(props, token); - FederatedAuthPlugin.tokenCache.set(cacheKey, new TokenInfo(token, tokenExpiry)); - } - - releaseResources(): Promise { - FederatedAuthPlugin.tokenCache.clear(); - return; +export class FederatedAuthPlugin extends BaseSamlAuthPlugin { + constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { + super(pluginService, credentialsProviderFactory, "federatedAuth.fetchToken.count", iamAuthUtils); } } diff --git a/common/lib/plugins/federated_auth/okta_auth_plugin.ts b/common/lib/plugins/federated_auth/okta_auth_plugin.ts index 9de79bd1..31b6144f 100644 --- a/common/lib/plugins/federated_auth/okta_auth_plugin.ts +++ b/common/lib/plugins/federated_auth/okta_auth_plugin.ts @@ -14,120 +14,13 @@ limitations under the License. */ -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { HostInfo } from "../../host_info"; -import { SamlUtils } from "../../utils/saml_utils"; -import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; import { PluginService } from "../../plugin_service"; import { CredentialsProviderFactory } from "./credentials_provider_factory"; -import { RdsUtils } from "../../utils/rds_utils"; -import { WrapperProperties } from "../../wrapper_property"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { AwsWrapperError } from "../../utils/errors"; -import { ClientWrapper } from "../../client_wrapper"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { RegionUtils } from "../../utils/region_utils"; -import { CanReleaseResources } from "../../can_release_resources"; +import { BaseSamlAuthPlugin } from "./saml_auth_plugin"; +import { IamAuthUtils } from "../../utils/iam_auth_utils"; -export class OktaAuthPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - protected static readonly tokenCache = new Map(); - private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); - protected pluginService: PluginService; - protected rdsUtils = new RdsUtils(); - private readonly credentialsProviderFactory: CredentialsProviderFactory; - private readonly fetchTokenCounter: TelemetryCounter; - - constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory) { - super(); - this.pluginService = pluginService; - this.credentialsProviderFactory = credentialsProviderFactory; - this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter("oktaAuth.fetchToken.count"); - } - - public getSubscribedMethods(): Set { - return OktaAuthPlugin.subscribedMethods; - } - - connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - forceConnect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - return this.connectInternal(hostInfo, props, connectFunc); - } - - async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { - SamlUtils.checkIdpCredentialsWithFallback(props); - - const host = IamAuthUtils.getIamHost(props, hostInfo); - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - const region = RegionUtils.getRegion(props.get(WrapperProperties.IAM_REGION.name), host); - - const cacheKey = IamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host, region); - const tokenInfo = OktaAuthPlugin.tokenCache.get(cacheKey); - - const isCachedToken = tokenInfo !== undefined && !tokenInfo.isExpired(); - - if (isCachedToken) { - logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); - WrapperProperties.PASSWORD.set(props, tokenInfo.token); - } else { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - } - WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); - this.pluginService.updateConfigWithProperties(props); - - try { - return await connectFunc(); - } catch (e: any) { - if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { - logger.debug(Messages.get("Authentication.connectError", e.message)); - throw e; - } - try { - await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host); - return await connectFunc(); - } catch (e: any) { - throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); - } - } - } - - public async updateAuthenticationToken(hostInfo: HostInfo, props: Map, region: string, cacheKey: string, iamHost): Promise { - const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); - if (tokenExpirationSec < 0) { - throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); - } - const tokenExpiry = Date.now() + tokenExpirationSec * 1000; - const port = IamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); - this.fetchTokenCounter.inc(); - const token = await IamAuthUtils.generateAuthenticationToken( - iamHost, - port, - region, - WrapperProperties.DB_USER.get(props), - await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props), - this.pluginService - ); - logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); - WrapperProperties.PASSWORD.set(props, token); - this.pluginService.updateConfigWithProperties(props); - OktaAuthPlugin.tokenCache.set(cacheKey, new TokenInfo(token, tokenExpiry)); - } - - releaseResources(): Promise { - OktaAuthPlugin.tokenCache.clear(); - return; +export class OktaAuthPlugin extends BaseSamlAuthPlugin { + constructor(pluginService: PluginService, credentialsProviderFactory: CredentialsProviderFactory, iamAuthUtils: IamAuthUtils = new IamAuthUtils()) { + super(pluginService, credentialsProviderFactory, "oktaAuth.fetchToken.count", iamAuthUtils); } } diff --git a/common/lib/plugins/federated_auth/saml_auth_plugin.ts b/common/lib/plugins/federated_auth/saml_auth_plugin.ts new file mode 100644 index 00000000..7aa48db0 --- /dev/null +++ b/common/lib/plugins/federated_auth/saml_auth_plugin.ts @@ -0,0 +1,167 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; +import { PluginService } from "../../plugin_service"; +import { RdsUtils } from "../../utils/rds_utils"; +import { HostInfo } from "../../host_info"; +import { IamAuthUtils, TokenInfo } from "../../utils/iam_auth_utils"; +import { WrapperProperties } from "../../wrapper_property"; +import { logger } from "../../../logutils"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { CredentialsProviderFactory } from "./credentials_provider_factory"; +import { SamlUtils } from "../../utils/saml_utils"; +import { ClientWrapper } from "../../client_wrapper"; +import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; +import { RegionUtils } from "../../utils/region_utils"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { GDBRegionUtils } from "../../utils/gdb_region_utils"; +import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; + +export class BaseSamlAuthPlugin extends AbstractConnectionPlugin { + protected static readonly tokenCache = new Map(); + protected rdsUtils: RdsUtils = new RdsUtils(); + protected pluginService: PluginService; + private static readonly subscribedMethods = new Set(["connect", "forceConnect"]); + protected readonly credentialsProviderFactory: CredentialsProviderFactory; + protected readonly fetchTokenCounter: TelemetryCounter; + protected regionUtils: RegionUtils; + protected readonly tokenCacheInstance: Map; + + private readonly iamAuthUtils: IamAuthUtils; + + public getSubscribedMethods(): Set { + return BaseSamlAuthPlugin.subscribedMethods; + } + + protected constructor( + pluginService: PluginService, + credentialsProviderFactory: CredentialsProviderFactory, + telemetryCounterName: string, + iamAuthUtils: IamAuthUtils = new IamAuthUtils() + ) { + super(); + this.credentialsProviderFactory = credentialsProviderFactory; + this.pluginService = pluginService; + this.fetchTokenCounter = this.pluginService.getTelemetryFactory().createCounter(telemetryCounterName); + this.tokenCacheInstance = BaseSamlAuthPlugin.tokenCache; + this.iamAuthUtils = iamAuthUtils; + } + + connect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + connectFunc: () => Promise + ): Promise { + return this.connectInternal(hostInfo, props, connectFunc); + } + + forceConnect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + forceConnectFunc: () => Promise + ): Promise { + return this.connectInternal(hostInfo, props, forceConnectFunc); + } + + async connectInternal(hostInfo: HostInfo, props: Map, connectFunc: () => Promise): Promise { + SamlUtils.checkIdpCredentialsWithFallback(props); + + const host = this.iamAuthUtils.getIamHost(props, hostInfo); + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); + + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host.host); + + let credentialsProvider: AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined = undefined; + if (type === RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + credentialsProvider = await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, null, props); + } + + this.regionUtils = type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER ? new GDBRegionUtils(credentialsProvider) : new RegionUtils(); + const region: string | null = await this.regionUtils.getRegion(WrapperProperties.IAM_REGION.name, host, props); + + if (!region) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unableToDetermineRegion", WrapperProperties.IAM_REGION.name)); + } + + const cacheKey = this.iamAuthUtils.getCacheKey(port, WrapperProperties.DB_USER.get(props), host.host, region); + const tokenInfo = this.tokenCacheInstance.get(cacheKey); + + const isCachedToken: boolean = tokenInfo !== undefined && !tokenInfo.isExpired(); + + if (isCachedToken && tokenInfo) { + logger.debug(Messages.get("AuthenticationToken.useCachedToken", tokenInfo.token)); + WrapperProperties.PASSWORD.set(props, tokenInfo.token); + } else { + await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host.host, credentialsProvider); + } + WrapperProperties.USER.set(props, WrapperProperties.DB_USER.get(props)); + this.pluginService.updateConfigWithProperties(props); + + try { + return await connectFunc(); + } catch (e: any) { + if (!this.pluginService.isLoginError(e as Error) || !isCachedToken) { + throw e; + } + try { + await this.updateAuthenticationToken(hostInfo, props, region, cacheKey, host.host, credentialsProvider); + return await connectFunc(); + } catch (e: any) { + throw new AwsWrapperError(Messages.get("SamlAuthPlugin.unhandledError", e.message)); + } + } + } + + public async updateAuthenticationToken( + hostInfo: HostInfo, + props: Map, + region: string, + cacheKey: string, + iamHost: string, + credentials?: AwsCredentialIdentity | AwsCredentialIdentityProvider + ): Promise { + const tokenExpirationSec = WrapperProperties.IAM_TOKEN_EXPIRATION.get(props); + if (tokenExpirationSec < 0) { + throw new AwsWrapperError(Messages.get("AuthenticationToken.tokenExpirationLessThanZero")); + } + const tokenExpiry: number = Date.now() + tokenExpirationSec * 1000; + const port = this.iamAuthUtils.getIamPort(props, hostInfo, this.pluginService.getDialect().getDefaultPort()); + + this.fetchTokenCounter.inc(); + + const token = await this.iamAuthUtils.generateAuthenticationToken( + iamHost, + port, + region, + WrapperProperties.DB_USER.get(props), + credentials ?? (await this.credentialsProviderFactory.getAwsCredentialsProvider(hostInfo.host, region, props)), + this.pluginService + ); + + logger.debug(Messages.get("AuthenticationToken.generatedNewToken", token)); + WrapperProperties.PASSWORD.set(props, token); + this.pluginService.updateConfigWithProperties(props); + this.tokenCacheInstance.set(cacheKey, new TokenInfo(token, tokenExpiry)); + } + + static releaseResources(): void { + BaseSamlAuthPlugin.tokenCache.clear(); + } +} diff --git a/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts b/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts index bfdfe7f6..e6dfc397 100644 --- a/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts +++ b/common/lib/plugins/federated_auth/saml_credentials_provider_factory.ts @@ -17,7 +17,6 @@ import { CredentialsProviderFactory } from "./credentials_provider_factory"; import { AssumeRoleWithSAMLCommand, STSClient } from "@aws-sdk/client-sts"; import { WrapperProperties } from "../../wrapper_property"; - import { AwsWrapperError } from "../../utils/errors"; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; import { decode } from "entities"; @@ -25,7 +24,7 @@ import { decode } from "entities"; export abstract class SamlCredentialsProviderFactory implements CredentialsProviderFactory { async getAwsCredentialsProvider( host: string, - region: string, + region: string | null, props: Map ): Promise { const samlAssertion = await this.getSamlAssertion(props); @@ -35,9 +34,7 @@ export abstract class SamlCredentialsProviderFactory implements CredentialsProvi PrincipalArn: WrapperProperties.IAM_IDP_ARN.get(props) }); - const stsClient = new STSClient({ - region: region - }); + const stsClient = region !== null ? new STSClient({ region }) : new STSClient(); const results = await stsClient.send(assumeRoleWithSamlRequest); const credentials = results["Credentials"]; diff --git a/common/lib/utils/gdb_region_utils.ts b/common/lib/utils/gdb_region_utils.ts new file mode 100644 index 00000000..6b1fa5a4 --- /dev/null +++ b/common/lib/utils/gdb_region_utils.ts @@ -0,0 +1,110 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RegionUtils } from "./region_utils"; +import { HostInfo } from "../host_info"; +import { AwsCredentialsManager } from "../authentication/aws_credentials_manager"; +import { DescribeGlobalClustersCommand, GlobalCluster, GlobalClusterMember, RDSClient } from "@aws-sdk/client-rds"; +import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; +import { logger } from "../../logutils"; +import { Messages } from "./messages"; +import { AwsWrapperError } from "./errors"; + +export class GDBRegionUtils extends RegionUtils { + private static readonly GDB_CLUSTER_ARN_PATTERN = /^arn:aws[^:]*:rds:(?[^:\n]*):([^:\n]*):([^:/\n]*[:/])?(.*)$/; + private static readonly REGION_GROUP = "region"; + private credentialsProvider: AwsCredentialIdentity | AwsCredentialIdentityProvider | undefined; + + constructor(credentialsProvider?: AwsCredentialIdentity | AwsCredentialIdentityProvider) { + super(); + this.credentialsProvider = credentialsProvider; + } + + async getRegion(regionKey: string, hostInfo?: HostInfo, props?: Map): Promise { + if (!hostInfo || !props) { + return null; + } + + if (props.get(regionKey)) { + return this.getRegionFromRegionString(props.get(regionKey)); + } + + const clusterId = GDBRegionUtils.rdsUtils.getRdsClusterId(hostInfo.host); + if (!clusterId) { + return null; + } + + const writerClusterArn = await this.findWriterClusterArn(hostInfo, props, clusterId); + return writerClusterArn ? this.getRegionFromClusterArn(writerClusterArn) : null; + } + + private async findWriterClusterArn(hostInfo: HostInfo, props: Map, globalClusterIdentifier: string): Promise { + if (!this.credentialsProvider) { + this.credentialsProvider = AwsCredentialsManager.getProvider(hostInfo, props); + } + + const rdsClient = this.getRdsClient(); + + try { + const command = new DescribeGlobalClustersCommand({ + GlobalClusterIdentifier: globalClusterIdentifier + }); + + const response = await rdsClient.send(command); + return this.extractWriterClusterArn(response.GlobalClusters); + } catch (error) { + if (error instanceof Error) { + logger.debug(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); + throw new AwsWrapperError(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); + } + } finally { + rdsClient.destroy(); + } + } + + private extractWriterClusterArn(globalClusters?: GlobalCluster[]): string | null { + if (!globalClusters) { + return null; + } + + for (const cluster of globalClusters) { + const writerArn = this.findWriterMemberArn(cluster.GlobalClusterMembers); + if (writerArn) { + return writerArn; + } + } + + return null; + } + + getRegionFromClusterArn(clusterArn: string): string | null { + const match = clusterArn.match(GDBRegionUtils.GDB_CLUSTER_ARN_PATTERN); + return match?.groups?.[GDBRegionUtils.REGION_GROUP] ?? null; + } + + private findWriterMemberArn(members?: GlobalClusterMember[]): string | null { + if (!members) { + return null; + } + + const writerMember = members.find((member) => member.IsWriter); + return writerMember?.DBClusterArn ?? null; + } + + private getRdsClient(): RDSClient { + return new RDSClient({ credentials: this.credentialsProvider }); + } +} diff --git a/common/lib/utils/iam_auth_utils.ts b/common/lib/utils/iam_auth_utils.ts index 42525ddf..7a8d29e5 100644 --- a/common/lib/utils/iam_auth_utils.ts +++ b/common/lib/utils/iam_auth_utils.ts @@ -16,23 +16,26 @@ import { logger } from "../../logutils"; import { HostInfo } from "../host_info"; -import { WrapperProperties, WrapperProperty } from "../wrapper_property"; -import { AwsWrapperError } from "./errors"; +import { WrapperProperties } from "../wrapper_property"; import { Messages } from "./messages"; -import { RdsUtils } from "./rds_utils"; import { Signer } from "@aws-sdk/rds-signer"; import { AwsCredentialIdentity, AwsCredentialIdentityProvider } from "@smithy/types/dist-types/identity/awsCredentialIdentity"; import { PluginService } from "../plugin_service"; import { TelemetryTraceLevel } from "./telemetry/telemetry_trace_level"; +import { HostInfoBuilder } from "../host_info_builder"; export class IamAuthUtils { private static readonly TELEMETRY_FETCH_TOKEN = "fetch IAM token"; - public static getIamHost(props: Map, hostInfo: HostInfo): string { - return WrapperProperties.IAM_HOST.get(props) ? WrapperProperties.IAM_HOST.get(props) : hostInfo.host; + public getIamHost(props: Map, hostInfo: HostInfo): HostInfo { + const iamHost: string | null = WrapperProperties.IAM_HOST.get(props); + + return iamHost + ? new HostInfoBuilder({ hostAvailabilityStrategy: hostInfo.hostAvailabilityStrategy }).copyFrom(hostInfo).withHost(iamHost).build() + : hostInfo; } - public static getIamPort(props: Map, hostInfo: HostInfo, defaultPort: number): number { + public getIamPort(props: Map, hostInfo: HostInfo, defaultPort: number): number { const port = WrapperProperties.IAM_DEFAULT_PORT.get(props); if (port) { if (isNaN(port) || port <= 0) { @@ -49,23 +52,11 @@ export class IamAuthUtils { } } - public static getRdsRegion(hostname: string, rdsUtils: RdsUtils, props: Map, wrapperProperty: WrapperProperty): string { - const rdsRegion = rdsUtils.getRdsRegion(hostname); - - if (!rdsRegion) { - const errorMessage = Messages.get("Authentication.unsupportedHostname", hostname); - logger.debug(errorMessage); - throw new AwsWrapperError(errorMessage); - } - - return wrapperProperty.get(props) ? wrapperProperty.get(props) : rdsRegion; - } - - public static getCacheKey(port: number, user?: string, hostname?: string, region?: string): string { + public getCacheKey(port: number, user?: string, hostname?: string, region?: string): string { return `${region}:${hostname}:${port}:${user}`; } - public static async generateAuthenticationToken( + public async generateAuthenticationToken( hostname: string, port: number, region: string, diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index abc4db8d..d7d72fe5 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -102,6 +102,7 @@ const MESSAGES: Record = { "Failover.timeoutError": "Internal failover task has timed out.", "Failover.newWriterNotAllowed": "The failover process identified the new writer but the host is not in the list of allowed hosts. New writer host: '%s'. Allowed hosts: '%s'.", + "GDBRegionUtils.unableToRetrieveGlobalClusterARN": "Unable to retrieve the primary global region for the provided global database cluster.", "StaleDnsHelper.clusterEndpointDns": "Cluster endpoint resolves to '%s'.", "StaleDnsHelper.writerHostInfo": "Writer host: '%s'.", "StaleDnsHelper.writerInetAddress": "Writer host address: '%s'", @@ -167,6 +168,8 @@ const MESSAGES: Record = { "Okta SAML Assertion request failed with HTTP status '%s', reason phrase '%s', and response '%s'", "SamlCredentialsProviderFactory.getSamlAssertionFailed": "Failed to get SAML Assertion due to error: '%s'", "SamlAuthPlugin.unhandledError": "Unhandled error: '%s'", + "SamlAuthPlugin.unableToDetermineRegion": + "Unable to determine connection region. If you are using a non-standard RDS URL, please set the '%s' property.", "HostAvailabilityStrategy.invalidMaxRetries": "Invalid value of '%s' for configuration parameter `hostAvailabilityStrategyMaxRetries`. It must be an integer greater or equal to 1.", "HostAvailabilityStrategy.invalidInitialBackoffTime": diff --git a/common/lib/utils/region_utils.ts b/common/lib/utils/region_utils.ts index 7b26a619..4c5a008a 100644 --- a/common/lib/utils/region_utils.ts +++ b/common/lib/utils/region_utils.ts @@ -17,6 +17,7 @@ import { RdsUtils } from "./rds_utils"; import { AwsWrapperError } from "./errors"; import { Messages } from "./messages"; +import { HostInfo } from "../host_info"; export class RegionUtils { static readonly REGIONS: string[] = [ @@ -67,21 +68,21 @@ export class RegionUtils { protected static readonly rdsUtils = new RdsUtils(); - static getRegion(regionString: string, host?: string): string | null { - const region = RegionUtils.getRegionFromRegionString(regionString); + async getRegion(regionKey: string, hostInfo?: HostInfo, props?: Map): Promise { + const region = this.getRegionFromRegionString(props?.get(regionKey)); if (region !== null) { return region; } - if (host) { - return RegionUtils.getRegionFromHost(host); + if (hostInfo) { + return this.getRegionFromHost(hostInfo.host); } return region; } - private static getRegionFromRegionString(regionString: string): string { + getRegionFromRegionString(regionString: string): string | null { if (!regionString) { return null; } @@ -94,12 +95,12 @@ export class RegionUtils { return region; } - private static getRegionFromHost(host: string): string | null { + getRegionFromHost(host: string): string | null { const regionString = RegionUtils.rdsUtils.getRdsRegion(host); if (!regionString) { throw new AwsWrapperError(Messages.get("AwsSdk.unsupportedRegion", regionString)); } - return RegionUtils.getRegionFromRegionString(regionString); + return this.getRegionFromRegionString(regionString); } } diff --git a/tests/unit/aws_secrets_manager_plugin.test.ts b/tests/unit/aws_secrets_manager_plugin.test.ts index e76278a1..4aa1271e 100644 --- a/tests/unit/aws_secrets_manager_plugin.test.ts +++ b/tests/unit/aws_secrets_manager_plugin.test.ts @@ -192,6 +192,20 @@ describe("testSecretsManager", () => { expect(secretKey.region).toBe(expectedRegionParsedFromARN); }); + it.each([ + ["arn:aws:secretsmanager:us-east-1:123456789012:secret:mysecret", "us-east-1"], + ["arn:aws-us-gov:secretsmanager:us-gov-west-1:123456789012:secret:mysecret", "us-gov-west-1"], + ["arn:aws-cn:secretsmanager:cn-north-1:123456789012:secret:mysecret", "cn-north-1"], + ["arn:aws-iso:secretsmanager:us-iso-east-1:123456789012:secret:mysecret", "us-iso-east-1"], + ["arn:aws-iso-b:secretsmanager:us-isob-east-1:123456789012:secret:mysecret", "us-isob-east-1"] + ])("connect using partition-agnostic arn: %s", async (arn, expectedRegion) => { + const props = new Map(); + WrapperProperties.SECRET_ID.set(props, arn); + const testPlugin = new AwsSecretsManagerPlugin(instance(mockPluginService), props); + const secretKey = testPlugin.secretKey; + expect(secretKey.region).toBe(expectedRegion); + }); + it.each([ [TEST_ARN_1, "us-east-2"], [TEST_ARN_2, "us-west-1"], diff --git a/tests/unit/federated_auth_plugin.test.ts b/tests/unit/federated_auth_plugin.test.ts deleted file mode 100644 index d696e451..00000000 --- a/tests/unit/federated_auth_plugin.test.ts +++ /dev/null @@ -1,164 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { HostInfo, HostRole, PluginManager } from "../../common/lib"; -import { FederatedAuthPlugin } from "../../common/lib/plugins/federated_auth/federated_auth_plugin"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; -import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { anything, instance, mock, spy, verify, when } from "ts-mockito"; -import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; -import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; -import { jest } from "@jest/globals"; -import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; - -const testToken = "testToken"; -const defaultPort = 5432; -const pgCacheKey = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; -const dbUser = "iamUser"; -const expirationFiveMinutes = 5 * 60 * 1000; -const tokenCache = new Map(); - -const host = "pg.testdb.us-east-2.rds.amazonaws.com"; -const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; -const hostInfo = new HostInfo(host, defaultPort, HostRole.WRITER); -const testTokenInfo = new TokenInfo(testToken, Date.now() + expirationFiveMinutes); - -const mockDialect = mock(); -const mockDialectInstance = instance(mockDialect); -const mockPluginService = mock(PluginServiceImpl); -const mockCredentialsProviderFactory = mock(); -const spyIamUtils = spy(IamAuthUtils); -const testCredentials = { - accessKeyId: "foo", - secretAccessKey: "bar", - sessionToken: "baz" -}; -const mockConnectFunc = jest.fn(() => { - return Promise.resolve(mock(PgClientWrapper)); -}); - -describe("federatedAuthTest", () => { - let spyPlugin: FederatedAuthPlugin; - let props: Map; - - beforeEach(() => { - when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - when(mockDialect.getDefaultPort()).thenReturn(defaultPort); - when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(instance(testCredentials)); - props = new Map(); - WrapperProperties.PLUGINS.set(props, "federatedAuth"); - WrapperProperties.DB_USER.set(props, dbUser); - spyPlugin = spy(new FederatedAuthPlugin(instance(mockPluginService), instance(mockCredentialsProviderFactory))); - }); - - afterEach(async () => { - await PluginManager.releaseResources(); - }); - - it("testCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - FederatedAuthPlugin["tokenCache"].set(pgCacheKey, testTokenInfo); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - tokenCache.set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testExpiredCachedToken", async () => { - const spyPluginInstance: FederatedAuthPlugin = instance(spyPlugin); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - const expiredToken = "expiredToken"; - const expiredTokenInfo = new TokenInfo(expiredToken, Date.now() - 300000); - - FederatedAuthPlugin["tokenCache"].set(key, expiredTokenInfo); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testNoCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testSpecifiedIamHostPortRegion", async () => { - const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; - const expectedPort = 9876; - const expectedRegion = "us-west-2"; - - WrapperProperties.IAM_HOST.set(props, expectedHost); - WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); - WrapperProperties.IAM_REGION.set(props, expectedRegion); - - const key = `us-west-2:pg.testdb.us-west-2.rds.amazonaws.com:${expectedPort}:iamUser`; - FederatedAuthPlugin["tokenCache"].set(key, testTokenInfo); - - const spyPluginInstance = instance(spyPlugin); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testIdpCredentialsFallback", async () => { - const expectedUser = "expectedUser"; - const expectedPassword = "expectedPassword"; - WrapperProperties.USER.set(props, expectedUser); - WrapperProperties.PASSWORD.set(props, expectedPassword); - - const spyPluginInstance = instance(spyPlugin); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - FederatedAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - expect(expectedUser).toBe(WrapperProperties.IDP_USERNAME.get(props)); - expect(expectedPassword).toBe(WrapperProperties.IDP_PASSWORD.get(props)); - }); - - it("testUsingIamHost", async () => { - WrapperProperties.IAM_HOST.set(props, iamHost); - const spyPluginInstance = instance(spyPlugin); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - verify(spyIamUtils.generateAuthenticationToken(iamHost, anything(), anything(), anything(), anything(), anything())).once(); - }); -}); diff --git a/tests/unit/gdb_region_utils.test.ts b/tests/unit/gdb_region_utils.test.ts new file mode 100644 index 00000000..243eae9e --- /dev/null +++ b/tests/unit/gdb_region_utils.test.ts @@ -0,0 +1,47 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { GDBRegionUtils } from "../../common/lib/utils/gdb_region_utils"; + +describe("GDBRegionUtils", () => { + let gdbRegionUtils: GDBRegionUtils; + + beforeEach(() => { + gdbRegionUtils = new GDBRegionUtils(); + }); + + describe("getRegionFromClusterArn", () => { + it.each([ + ["arn:aws:rds:us-east-1:123456789012:cluster:my-cluster", "us-east-1"], + ["arn:aws:rds:us-west-2:123456789012:cluster:my-cluster", "us-west-2"], + ["arn:aws:rds:eu-west-1:123456789012:cluster:my-cluster", "eu-west-1"], + ["arn:aws-us-gov:rds:us-gov-west-1:123456789012:cluster:my-cluster", "us-gov-west-1"], + ["arn:aws-us-gov:rds:us-gov-east-1:123456789012:cluster:my-cluster", "us-gov-east-1"], + ["arn:aws-cn:rds:cn-north-1:123456789012:cluster:my-cluster", "cn-north-1"], + ["arn:aws-cn:rds:cn-northwest-1:123456789012:cluster:my-cluster", "cn-northwest-1"], + ["arn:aws-iso:rds:us-iso-east-1:123456789012:cluster:my-cluster", "us-iso-east-1"], + ["arn:aws-iso-b:rds:us-isob-east-1:123456789012:cluster:my-cluster", "us-isob-east-1"] + ])("should extract region from partition-agnostic ARN: %s", (arn, expectedRegion) => { + const region = gdbRegionUtils.getRegionFromClusterArn(arn); + expect(region).toBe(expectedRegion); + }); + + it.each(["invalid-arn", "arn:aws:s3:::my-bucket", "arn:aws:rds", ""])("should return null for invalid ARN: %s", (invalidArn) => { + const region = gdbRegionUtils.getRegionFromClusterArn(invalidArn); + expect(region).toBeNull(); + }); + }); +}); diff --git a/tests/unit/iam_authentication_plugin.test.ts b/tests/unit/iam_authentication_plugin.test.ts index c6d384d4..f2ee620f 100644 --- a/tests/unit/iam_authentication_plugin.test.ts +++ b/tests/unit/iam_authentication_plugin.test.ts @@ -21,7 +21,7 @@ import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availabili import { AwsClient } from "../../common/lib/aws_client"; import { WrapperProperties } from "../../common/lib/wrapper_property"; import fetch from "node-fetch"; -import { anything, instance, mock, spy, when } from "ts-mockito"; +import { anything, instance, mock, reset, spy, when } from "ts-mockito"; import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; @@ -58,22 +58,11 @@ const props = new Map(); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockClient: AwsClient = mock(AwsClient); -const spyIamAuthUtils = spy(IamAuthUtils); class IamAuthenticationPluginTestClass extends IamAuthenticationPlugin { put(key: string, token: TokenInfo) { IamAuthenticationPlugin.tokenCache.set(key, token); } - - public async generateAuthenticationToken( - hostInfo: HostInfo, - props: Map, - hostname: string, - port: number, - region: string - ): Promise { - return Promise.resolve(GENERATED_TOKEN); - } } async function testGenerateToken(info: HostInfo, plugin: IamAuthenticationPluginTestClass) { @@ -100,8 +89,11 @@ async function testToken(info: HostInfo, plugin: IamAuthenticationPlugin) { } describe("testIamAuth", () => { + let spyIamAuthUtils: IamAuthUtils; + beforeEach(() => { PluginManager.releaseResources(); + spyIamAuthUtils = spy(new IamAuthUtils()); props.clear(); props.set(WrapperProperties.USER.name, "postgresqlUser"); @@ -110,15 +102,20 @@ describe("testIamAuth", () => { when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( GENERATED_TOKEN ); }); + afterEach(() => { + reset(spyIamAuthUtils); + }); + it("testPostgresConnectValidTokenInCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(PG_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO, plugin); }); @@ -129,7 +126,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_MYSQL_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(MYSQL_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(MYSQL_HOST_INFO, plugin); @@ -139,7 +136,7 @@ describe("testIamAuth", () => { props.set(WrapperProperties.IAM_DEFAULT_PORT.name, 0); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${PG_HOST_INFO_WITH_PORT.port}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); @@ -151,7 +148,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${DEFAULT_PG_PORT}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); @@ -160,7 +157,7 @@ describe("testIamAuth", () => { it("testConnectExpiredTokenInCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(PG_CACHE_KEY, new TokenInfo(TEST_TOKEN, Date.now() - 300000)); await testGenerateToken(PG_HOST_INFO, plugin); @@ -168,13 +165,13 @@ describe("testIamAuth", () => { it("testConnectEmptyCache", async () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); await testGenerateToken(PG_HOST_INFO, plugin); }); it("testConnectWithSpecifiedPort", async () => { const cacheKeyWithSpecifiedPort: string = "us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:1234:postgresqlUser"; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithSpecifiedPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_PORT, plugin); @@ -185,7 +182,7 @@ describe("testIamAuth", () => { props.set(WrapperProperties.IAM_DEFAULT_PORT.name, iamDefaultPort); const cacheKeyWithNewPort: string = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${iamDefaultPort}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewPort, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_PORT, plugin); @@ -196,7 +193,7 @@ describe("testIamAuth", () => { when(mockClient.defaultPort).thenReturn(DEFAULT_PG_PORT); const cacheKeyWithNewRegion: string = `us-west-1:pg.testdb.us-west-1.rds.amazonaws.com:${DEFAULT_PG_PORT}:postgresqlUser`; - const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService)); + const plugin = new IamAuthenticationPluginTestClass(instance(mockPluginService), instance(spyIamAuthUtils)); plugin.put(cacheKeyWithNewRegion, new TokenInfo(TEST_TOKEN, Date.now() + 300000)); await testToken(PG_HOST_INFO_WITH_REGION, plugin); diff --git a/tests/unit/okta_auth_plugin.test.ts b/tests/unit/okta_auth_plugin.test.ts deleted file mode 100644 index 59a3c281..00000000 --- a/tests/unit/okta_auth_plugin.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { anything, instance, mock, spy, verify, when } from "ts-mockito"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; -import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; -import { HostInfo, PluginManager } from "../../common/lib"; -import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; - -import { OktaAuthPlugin } from "../../common/lib/plugins/federated_auth/okta_auth_plugin"; -import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; -import { jest } from "@jest/globals"; -import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; - -const defaultPort = 1234; -const host = "pg.testdb.us-east-2.rds.amazonaws.com"; -const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; -const hostInfo = new HostInfo(host, defaultPort); -const dbUser = "iamUser"; -const region = "us-east-2"; -const testToken = "someTestToken"; -const testTokenInfo = new TokenInfo(testToken, Date.now() + 300000); - -const mockPluginService = mock(PluginServiceImpl); -const mockDialect = mock(); -const mockDialectInstance = instance(mockDialect); -const testCredentials = { - accessKeyId: "foo", - secretAccessKey: "bar", - sessionToken: "baz" -}; -const spyIamUtils = spy(IamAuthUtils); -const mockCredentialsProviderFactory = mock(); -const mockConnectFunc = jest.fn(() => { - return Promise.resolve(mock(PgClientWrapper)); -}); - -describe("oktaAuthTest", () => { - let spyPlugin: OktaAuthPlugin; - let props: Map; - - beforeEach(() => { - when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - when(mockDialect.getDefaultPort()).thenReturn(defaultPort); - when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(testCredentials); - props = new Map(); - WrapperProperties.PLUGINS.set(props, "okta"); - WrapperProperties.DB_USER.set(props, dbUser); - spyPlugin = spy(new OktaAuthPlugin(instance(mockPluginService), instance(mockCredentialsProviderFactory))); - }); - - afterEach(async () => { - await PluginManager.releaseResources(); - }); - - it("testCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, false, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testExpiredCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - - const someExpiredToken = "someExpiredToken"; - const expiredTokenInfo = new TokenInfo(someExpiredToken, Date.now() - 300000); - - OktaAuthPlugin["tokenCache"].set(key, expiredTokenInfo); - - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, false, mockConnectFunc); - - verify( - spyIamUtils.generateAuthenticationToken(hostInfo.host, defaultPort, region, dbUser, testCredentials, instance(mockPluginService)) - ).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testNoCachedToken", async () => { - const spyPluginInstance = instance(spyPlugin); - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - verify( - spyIamUtils.generateAuthenticationToken(hostInfo.host, defaultPort, region, dbUser, testCredentials, instance(mockPluginService)) - ).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testSpecifiedIamHostPortRegion", async () => { - const spyPluginInstance = instance(spyPlugin); - const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; - const expectedPort = 9876; - const expectedRegion = "us-west-2"; - - WrapperProperties.IAM_HOST.set(props, expectedHost); - WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); - WrapperProperties.IAM_REGION.set(props, expectedRegion); - - const key = `us-west-2:pg.testdb.us-west-2.rds.amazonaws.com:${expectedPort}:iamUser`; - - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); - - it("testIdpCredentialsFallback", async () => { - const spyPluginInstance = instance(spyPlugin); - const expectedUser = "expectedUser"; - const expectedPassword = "expectedPassword"; - - WrapperProperties.USER.set(props, expectedUser); - WrapperProperties.PASSWORD.set(props, expectedPassword); - - const key = `us-east-2:pg.testdb.us-east-2.rds.amazonaws.com:${defaultPort}:iamUser`; - OktaAuthPlugin["tokenCache"].set(key, testTokenInfo); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - expect(expectedUser).toBe(WrapperProperties.IDP_USERNAME.get(props)); - expect(expectedPassword).toBe(WrapperProperties.IDP_PASSWORD.get(props)); - }); - - it("testUsingIamHost", async () => { - WrapperProperties.IAM_HOST.set(props, iamHost); - const spyPluginInstance = instance(spyPlugin); - when(spyIamUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); - - await spyPluginInstance.connect(hostInfo, props, true, mockConnectFunc); - - verify(spyIamUtils.generateAuthenticationToken(iamHost, defaultPort, region, dbUser, testCredentials, instance(mockPluginService))).called(); - expect(dbUser).toBe(WrapperProperties.USER.get(props)); - expect(testToken).toBe(WrapperProperties.PASSWORD.get(props)); - }); -}); diff --git a/tests/unit/region_utils.test.ts b/tests/unit/region_utils.test.ts new file mode 100644 index 00000000..7be55576 --- /dev/null +++ b/tests/unit/region_utils.test.ts @@ -0,0 +1,65 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RegionUtils } from "../../common/lib/utils/region_utils"; + +describe("RegionUtils", () => { + let regionUtils: RegionUtils; + + beforeEach(() => { + regionUtils = new RegionUtils(); + }); + + describe("getRegion", () => { + it("should return null when props is invalid", async () => { + let result = await regionUtils.getRegion("region", undefined, undefined); + expect(result).toBeNull(); + + result = await regionUtils.getRegion("region", undefined, null); + expect(result).toBeNull(); + + const props = new Map(); + result = await regionUtils.getRegion("region", undefined, props); + expect(result).toBeNull(); + }); + + it.each([ + ["undefinedRegionKey", undefined], + ["nullRegionKey", null], + ["emptyRegionKey", ""] + ])("should return null when region key is invalid", async (regionKey: string, regionKeyVal: any) => { + const props = new Map(); + let result = await regionUtils.getRegion("region", undefined, props); + expect(result).toBeNull(); + + props.set(regionKey, regionKeyVal); + result = await regionUtils.getRegion(regionKey, undefined, props); + expect(result).toBeNull(); + }); + }); + + describe("getRegionFromRegionString", () => { + it.each([undefined, null, ""])("should return null for invalid regionString", (regionString: any) => { + const result = regionUtils.getRegionFromRegionString(regionString); + expect(result).toBeNull(); + }); + + it("should return region for valid region string", () => { + const result = regionUtils.getRegionFromRegionString("us-east-1"); + expect(result).toBe("us-east-1"); + }); + }); +}); diff --git a/tests/unit/saml_auth_plugin.test.ts b/tests/unit/saml_auth_plugin.test.ts new file mode 100644 index 00000000..76b55d24 --- /dev/null +++ b/tests/unit/saml_auth_plugin.test.ts @@ -0,0 +1,161 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostInfo, HostRole } from "../../common/lib"; +import { FederatedAuthPlugin } from "../../common/lib/plugins/federated_auth/federated_auth_plugin"; +import { OktaAuthPlugin } from "../../common/lib/plugins/federated_auth/okta_auth_plugin"; +import { BaseSamlAuthPlugin } from "../../common/lib/plugins/federated_auth/saml_auth_plugin"; +import { PluginServiceImpl } from "../../common/lib/plugin_service"; +import { IamAuthUtils, TokenInfo } from "../../common/lib/utils/iam_auth_utils"; +import { WrapperProperties } from "../../common/lib/wrapper_property"; +import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; +import { CredentialsProviderFactory } from "../../common/lib/plugins/federated_auth/credentials_provider_factory"; +import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; +import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; +import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; + +const testToken = "testToken"; +const defaultPort = 5432; +const dbUser = "iamUser"; +const expirationFiveMinutes = 5 * 60 * 1000; + +const host = "pg.testdb.us-east-2.rds.amazonaws.com"; +const iamHost = "pg-123.testdb.us-east-2.rds.amazonaws.com"; +const testTokenInfo = new TokenInfo(testToken, Date.now() + expirationFiveMinutes); +const hostInfo = new HostInfo(host, defaultPort, HostRole.WRITER); + +const mockDialect = mock(); +const mockDialectInstance = instance(mockDialect); +const mockPluginService = mock(PluginServiceImpl); +const mockCredentialsProviderFactory = mock(); +const mockClientWrapper = mock(PgClientWrapper); +const testCredentials = { + accessKeyId: "foo", + secretAccessKey: "bar", + sessionToken: "baz" +}; +const mockConnectFunc = () => Promise.resolve(instance(mockClientWrapper)); + +describe.each([ + { pluginName: "federatedAuth", PluginClass: FederatedAuthPlugin }, + { pluginName: "okta", PluginClass: OktaAuthPlugin } +])("$pluginName plugin tests", ({ pluginName, PluginClass }) => { + let plugin: FederatedAuthPlugin | OktaAuthPlugin; + let spyIamAuthUtils: IamAuthUtils; + let props: Map; + + beforeEach(() => { + spyIamAuthUtils = spy(new IamAuthUtils()); + + when(mockPluginService.getDialect()).thenReturn(mockDialectInstance); + when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); + when(mockDialect.getDefaultPort()).thenReturn(defaultPort); + when(mockCredentialsProviderFactory.getAwsCredentialsProvider(anything(), anything(), anything())).thenResolve(testCredentials); + + props = new Map(); + WrapperProperties.PLUGINS.set(props, pluginName); + WrapperProperties.DB_USER.set(props, dbUser); + + plugin = new PluginClass(instance(mockPluginService), instance(mockCredentialsProviderFactory), instance(spyIamAuthUtils)); + }); + + afterEach(() => { + BaseSamlAuthPlugin.releaseResources(); + reset(spyIamAuthUtils); + }); + + it("testCachedToken", async () => { + const pgCacheKey = `us-east-2:${host}:${defaultPort}:${dbUser}`; + + BaseSamlAuthPlugin["tokenCache"].set(pgCacheKey, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testExpiredCachedToken", async () => { + const key = `us-east-2:${host}:${defaultPort}:${dbUser}`; + const expiredToken = "expiredToken"; + const expiredTokenInfo = new TokenInfo(expiredToken, Date.now() - 300000); + + BaseSamlAuthPlugin["tokenCache"].set(key, expiredTokenInfo); + + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testNoCachedToken", async () => { + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testSpecifiedIamHostPortRegion", async () => { + const expectedHost = "pg.testdb.us-west-2.rds.amazonaws.com"; + const expectedPort = 9876; + const expectedRegion = "us-west-2"; + + WrapperProperties.IAM_HOST.set(props, expectedHost); + WrapperProperties.IAM_DEFAULT_PORT.set(props, expectedPort); + WrapperProperties.IAM_REGION.set(props, expectedRegion); + + const key = `${expectedRegion}:${expectedHost}:${expectedPort}:${dbUser}`; + BaseSamlAuthPlugin["tokenCache"].set(key, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + }); + + it("testIdpCredentialsFallback", async () => { + const expectedUser = "expectedUser"; + const expectedPassword = "expectedPassword"; + WrapperProperties.USER.set(props, expectedUser); + WrapperProperties.PASSWORD.set(props, expectedPassword); + + const key = `us-east-2:${host}:${defaultPort}:${dbUser}`; + BaseSamlAuthPlugin["tokenCache"].set(key, testTokenInfo); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + expect(WrapperProperties.IDP_USERNAME.get(props)).toBe(expectedUser); + expect(WrapperProperties.IDP_PASSWORD.get(props)).toBe(expectedPassword); + }); + + it("testUsingIamHost", async () => { + WrapperProperties.IAM_HOST.set(props, iamHost); + + when(spyIamAuthUtils.generateAuthenticationToken(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve(testToken); + + await plugin.connect(hostInfo, props, true, mockConnectFunc); + + expect(WrapperProperties.USER.get(props)).toBe(dbUser); + expect(WrapperProperties.PASSWORD.get(props)).toBe(testToken); + verify(spyIamAuthUtils.generateAuthenticationToken(iamHost, anything(), anything(), anything(), anything(), anything())).once(); + }); +}); From 5efef3bd5a2d13c95278afcf99f126396908be84 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Wed, 11 Mar 2026 02:29:30 -0700 Subject: [PATCH 12/20] refactor: read write splitting (#610) --- common/lib/connection_info.ts | 35 ++ common/lib/connection_provider.ts | 4 +- common/lib/driver_connection_provider.ts | 5 +- .../internal_pooled_connection_provider.ts | 7 +- common/lib/pg_client_wrapper.ts | 7 +- common/lib/plugin_manager.ts | 3 +- common/lib/plugin_service.ts | 14 + common/lib/plugins/default_plugin.ts | 8 +- .../lib/plugins/failover/failover_plugin.ts | 9 +- .../abstract_read_write_splitting_plugin.ts | 311 +++++++++++++++ .../read_write_splitting_plugin.ts | 372 ++++-------------- .../read_write_splitting_plugin_factory.ts | 28 +- common/lib/utils/messages.ts | 3 +- common/lib/utils/utils.ts | 8 + common/lib/wrapper_property.ts | 7 + .../tests/read_write_splitting.test.ts | 1 + tests/unit/read_write_splitting.test.ts | 72 ++-- 17 files changed, 544 insertions(+), 350 deletions(-) create mode 100644 common/lib/connection_info.ts create mode 100644 common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts diff --git a/common/lib/connection_info.ts b/common/lib/connection_info.ts new file mode 100644 index 00000000..2d195120 --- /dev/null +++ b/common/lib/connection_info.ts @@ -0,0 +1,35 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { ClientWrapper } from "./client_wrapper"; + +export class ConnectionInfo { + private readonly _client: ClientWrapper; + private readonly _isPooled: boolean; + + constructor(client: ClientWrapper, isPooled: boolean) { + this._client = client; + this._isPooled = isPooled; + } + + get client(): ClientWrapper { + return this._client; + } + + get isPooled(): boolean { + return this._isPooled; + } +} diff --git a/common/lib/connection_provider.ts b/common/lib/connection_provider.ts index 8da16f5d..9f60b31b 100644 --- a/common/lib/connection_provider.ts +++ b/common/lib/connection_provider.ts @@ -17,10 +17,10 @@ import { HostRole } from "./host_role"; import { HostInfo } from "./host_info"; import { PluginService } from "./plugin_service"; -import { ClientWrapper } from "./client_wrapper"; +import { ConnectionInfo } from "./connection_info"; export interface ConnectionProvider { - connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise; + connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise; acceptsUrl(hostInfo: HostInfo, props: Map): boolean; acceptsStrategy(role: HostRole, strategy: string): boolean; getHostInfoByStrategy(hosts: HostInfo[], role: HostRole, strategy: string, props?: Map): HostInfo; diff --git a/common/lib/driver_connection_provider.ts b/common/lib/driver_connection_provider.ts index 7affca52..6a4cd034 100644 --- a/common/lib/driver_connection_provider.ts +++ b/common/lib/driver_connection_provider.ts @@ -30,6 +30,7 @@ import { logger } from "../logutils"; import { ClientWrapper } from "./client_wrapper"; import { RoundRobinHostSelector } from "./round_robin_host_selector"; import { DriverDialect } from "./driver_dialect/driver_dialect"; +import { ConnectionInfo } from "./connection_info"; export class DriverConnectionProvider implements ConnectionProvider { private static readonly acceptedStrategies: Map = new Map([ @@ -46,7 +47,7 @@ export class DriverConnectionProvider implements ConnectionProvider { return true; } - async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { + async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { let resultTargetClient; const resultProps = new Map(props); resultProps.set(WrapperProperties.HOST.name, hostInfo.host); @@ -92,7 +93,7 @@ export class DriverConnectionProvider implements ConnectionProvider { resultTargetClient = driverDialect.connect(hostInfo, resultProps); } pluginService.attachErrorListener(resultTargetClient); - return resultTargetClient; + return new ConnectionInfo(resultTargetClient, false); } getHostInfoByStrategy(hosts: HostInfo[], role: HostRole, strategy: string, props?: Map): HostInfo { diff --git a/common/lib/internal_pooled_connection_provider.ts b/common/lib/internal_pooled_connection_provider.ts index 300442ad..aa9846aa 100644 --- a/common/lib/internal_pooled_connection_provider.ts +++ b/common/lib/internal_pooled_connection_provider.ts @@ -39,6 +39,7 @@ import { LeastConnectionsHostSelector } from "./least_connections_host_selector" import { PoolClientWrapper } from "./pool_client_wrapper"; import { logger } from "../logutils"; import { SlidingExpirationCacheWithCleanupTask } from "./utils/sliding_expiration_cache_with_cleanup_task"; +import { ConnectionInfo } from "./connection_info"; export class InternalPooledConnectionProvider implements PooledConnectionProvider, CanReleaseResources { static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes @@ -79,7 +80,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide return RdsUrlType.RDS_INSTANCE === urlType; } - async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { + async connect(hostInfo: HostInfo, pluginService: PluginService, props: Map): Promise { const resultProps = new Map(props); resultProps.set(WrapperProperties.HOST.name, hostInfo.host); if (hostInfo.isPortSpecified()) { @@ -122,7 +123,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide const poolClient = await this.getPoolConnection(connectionHostInfo, props); pluginService.attachErrorListener(poolClient); - return poolClient; + return new ConnectionInfo(poolClient, true); } async getPoolConnection(hostInfo: HostInfo, props: Map) { @@ -179,7 +180,7 @@ export class InternalPooledConnectionProvider implements PooledConnectionProvide return; } - const connections = Array.from(InternalPooledConnectionProvider.databasePools.entries).map(([v, k]) => [v, k.item.constructor.name]); + const connections = Array.from(InternalPooledConnectionProvider.databasePools.entries).map(([v, k]) => [v, k.get().constructor.name]); logger.debug(`Internal Pooled Connections: [\r\n${connections.join("\r\n")}\r\n]`); } diff --git a/common/lib/pg_client_wrapper.ts b/common/lib/pg_client_wrapper.ts index 99e3ec76..71af129c 100644 --- a/common/lib/pg_client_wrapper.ts +++ b/common/lib/pg_client_wrapper.ts @@ -17,6 +17,7 @@ import { ClientWrapper } from "./client_wrapper"; import { HostInfo } from "./host_info"; import { uniqueId } from "../logutils"; +import { ClientUtils } from "./utils/client_utils"; /* This an internal wrapper class for a target community driver client created by the NodePostgresPgDriverDialect. @@ -45,8 +46,8 @@ export class PgClientWrapper implements ClientWrapper { return this.client?.query(config, values, callback); } - queryWithTimeout(sql: string): Promise { - return this.client?.queryWithTimeout(this.client.query(sql), this.properties); + async queryWithTimeout(sql: string): Promise { + return await ClientUtils.queryWithTimeout(this.client.query(sql), this.properties); } end(): Promise { @@ -59,7 +60,7 @@ export class PgClientWrapper implements ClientWrapper { async abort(): Promise { try { - return await this.end(); + return await ClientUtils.queryWithTimeout(this.end(), this.properties); } catch (error: any) { // Ignore } diff --git a/common/lib/plugin_manager.ts b/common/lib/plugin_manager.ts index f66e65ba..410b4cf9 100644 --- a/common/lib/plugin_manager.ts +++ b/common/lib/plugin_manager.ts @@ -32,7 +32,7 @@ import { TelemetryTraceLevel } from "./utils/telemetry/telemetry_trace_level"; import { ConnectionProvider } from "./connection_provider"; import { ConnectionPluginFactory } from "./plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { BaseSamlAuthPlugin } from "./plugins/federated_auth/saml_auth_plugin"; +import { CoreServicesContainer } from "./utils/core_services_container"; type PluginFunc = (plugin: ConnectionPlugin, targetFunc: () => Promise) => Promise; @@ -396,6 +396,7 @@ export class PluginManager { } PluginManager.STRATEGY_PLUGIN_CHAIN_CACHE.clear(); + CoreServicesContainer.releaseResources(); PluginManager.PLUGINS = new Set(); } diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 5421b8bb..18144455 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -150,6 +150,10 @@ export interface PluginService extends ErrorHandler { getStatus(clazz: any, key: string): T; isPluginInUse(plugin: any): boolean; + + isPooledClient(): boolean; + + setIsPooledClient(isPooledClient: boolean): void; } export class PluginServiceImpl implements PluginService, HostListProviderService { @@ -172,6 +176,8 @@ export class PluginServiceImpl implements PluginService, HostListProviderService protected static readonly statusesExpiringCache: CacheMap = new CacheMap(); protected static readonly DEFAULT_STATUS_CACHE_EXPIRE_NANO: number = 3_600_000_000_000; // 60 minutes + protected _isPooledClient: boolean = false; + constructor( container: PluginServiceManagerContainer, client: AwsClient, @@ -782,4 +788,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService isPluginInUse(plugin: any) { return this.pluginServiceManagerContainer.pluginManager!.isPluginInUse(plugin); } + + isPooledClient(): boolean { + return this._isPooledClient; + } + + setIsPooledClient(isPooledClient: boolean): void { + this._isPooledClient = isPooledClient; + } } diff --git a/common/lib/plugins/default_plugin.ts b/common/lib/plugins/default_plugin.ts index ddfcf5f9..3af9b3fc 100644 --- a/common/lib/plugins/default_plugin.ts +++ b/common/lib/plugins/default_plugin.ts @@ -29,6 +29,7 @@ import { AwsWrapperError } from "../utils/errors"; import { HostAvailability } from "../host_availability/host_availability"; import { ClientWrapper } from "../client_wrapper"; import { TelemetryTraceLevel } from "../utils/telemetry/telemetry_trace_level"; +import { ConnectionInfo } from "../connection_info"; export class DefaultPlugin extends AbstractConnectionPlugin { id: string = uniqueId("_defaultPlugin"); @@ -79,10 +80,11 @@ export class DefaultPlugin extends AbstractConnectionPlugin { TelemetryTraceLevel.NESTED ); - const result = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); + const result: ConnectionInfo = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.AVAILABLE); - await this.pluginService.updateDialect(result); - return result; + this.pluginService.setIsPooledClient(result.isPooled); + await this.pluginService.updateDialect(result.client); + return result.client; } override async execute(methodName: string, methodFunc: () => Promise): Promise { diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index e6d7f4d7..4145e519 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -17,14 +17,14 @@ import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { logger } from "../../../logutils"; import { - HostInfo, AwsWrapperError, FailoverFailedError, FailoverSuccessError, - TransactionResolutionUnknownError, - UnavailableHostError, + HostAvailability, + HostInfo, HostRole, - HostAvailability + TransactionResolutionUnknownError, + UnavailableHostError } from "../../"; import { PluginService } from "../../plugin_service"; import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; @@ -255,6 +255,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { // Verify there aren't any unexpected error emitted while the connection was idle. if (this.pluginService.hasNetworkError()) { // Throw the unexpected error directly to be handled. + throw this.pluginService.getUnexpectedError(); } diff --git a/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts new file mode 100644 index 00000000..65482211 --- /dev/null +++ b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts @@ -0,0 +1,311 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { CanReleaseResources } from "../../can_release_resources"; +import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; +import { ClientWrapper } from "../../client_wrapper"; +import { PluginService } from "../../plugin_service"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { HostInfo } from "../../host_info"; +import { HostChangeOptions } from "../../host_change_options"; +import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; +import { Messages } from "../../utils/messages"; +import { logger } from "../../../logutils"; +import { HostRole } from "../../host_role"; +import { SqlMethodUtils } from "../../utils/sql_method_utils"; +import { FailoverError } from "../../utils/errors"; +import { WrapperProperties } from "../../wrapper_property"; +import { convertMsToNanos, getTimeInNanos, logAndThrowError } from "../../utils/utils"; +import { CacheItem } from "../../utils/cache_map"; + +export abstract class AbstractReadWriteSplittingPlugin extends AbstractConnectionPlugin implements CanReleaseResources { + private static readonly subscribedMethods: Set = new Set(["initHostProvider", "connect", "notifyConnectionChanged", "query"]); + + protected _hostListProviderService: HostListProviderService | undefined; + protected pluginService: PluginService; + protected readonly _properties: Map; + protected readerHostInfo?: HostInfo = undefined; + protected writerHostInfo?: HostInfo = undefined; + protected isReaderClientFromInternalPool: boolean = false; + protected isWriterClientFromInternalPool: boolean = false; + + protected writerTargetClient?: ClientWrapper = undefined; + protected readerCacheItem?: CacheItem = undefined; + protected readonly readerSelectorStrategy: string = ""; + + private _inReadWriteSplit = false; + + protected constructor(pluginService: PluginService, properties: Map) { + super(); + this.pluginService = pluginService; + this._properties = properties; + this.readerSelectorStrategy = WrapperProperties.READER_HOST_SELECTOR_STRATEGY.get(properties); + } + + override getSubscribedMethods(): Set { + return AbstractReadWriteSplittingPlugin.subscribedMethods; + } + + override initHostProvider( + hostInfo: HostInfo, + props: Map, + hostListProviderService: HostListProviderService, + initHostProviderFunc: () => void + ) { + this._hostListProviderService = hostListProviderService; + initHostProviderFunc(); + } + + override async notifyConnectionChanged(changes: Set): Promise { + try { + await this.updateInternalClientInfo(); + } catch (e) { + // pass + } + if (this._inReadWriteSplit) { + return Promise.resolve(OldConnectionSuggestionAction.PRESERVE); + } + return Promise.resolve(OldConnectionSuggestionAction.NO_OPINION); + } + + async updateInternalClientInfo(): Promise { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + const currentHost = this.pluginService.getCurrentHostInfo(); + if (currentHost === null || currentTargetClient === null) { + return; + } + + if (this.shouldUpdateWriterClient(currentTargetClient, currentHost)) { + this.setWriterClient(currentTargetClient, currentHost); + } else if (this.shouldUpdateReaderClient(currentTargetClient, currentHost)) { + await this.setReaderClient(currentTargetClient, currentHost); + } + } + + setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo): void { + this.writerTargetClient = writerTargetClient; + this.writerHostInfo = writerHostInfo; + logger.debug(Messages.get("ReadWriteSplittingPlugin.setWriterClient", writerHostInfo.hostAndPort)); + } + + async setReaderClient(readerTargetClient: ClientWrapper | undefined, readerHost: HostInfo): Promise { + await this.closeReaderClientIfIdle(); + this.readerCacheItem = new CacheItem(readerTargetClient, this.getKeepAliveTimeout(this.isReaderClientFromInternalPool)); + this.readerHostInfo = readerHost; + logger.debug(Messages.get("ReadWriteSplittingPlugin.setReaderClient", readerHost.hostAndPort)); + } + + async switchClientIfRequired(readOnly: boolean) { + const currentClient = this.pluginService.getCurrentClient(); + if (!(await currentClient.isValid())) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyOnClosedClient", currentClient.targetClient?.id ?? "undefined client")); + } + + await this.refreshAndStoreTopology(currentClient.targetClient); + + const currentHost = this.pluginService.getCurrentHostInfo(); + if (currentHost == null) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.unavailableHostInfo")); + } else if (readOnly) { + if (!this.pluginService.isInTransaction() && currentHost.role != HostRole.READER) { + try { + await this.switchToReaderTargetClient(); + } catch (error: any) { + if (!(await currentClient.isValid())) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToReader", error.message)); + } + logger.warn(Messages.get("ReadWriteSplittingPlugin.fallbackToWriter", currentHost.url)); + } + } + } else if (currentHost.role != HostRole.WRITER) { + if (this.pluginService.isInTransaction()) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyFalseInTransaction")); + } + try { + await this.switchToWriterTargetClient(); + } catch (error: any) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToWriter", error.message)); + } + } + } + + override async execute(methodName: string, executeFunc: () => Promise, methodArgs: any): Promise { + const statement = SqlMethodUtils.parseMethodArgs(methodArgs, this.pluginService.getDriverDialect()); + const statements = SqlMethodUtils.parseMultiStatementQueries(statement); + + const updateReadOnly: boolean | undefined = SqlMethodUtils.doesSetReadOnly(statements, this.pluginService.getDialect()); + if (updateReadOnly !== undefined) { + try { + await this.switchClientIfRequired(updateReadOnly); + } catch (error) { + await this.closeIdleClients(); + throw error; + } + } + + try { + return await executeFunc(); + } catch (error: any) { + if (error instanceof FailoverError) { + logger.debug(Messages.get("ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand", methodName)); + await this.closeIdleClients(); + } else { + logger.debug(Messages.get("ReadWriteSplittingPlugin.errorWhileExecutingCommand", methodName, error.message)); + } + + throw error; + } + } + + async switchCurrentTargetClientTo(newTargetClient: ClientWrapper | undefined, newClientHost: HostInfo | undefined) { + if (!newTargetClient || !newClientHost) { + return; + } + + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + + if (currentTargetClient === newTargetClient) { + return; + } + + try { + await this.pluginService.setCurrentClient(newTargetClient, newClientHost); + logger.debug(Messages.get("ReadWriteSplittingPlugin.settingCurrentClient", newTargetClient.id, newClientHost.hostAndPort)); + } catch (error) { + // pass + } + } + + async switchToWriterTargetClient() { + const currentHost = this.pluginService.getCurrentHostInfo(); + const currentClient = this.pluginService.getCurrentClient(); + if (this.isWriter(currentHost) && (await this.isTargetClientUsable(currentClient.targetClient))) { + // Already connected to writer. + return; + } + this._inReadWriteSplit = true; + if (!(await this.isTargetClientUsable(this.writerTargetClient))) { + await this.initializeWriterClient(); + } else { + await this.switchCurrentTargetClientTo(this.writerTargetClient, this.writerHostInfo); + } + + if (this.isReaderClientFromInternalPool) { + await this.closeReaderClientIfIdle(); + } + + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromReaderToWriter", this.writerHostInfo?.hostAndPort ?? "unknown")); + } + + async switchToReaderTargetClient() { + const currentHost = this.pluginService.getCurrentHostInfo(); + const currentClient = this.pluginService.getCurrentClient(); + if (this.isReader(currentHost) && currentClient) { + // Already connected to reader. + return; + } + + await this.closeReaderIfNecessary(); + + this._inReadWriteSplit = true; + if (this.readerCacheItem == null || !(await this.isTargetClientUsable(this.readerCacheItem.get()))) { + await this.initializeReaderClient(); + } else { + try { + await this.switchCurrentTargetClientTo(this.readerCacheItem.get(), this.readerHostInfo); + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this.readerHostInfo?.hostAndPort ?? "unknown")); + } catch (error: any) { + logger.debug( + Messages.get( + "ReadWriteSplittingPlugin.errorSwitchingToCachedReader", + this.readerHostInfo?.hostAndPort ?? "undefined reader host", + error.message + ) + ); + await this.closeReaderClientIfIdle(); + await this.initializeReaderClient(); + } + } + if (this.isWriterClientFromInternalPool) { + await this.closeWriterClientIfIdle(); + } + } + + async isTargetClientUsable(targetClient: ClientWrapper | undefined): Promise { + if (!targetClient) { + return Promise.resolve(false); + } + return await this.pluginService.isClientValid(targetClient); + } + + async closeWriterClientIfIdle() { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + if (this.writerTargetClient == null || this.writerTargetClient === currentTargetClient) { + return; + } + + try { + await this.pluginService.abortTargetClient(this.writerTargetClient); + } catch (error) { + // ignore + } + this.writerTargetClient = undefined; + } + + async closeReaderClientIfIdle(): Promise { + const currentTargetClient = this.pluginService.getCurrentClient().targetClient; + const readerClient = this.readerCacheItem?.get(true); + if (readerClient == null || readerClient === currentTargetClient) { + return; + } + + try { + await this.pluginService.abortTargetClient(readerClient); + } catch (error) { + // ignore + } + this.readerCacheItem = undefined; + this.readerHostInfo = undefined; + } + + async closeIdleClients() { + logger.debug(Messages.get("ReadWriteSplittingPlugin.closingInternalClients")); + await this.closeReaderClientIfIdle(); + await this.closeWriterClientIfIdle(); + } + + protected getKeepAliveTimeout(isPooledClient: boolean): bigint { + if (isPooledClient) { + return BigInt(0); + } + const keepAliveMs = WrapperProperties.CACHED_READER_KEEP_ALIVE_TIMEOUT.get(this._properties); + + return keepAliveMs > 0 ? getTimeInNanos() + convertMsToNanos(keepAliveMs) : BigInt(0); + } + + async releaseResources() { + await this.closeIdleClients(); + } + + protected abstract shouldUpdateReaderClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean; + protected abstract shouldUpdateWriterClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean; + protected abstract isWriter(currentHost: HostInfo): boolean; + protected abstract isReader(currentHost: HostInfo): boolean; + protected abstract refreshAndStoreTopology(currentClient: ClientWrapper | undefined): Promise; + protected abstract initializeWriterClient(): Promise; + protected abstract initializeReaderClient(): Promise; + protected abstract closeReaderIfNecessary(): Promise; +} diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts index c48fd009..397d5dfb 100644 --- a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts @@ -1,51 +1,32 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { - HostInfo, - FailoverError, - HostRole -} from "../../index"; + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { HostInfo, HostRole } from "../../index"; import { PluginService } from "../../plugin_service"; import { HostListProviderService } from "../../host_list_provider_service"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { HostChangeOptions } from "../../host_change_options"; -import { WrapperProperties } from "../../wrapper_property"; import { Messages } from "../../utils/messages"; -import { logger } from "../../../logutils"; -import { SqlMethodUtils } from "../../utils/sql_method_utils"; import { ClientWrapper } from "../../client_wrapper"; -import { getWriter, logAndThrowError } from "../../utils/utils"; -import { CanReleaseResources } from "../../can_release_resources"; -import { PoolClientWrapper } from "../../pool_client_wrapper"; - -export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - private static readonly subscribedMethods: Set = new Set(["initHostProvider", "connect", "notifyConnectionChanged", "query"]); - private readonly readerSelectorStrategy: string = ""; +import { containsHostAndPort, getWriter, logAndThrowError, logTopology } from "../../utils/utils"; +import { AbstractReadWriteSplittingPlugin } from "./abstract_read_write_splitting_plugin"; +import { WrapperProperties } from "../../wrapper_property"; +import { logger } from "../../../logutils"; +import { CacheItem } from "../../utils/cache_map"; - private _hostListProviderService: HostListProviderService | undefined; - private pluginService: PluginService; - private readonly _properties: Map; - private _readerHostInfo?: HostInfo = undefined; - private isReaderClientFromInternalPool: boolean = false; - private isWriterClientFromInternalPool: boolean = false; - private _inReadWriteSplit = false; - writerTargetClient: ClientWrapper | undefined; - readerTargetClient: ClientWrapper | undefined; +export class ReadWriteSplittingPlugin extends AbstractReadWriteSplittingPlugin { + protected hosts: HostInfo[] = []; constructor(pluginService: PluginService, properties: Map); constructor( @@ -62,53 +43,10 @@ export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implement writerClient?: ClientWrapper, readerClient?: ClientWrapper ) { - super(); - this.pluginService = pluginService; - this._properties = properties; - this.readerSelectorStrategy = WrapperProperties.READER_HOST_SELECTOR_STRATEGY.get(properties); + super(pluginService, properties); this._hostListProviderService = hostListProviderService; this.writerTargetClient = writerClient; - this.readerTargetClient = readerClient; - } - - override getSubscribedMethods(): Set { - return ReadWriteSplittingPlugin.subscribedMethods; - } - - override initHostProvider( - hostInfo: HostInfo, - props: Map, - hostListProviderService: HostListProviderService, - initHostProviderFunc: () => void - ) { - this._hostListProviderService = hostListProviderService; - initHostProviderFunc(); - } - - override notifyConnectionChanged(changes: Set): Promise { - try { - this.updateInternalClientInfo(); - } catch (e) { - // pass - } - if (this._inReadWriteSplit) { - return Promise.resolve(OldConnectionSuggestionAction.PRESERVE); - } - return Promise.resolve(OldConnectionSuggestionAction.NO_OPINION); - } - - updateInternalClientInfo(): void { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - const currentHost = this.pluginService.getCurrentHostInfo(); - if (currentHost === null || currentTargetClient === null) { - return; - } - - if (currentHost.role === HostRole.WRITER) { - this.setWriterClient(currentTargetClient, currentHost); - } else { - this.setReaderClient(currentTargetClient, currentHost); - } + this.readerCacheItem = new CacheItem(readerClient, BigInt(0)); } override async connect( @@ -143,151 +81,87 @@ export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implement return result; } - override async execute(methodName: string, executeFunc: () => Promise, methodArgs: any): Promise { - const statement = SqlMethodUtils.parseMethodArgs(methodArgs, this.pluginService.getDriverDialect()); - const statements = SqlMethodUtils.parseMultiStatementQueries(statement); + protected override isWriter(currentHost?: HostInfo): boolean { + return currentHost?.role === HostRole.WRITER; + } + + protected override isReader(currentHost?: HostInfo): boolean { + return currentHost?.role === HostRole.READER; + } - const updateReadOnly: boolean | undefined = SqlMethodUtils.doesSetReadOnly(statements, this.pluginService.getDialect()); - if (updateReadOnly !== undefined) { + protected override async refreshAndStoreTopology(currentClient: ClientWrapper | undefined): Promise { + if (await this.pluginService.isClientValid(currentClient)) { try { - await this.switchClientIfRequired(updateReadOnly); - } catch (error) { - await this.closeIdleClients(); - throw error; + await this.pluginService.refreshHostList(); + } catch { + // pass } } - try { - return await executeFunc(); - } catch (error: any) { - if (error instanceof FailoverError) { - logger.debug(Messages.get("ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand", methodName)); - await this.closeIdleClients(); - } else { - logger.debug(Messages.get("ReadWriteSplittingPlugin.errorWhileExecutingCommand", methodName, error.message)); - } - - throw error; + this.hosts = this.pluginService.getHosts(); + if (!this.hosts?.length) { + logAndThrowError(Messages.get("ReadWriteSplittingPlugin.emptyHostList")); } - } - setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo): void { - this.writerTargetClient = writerTargetClient; - logger.debug(Messages.get("ReadWriteSplittingPlugin.setWriterClient", writerHostInfo.url)); + this.writerHostInfo = getWriter(this.hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); } - setReaderClient(readerTargetClient: ClientWrapper | undefined, readerHost: HostInfo): void { - this.readerTargetClient = readerTargetClient; - this._readerHostInfo = readerHost; - logger.debug(Messages.get("ReadWriteSplittingPlugin.setReaderClient", readerHost.url)); + protected override async initializeWriterClient(): Promise { + const client: ClientWrapper = await this.connectToWriter(); + this.isWriterClientFromInternalPool = this.pluginService.isPooledClient(); + this.setWriterClient(client, this.writerHostInfo); + await this.switchCurrentTargetClientTo(this.writerTargetClient, this.writerHostInfo); } - async getNewWriterClient(writerHost: HostInfo) { - const props = new Map(this._properties); - props.set(WrapperProperties.HOST.name, writerHost.host); - try { - const copyProps = new Map(props); - WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.set(copyProps, false); - const targetClient = await this.pluginService.connect(writerHost, copyProps, this); - this.isWriterClientFromInternalPool = targetClient instanceof PoolClientWrapper; - this.setWriterClient(targetClient, writerHost); - await this.switchCurrentTargetClientTo(this.writerTargetClient, writerHost); - } catch (any) { - logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToWriter", writerHost.url)); + private async connectToWriter() { + if (!this.writerHostInfo) { + throw new Error(Messages.get("ReadWriteSplittingPlugin.noWriterFound")); } + const copyProps = new Map(this._properties); + copyProps.set(WrapperProperties.HOST.name, this.writerHostInfo.host); + return await this.pluginService.connect(this.writerHostInfo, copyProps, this); } - async switchClientIfRequired(readOnly: boolean) { - const currentClient = this.pluginService.getCurrentClient(); - if (!(await currentClient.isValid())) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyOnClosedClient", currentClient.targetClient?.id ?? "undefined client")); - } - try { - await this.pluginService.refreshHostList(); - } catch { - // pass - } - - const hosts: HostInfo[] = this.pluginService.getHosts(); - if (hosts == null || hosts.length === 0) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.emptyHostList")); - } - const currentHost = this.pluginService.getCurrentHostInfo(); - if (currentHost == null) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.unavailableHostInfo")); - } else if (readOnly) { - if (!this.pluginService.isInTransaction() && currentHost.role != HostRole.READER) { - try { - await this.switchToReaderTargetClient(hosts); - } catch (error: any) { - if (!(await currentClient.isValid())) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToReader", error.message)); - } - logger.warn(Messages.get("ReadWriteSplittingPlugin.fallbackToWriter", currentHost.url)); - } - } - } else if (currentHost.role != HostRole.WRITER) { - if (this.pluginService.isInTransaction()) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.setReadOnlyFalseInTransaction")); - } - try { - await this.switchToWriterTargetClient(hosts); - } catch (error: any) { - logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToWriter", error.message)); + protected override async initializeReaderClient() { + if (this.hosts?.length === 1) { + if (!(await this.isTargetClientUsable(this.writerTargetClient))) { + await this.initializeWriterClient(); } + logger.warn(Messages.get("ReadWriteSplittingPlugin.noReadersFound", this.writerHostInfo?.hostAndPort ?? "unknown")); + } else { + await this.getNewReaderClient(); + logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this.readerHostInfo?.hostAndPort ?? "unknown")); } } - async switchCurrentTargetClientTo(newTargetClient: ClientWrapper | undefined, newClientHost: HostInfo | undefined) { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - - if (currentTargetClient === newTargetClient) { - return; - } - if (newClientHost && newTargetClient) { - try { - await this.pluginService.setCurrentClient(newTargetClient, newClientHost); - logger.debug(Messages.get("ReadWriteSplittingPlugin.settingCurrentClient", newTargetClient.id, newClientHost.url)); - } catch (error) { - // pass - } - } + protected override shouldUpdateReaderClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean { + return this.isReader(host); } - async initializeReaderClient(hosts: HostInfo[]) { - if (hosts.length === 1) { - const writerHost = getWriter(hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); - if (writerHost) { - if (!(await this.isTargetClientUsable(this.writerTargetClient))) { - await this.getNewWriterClient(writerHost); - } - logger.warn(Messages.get("ReadWriteSplittingPlugin.noReadersFound", writerHost.url)); - } - } else { - await this.getNewReaderClient(); - } + protected override shouldUpdateWriterClient(currentClient: ClientWrapper | undefined, host: HostInfo): boolean { + return this.isWriter(host); } - async getNewReaderClient() { + protected async getNewReaderClient() { let targetClient = undefined; let readerHost: HostInfo | undefined = undefined; - const connectAttempts = this.pluginService.getHosts().length; + + const hostCandidates: HostInfo[] = this.getReaderHostCandidates(); + + const connectAttempts = hostCandidates.length * 2; for (let i = 0; i < connectAttempts; i++) { - const host = this.pluginService.getHostInfoByStrategy(HostRole.READER, this.readerSelectorStrategy); + const host = this.pluginService.getHostInfoByStrategy(HostRole.READER, this.readerSelectorStrategy, hostCandidates); if (host) { - const props = new Map(this._properties); - props.set(WrapperProperties.HOST.name, host.host); - try { - const copyProps = new Map(props); - WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.set(copyProps, false); + const copyProps = new Map(this._properties); + copyProps.set(WrapperProperties.HOST.name, host.host); targetClient = await this.pluginService.connect(host, copyProps, this); - this.isReaderClientFromInternalPool = targetClient instanceof PoolClientWrapper; + this.isReaderClientFromInternalPool = this.pluginService.isPooledClient(); readerHost = host; break; } catch (any) { - logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToReader", host.url)); + logger.warn(Messages.get("ReadWriteSplittingPlugin.failedToConnectToReader", host.hostAndPort)); } } } @@ -295,99 +169,19 @@ export class ReadWriteSplittingPlugin extends AbstractConnectionPlugin implement logAndThrowError(Messages.get("ReadWriteSplittingPlugin.noReadersAvailable")); return; } - logger.debug(Messages.get("ReadWriteSplittingPlugin.successfullyConnectedToReader", readerHost.url)); - this.setReaderClient(targetClient, readerHost); - await this.switchCurrentTargetClientTo(this.readerTargetClient, this._readerHostInfo); + logger.debug(Messages.get("ReadWriteSplittingPlugin.successfullyConnectedToReader", readerHost.hostAndPort)); + await this.setReaderClient(targetClient, readerHost); + await this.switchCurrentTargetClientTo(this.readerCacheItem?.get(), this.readerHostInfo); } - async switchToWriterTargetClient(hosts: HostInfo[]) { - const currentHost = this.pluginService.getCurrentHostInfo(); - const currentClient = this.pluginService.getCurrentClient(); - if (currentHost !== null && currentHost?.role === HostRole.WRITER && (await currentClient.isValid())) { - return; + protected override async closeReaderIfNecessary(): Promise { + if (this.readerHostInfo != null && !containsHostAndPort(this.hosts, this.readerHostInfo.hostAndPort)) { + logger.debug(Messages.get("ReadWriteSplittingPlugin.previousReaderNotAllowed", this.readerHostInfo.toString(), logTopology(this.hosts, ""))); + await this.closeReaderClientIfIdle(); } - this._inReadWriteSplit = true; - const writerHost = getWriter(hosts, Messages.get("ReadWriteSplittingPlugin.noWriterFound")); - if (!writerHost) { - return; - } - if (!(await this.isTargetClientUsable(this.writerTargetClient))) { - await this.getNewWriterClient(writerHost); - } else if (this.writerTargetClient) { - await this.switchCurrentTargetClientTo(this.writerTargetClient, writerHost); - } - - logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromReaderToWriter", writerHost.url)); - if (this.isReaderClientFromInternalPool) { - await this.closeTargetClientIfIdle(this.readerTargetClient); - } - } - - async switchToReaderTargetClient(hosts: HostInfo[]) { - const currentHost = this.pluginService.getCurrentHostInfo(); - const currentClient = this.pluginService.getCurrentClient(); - if (currentHost !== null && currentHost?.role === HostRole.READER && currentClient) { - return; - } - - if (this._readerHostInfo && !hosts.some((hostInfo: HostInfo) => hostInfo.host === this._readerHostInfo?.host)) { - // The old reader cannot be used anymore because it is no longer in the list of allowed hosts. - await this.closeTargetClientIfIdle(this.readerTargetClient); - } - - this._inReadWriteSplit = true; - if (!(await this.isTargetClientUsable(this.readerTargetClient))) { - await this.initializeReaderClient(hosts); - } else if (this.readerTargetClient != null && this._readerHostInfo != null) { - try { - await this.switchCurrentTargetClientTo(this.readerTargetClient, this._readerHostInfo); - logger.debug(Messages.get("ReadWriteSplittingPlugin.switchedFromWriterToReader", this._readerHostInfo.url)); - } catch (error: any) { - logger.debug(Messages.get("ReadWriteSplittingPlugin.errorSwitchingToCachedReader", this._readerHostInfo.url)); - await this.pluginService.abortTargetClient(this.readerTargetClient); - this.readerTargetClient = undefined; - this._readerHostInfo = undefined; - await this.initializeReaderClient(hosts); - } - } - if (this.isWriterClientFromInternalPool) { - await this.closeTargetClientIfIdle(this.writerTargetClient); - } - } - - async isTargetClientUsable(targetClient: ClientWrapper | undefined): Promise { - if (!targetClient) { - return Promise.resolve(false); - } - return await this.pluginService.isClientValid(targetClient); - } - - async closeTargetClientIfIdle(internalTargetClient: ClientWrapper | undefined) { - const currentTargetClient = this.pluginService.getCurrentClient().targetClient; - try { - if (internalTargetClient != null && internalTargetClient !== currentTargetClient && (await this.isTargetClientUsable(internalTargetClient))) { - await this.pluginService.abortTargetClient(internalTargetClient); - - if (internalTargetClient === this.writerTargetClient) { - this.writerTargetClient = undefined; - } - if (internalTargetClient === this.readerTargetClient) { - this.readerTargetClient = undefined; - this._readerHostInfo = undefined; - } - } - } catch (error) { - // ignore - } - } - - async closeIdleClients() { - logger.debug(Messages.get("ReadWriteSplittingPlugin.closingInternalClients")); - await this.closeTargetClientIfIdle(this.readerTargetClient); - await this.closeTargetClientIfIdle(this.writerTargetClient); } - async releaseResources() { - await this.closeIdleClients(); + protected getReaderHostCandidates(): HostInfo[] | undefined { + return this.pluginService.getHosts(); } } diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts index 881b59db..62485db1 100644 --- a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { ConnectionPluginFactory } from "../../plugin_factory"; import { PluginService } from "../../plugin_service"; diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index d7d72fe5..1f46e624 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -117,7 +117,7 @@ const MESSAGES: Record = { "AuroraStaleDnsHelper.staleDnsDetected": "Stale DNS data detected. Opening a connection to '%s'.", "ReadWriteSplittingPlugin.setReadOnlyOnClosedClient": "setReadOnly cannot be called on a closed client '%s'.", "ReadWriteSplittingPlugin.errorSwitchingToCachedReader": - "An error occurred while trying to switch to a cached reader client: '%s'. The driver will attempt to establish a new reader client.", + "An error occurred while trying to switch to a cached reader client: '%s'. Error message: '%s'. The driver will attempt to establish a new reader client.", "ReadWriteSplittingPlugin.errorSwitchingToReader": "An error occurred while trying to switch to a reader client: '%s'.", "ReadWriteSplittingPlugin.errorSwitchingToWriter": "An error occurred while trying to switch to a writer client: '%s'.", "ReadWriteSplittingPlugin.closingInternalClients": "Closing all internal clients except for the current one.", @@ -138,6 +138,7 @@ const MESSAGES: Record = { "ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand": "Detected a failover error while executing a command: '%s'", "ReadWriteSplittingPlugin.noReadersAvailable": "The plugin was unable to establish a reader client to any reader instance.", "ReadWriteSplittingPlugin.successfullyConnectedToReader": "Successfully connected to a new reader host: '%s'", + "ReadWriteSplittingPlugin.previousReaderNotAllowed": "The previous reader connection cannot be used because it is no longer in the list of allowed hosts. Previous reader: %s. Allowed hosts: %s", "ReadWriteSplittingPlugin.failedToConnectToReader": "Failed to connect to reader host: '%s'", "ReadWriteSplittingPlugin.unsupportedHostSelectorStrategy": "Unsupported host selection strategy '%s' specified in plugin configuration parameter 'readerHostSelectorStrategy'. Please visit the Read/Write Splitting Plugin documentation for all supported strategies.", diff --git a/common/lib/utils/utils.ts b/common/lib/utils/utils.ts index ef1e062b..1e159510 100644 --- a/common/lib/utils/utils.ts +++ b/common/lib/utils/utils.ts @@ -117,6 +117,14 @@ export function isDialectTopologyAware(dialect: any): dialect is TopologyAwareDa return dialect; } +export function containsHostAndPort(hosts: HostInfo[] | null | undefined, hostAndPort: string): boolean { + if (hosts?.length === 0) { + return false; + } + + return hosts.some((host) => host.hostAndPort === hostAndPort); +} + export class Pair { private readonly _left: K; private readonly _right: V; diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index 16aff1dc..b39d6c3e 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -471,6 +471,13 @@ export class WrapperProperties { false ); + static readonly CACHED_READER_KEEP_ALIVE_TIMEOUT = new WrapperProperty( + "cachedReaderKeepAliveTimeoutMs", + "The time in milliseconds to keep a reader connection alive in the cache. " + + "Default value 0 means the Wrapper will keep reusing the same cached reader connection.", + 0 + ); + private static readonly PREFIXES = [ WrapperProperties.MONITORING_PROPERTY_PREFIX, ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX, diff --git a/tests/integration/container/tests/read_write_splitting.test.ts b/tests/integration/container/tests/read_write_splitting.test.ts index f1355fe9..bb838105 100644 --- a/tests/integration/container/tests/read_write_splitting.test.ts +++ b/tests/integration/container/tests/read_write_splitting.test.ts @@ -112,6 +112,7 @@ describe("aurora read write splitting", () => { }, 1320000); afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); if (client !== null) { try { await client.end(); diff --git a/tests/unit/read_write_splitting.test.ts b/tests/unit/read_write_splitting.test.ts index aba156d7..bb291e50 100644 --- a/tests/unit/read_write_splitting.test.ts +++ b/tests/unit/read_write_splitting.test.ts @@ -82,6 +82,16 @@ const mockExecuteFuncThrowsFailoverSuccessError = jest.fn(() => { throw new FailoverSuccessError("test"); }); +class TestReadWriteSplitting extends ReadWriteSplittingPlugin { + getWriterTargetClient(): ClientWrapper | undefined { + return this.writerTargetClient; + } + + getReaderTargetClient(): ClientWrapper | undefined { + return this.readerCacheItem?.get(); + } +} + describe("reader write splitting test", () => { beforeEach(() => { when(mockPluginService.getHostListProvider()).thenReturn(instance(mockHostListProvider)); @@ -92,6 +102,8 @@ describe("reader write splitting test", () => { when(mockDriverDialect.connect(anything(), anything())).thenReturn(Promise.resolve(mockReaderWrapper)); when(mockDriverDialect.getQueryFromMethodArg("test")).thenReturn("test"); when(mockPluginService.getDriverDialect()).thenReturn(instance(mockDriverDialect)); + when(mockWriterClient.targetClient).thenReturn(instance(mockWriterWrapper)); + when(mockReaderClient.targetClient).thenReturn(instance(mockReaderWrapper)); properties.clear(); }); @@ -110,13 +122,14 @@ describe("reader write splitting test", () => { it("test set read only true", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getAllHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(anything(), anything(), anything())).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(await mockPluginService.isClientValid(anything())).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -127,7 +140,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.refreshHostList()).once(); verify(mockPluginService.setCurrentClient(mockReaderWrapper, readerHost1)).once(); - expect(target.readerTargetClient).toBe(mockReaderWrapper); + expect(target.getReaderTargetClient()).toBe(mockReaderWrapper); }); it("test set read only false", async () => { @@ -140,7 +153,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockWriterWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -150,7 +163,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(false); verify(mockPluginService.setCurrentClient(mockWriterWrapper, writerHost)).once(); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); }); it("test set read only true already on reader", async () => { @@ -164,7 +177,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -174,8 +187,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(mockReaderWrapper); - expect(target.writerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(mockReaderWrapper); + expect(target.getWriterTargetClient()).toEqual(undefined); }); it("test set read only false already on reader", async () => { @@ -188,7 +201,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(mockReaderWrapper); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -198,8 +211,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(false); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test set read only true one host", async () => { @@ -214,7 +227,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(anything(), anything(), anything())).thenReturn(Promise.resolve(mockWriterWrapper)); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -225,8 +238,8 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); - expect(target.writerTargetClient).toEqual(mockWriterWrapper); + expect(target.getReaderTargetClient()).toEqual(undefined); + expect(target.getWriterTargetClient()).toEqual(mockWriterWrapper); }); it("test connect incorrect host role", async () => { @@ -242,7 +255,7 @@ describe("reader write splitting test", () => { when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); when(mockHostListProviderService.getHostListProvider()).thenReturn(mockHostListProviderInstance); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -264,7 +277,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(await mockPluginService.connect(writerHost, properties)).thenReject(); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -287,7 +300,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.connect(readerHost1 || readerHost2, properties)).thenReject(); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -297,7 +310,7 @@ describe("reader write splitting test", () => { await target.switchClientIfRequired(true); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test set read only on closed connection", async () => { @@ -308,7 +321,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClosedWriterClient)); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -318,7 +331,7 @@ describe("reader write splitting test", () => { await expect(async () => await target.switchClientIfRequired(true)).rejects.toThrow(AwsWrapperError); verify(mockPluginService.setCurrentClient(anything(), anything())).never(); - expect(target.readerTargetClient).toEqual(undefined); + expect(target.getReaderTargetClient()).toEqual(undefined); }); it("test execute failover to new writer", async () => { @@ -330,7 +343,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(mockNewWriterClient); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(await mockPluginService.isClientValid(mockWriterWrapper)).thenReturn(true); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -352,7 +365,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentClient()).thenReturn(mockWriterClient); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -372,7 +385,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderService, @@ -394,7 +407,7 @@ describe("reader write splitting test", () => { when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); - const target = new ReadWriteSplittingPlugin( + const target = new TestReadWriteSplitting( mockPluginServiceInstance, properties, mockHostListProviderServiceInstance, @@ -409,8 +422,9 @@ describe("reader write splitting test", () => { it("test pooled reader connection after set read only", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(anything(), anything(), anything())).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(mockPluginService.isPooledClient()).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHost).thenReturn(writerHost).thenReturn(readerHost1); when(mockDriverDialect.connect(anything(), anything())).thenReturn(Promise.resolve(poolClientWrapper)); @@ -431,14 +445,15 @@ describe("reader write splitting test", () => { const spyTarget = instance(target); await spyTarget.switchClientIfRequired(true); await spyTarget.switchClientIfRequired(false); - verify(target.closeTargetClientIfIdle(poolClientWrapper)).once(); + verify(target.closeReaderClientIfIdle()).twice(); }); it("test pooled writer connection after set read only", async () => { const mockPluginServiceInstance = instance(mockPluginService); when(mockPluginService.getHosts()).thenReturn(singleReaderTopology); - when(mockPluginService.getHostInfoByStrategy(HostRole.READER, "random")).thenReturn(readerHost1); + when(mockPluginService.getHostInfoByStrategy(HostRole.READER, "random", singleReaderTopology)).thenReturn(readerHost1); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockWriterClient)); + when(mockPluginService.isPooledClient()).thenReturn(true); when(await mockWriterClient.isValid()).thenReturn(true); when(mockPluginService.getCurrentHostInfo()) .thenReturn(writerHost) @@ -480,6 +495,7 @@ describe("reader write splitting test", () => { await spyTarget.switchClientIfRequired(false); await spyTarget.switchClientIfRequired(true); - verify(target.closeTargetClientIfIdle(poolClientWrapper)).twice(); + verify(target.closeReaderClientIfIdle()).times(3); + verify(target.closeWriterClientIfIdle()).once(); }); }); From 1b9e03c68be46273fed6e09866da0b3cf598d117 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Fri, 10 Apr 2026 20:06:52 +0000 Subject: [PATCH 13/20] feat: GDB Failover (#625) --- .github/workflows/integration_tests.yml | 80 +- .../aws_secrets_manager_plugin.ts | 22 +- .../aws_secrets_manager_plugin_factory.ts | 6 +- .../iam_authentication_plugin_factory.ts | 6 +- common/lib/aws_client.ts | 57 +- common/lib/connection_info.ts | 28 +- common/lib/connection_plugin_chain_builder.ts | 10 +- .../lib/database_dialect/database_dialect.ts | 3 +- .../database_dialect_codes.ts | 2 + .../database_dialect_manager.ts | 16 + .../topology_aware_database_dialect.ts | 32 +- common/lib/driver_dialect/driver_dialect.ts | 2 +- common/lib/highest_weight_host_selector.ts | 2 +- .../host_availability_cache_item.ts} | 24 +- .../aurora_topology_utils.ts | 68 ++ .../connection_string_host_list_provider.ts | 24 +- .../global_aurora_host_list_provider.ts | 71 ++ .../global_topology_utils.ts | 57 +- .../host_list_provider/host_list_provider.ts | 16 +- .../monitoring/cluster_topology_monitor.ts | 702 ++++++++++++------ .../global_aurora_topology_monitor.ts | 69 ++ .../monitoring_host_list_provider.ts | 112 --- .../rds_host_list_provider.ts | 165 ++-- common/lib/host_list_provider/topology.ts | 28 +- .../lib/host_list_provider/topology_utils.ts | 98 +-- common/lib/host_list_provider_service.ts | 6 +- common/lib/host_selector.ts | 2 +- .../internal_pooled_connection_provider.ts | 2 +- common/lib/least_connections_host_selector.ts | 2 +- common/lib/mysql_client_wrapper.ts | 2 +- common/lib/partial_plugin_service.ts | 550 ++++++++++++++ common/lib/plugin_factory.ts | 3 +- common/lib/plugin_manager.ts | 39 +- common/lib/plugin_service.ts | 137 ++-- ...rora_initial_connection_strategy_plugin.ts | 8 +- ...tial_connection_strategy_plugin_factory.ts | 8 +- .../plugins/bluegreen/blue_green_plugin.ts | 2 +- .../bluegreen/blue_green_plugin_factory.ts | 6 +- .../plugins/bluegreen/blue_green_status.ts | 7 +- .../bluegreen/blue_green_status_monitor.ts | 16 +- .../bluegreen/blue_green_status_provider.ts | 63 +- ...orresponding_host_found_connect_routing.ts | 6 +- .../plugins/connect_time_plugin_factory.ts | 4 +- ...urora_connection_tracker_plugin_factory.ts | 6 +- .../custom_endpoint_plugin_factory.ts | 6 +- .../custom_endpoint_role_type.ts | 21 +- common/lib/plugins/default_plugin.ts | 7 +- .../developer_connection_plugin_factory.ts | 10 +- .../efm/host_monitoring_plugin_factory.ts | 10 +- .../efm2/host_monitoring2_plugin_factory.ts | 10 +- .../plugins/execute_time_plugin_factory.ts | 4 +- common/lib/plugins/failover/failover_mode.ts | 23 +- .../lib/plugins/failover/failover_plugin.ts | 3 +- .../failover/failover_plugin_factory.ts | 6 +- .../failover/writer_failover_handler.ts | 4 +- .../lib/plugins/failover2/failover2_plugin.ts | 102 +-- .../failover2/failover2_plugin_factory.ts | 6 +- .../federated_auth_plugin_factory.ts | 5 +- .../okta_auth_plugin_factory.ts | 5 +- .../gdb_failover/global_db_failover_mode.ts | 34 + .../gdb_failover/global_db_failover_plugin.ts | 367 +++++++++ .../global_db_failover_plugin_factory.ts | 38 + .../limitless_connection_plugin_factory.ts | 6 +- .../abstract_read_write_splitting_plugin.ts | 1 - .../read_write_splitting_plugin.ts | 2 +- .../read_write_splitting_plugin_factory.ts | 6 +- .../lib/plugins/stale_dns/stale_dns_helper.ts | 93 +-- .../stale_dns/stale_dns_plugin_factory.ts | 6 +- ...fastest_respose_strategy_plugin_factory.ts | 9 +- common/lib/random_host_selector.ts | 4 +- common/lib/round_robin_host_selector.ts | 2 +- common/lib/session_state_client.ts | 8 +- common/lib/types.ts | 28 +- common/lib/utils/cache_map.ts | 2 +- common/lib/utils/core_services_container.ts | 57 +- common/lib/utils/errors.ts | 4 +- .../utils/events/batching_event_publisher.ts | 99 +++ common/lib/utils/events/data_access_event.ts | 35 + common/lib/utils/events/event.ts | 64 ++ .../lib/utils/events/monitor_reset_event.ts | 32 + common/lib/utils/events/monitor_stop_event.ts | 33 + common/lib/utils/full_services_container.ts | 67 ++ common/lib/utils/gdb_region_utils.ts | 6 +- common/lib/utils/important_event_service.ts | 88 +++ common/lib/utils/messages.ts | 64 +- common/lib/utils/monitoring/monitor.ts | 128 ++++ .../lib/utils/monitoring/monitor_service.ts | 450 +++++++++++ common/lib/utils/rds_url_type.ts | 1 + common/lib/utils/rds_utils.ts | 214 ++++-- common/lib/utils/service_utils.ts | 124 ++++ common/lib/utils/status_cache_item.ts | 23 + common/lib/utils/storage/expiration_cache.ts | 28 +- common/lib/utils/storage/storage_service.ts | 86 ++- common/lib/utils/utils.ts | 63 +- common/lib/wrapper_property.ts | 93 ++- .../UsingTheConnectionPool.md | 2 +- eslint.config.js | 2 +- mysql/lib/client.ts | 18 +- .../dialect/aurora_mysql_database_dialect.ts | 27 +- .../global_aurora_mysql_database_dialect.ts | 45 +- mysql/lib/dialect/mysql2_driver_dialect.ts | 5 + mysql/lib/dialect/mysql_database_dialect.ts | 5 +- .../rds_multi_az_mysql_database_dialect.ts | 23 +- package.json | 2 +- pg/lib/client.ts | 12 +- pg/lib/dialect/aurora_pg_database_dialect.ts | 27 +- .../global_aurora_pg_database_dialect.ts | 43 +- .../dialect/node_postgres_driver_dialect.ts | 4 + pg/lib/dialect/pg_database_dialect.ts | 5 +- .../rds_multi_az_pg_database_dialect.ts | 23 +- .../container/tests/aurora_failover.test.ts | 315 -------- .../container/tests/aurora_failover2.test.ts | 259 ------- .../tests/connect_execute_time_plugin.test.ts | 3 +- .../tests/failover/aurora_failover.test.ts | 125 ++++ .../tests/failover/aurora_failover2.test.ts | 19 + .../tests/failover/failover_tests.ts | 271 +++++++ .../tests/failover/gdb_failover.test.ts | 184 +++++ .../tests/fastest_response_strategy.test.ts | 3 +- .../tests/initial_connection_strategy.test.ts | 3 +- .../tests/read_write_splitting.test.ts | 15 +- .../container/tests/session_state.test.ts | 4 +- .../tests/utils/aurora_test_utility.ts | 35 + .../container/tests/utils/test_environment.ts | 9 + tests/plugin_benchmarks.ts | 8 +- tests/plugin_manager_benchmarks.ts | 10 +- tests/plugin_manager_telemetry_benchmarks.ts | 10 +- tests/plugin_telemetry_benchmarks.ts | 12 +- tests/testplugin/benchmark_plugin_factory.ts | 5 +- ...initial_connection_strategy_plugin.test.ts | 4 +- tests/unit/batching_event_publisher.test.ts | 141 ++++ .../connection_plugin_chain_builder.test.ts | 24 +- tests/unit/database_dialect.test.ts | 34 +- tests/unit/failover2_plugin.test.ts | 6 +- tests/unit/failover_plugin.test.ts | 2 +- tests/unit/iam_authentication_plugin.test.ts | 1 + tests/unit/notification_pipeline.test.ts | 8 +- tests/unit/plugin_service.test.ts | 4 +- tests/unit/rds_host_list_provider.test.ts | 36 +- tests/unit/read_write_splitting.test.ts | 4 +- tests/unit/sliding_expiration_cache.test.ts | 4 +- tests/unit/stale_dns_helper.test.ts | 211 ------ tests/unit/storage_service.test.ts | 30 +- tests/unit/topology_utils.test.ts | 15 +- tests/unit/writer_failover_handler.test.ts | 2 +- 144 files changed, 5008 insertions(+), 2207 deletions(-) rename common/lib/{plugin_service_manager_container.ts => host_availability/host_availability_cache_item.ts} (50%) create mode 100644 common/lib/host_list_provider/aurora_topology_utils.ts create mode 100644 common/lib/host_list_provider/global_aurora_host_list_provider.ts create mode 100644 common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts delete mode 100644 common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts create mode 100644 common/lib/partial_plugin_service.ts create mode 100644 common/lib/plugins/gdb_failover/global_db_failover_mode.ts create mode 100644 common/lib/plugins/gdb_failover/global_db_failover_plugin.ts create mode 100644 common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts create mode 100644 common/lib/utils/events/batching_event_publisher.ts create mode 100644 common/lib/utils/events/data_access_event.ts create mode 100644 common/lib/utils/events/event.ts create mode 100644 common/lib/utils/events/monitor_reset_event.ts create mode 100644 common/lib/utils/events/monitor_stop_event.ts create mode 100644 common/lib/utils/full_services_container.ts create mode 100644 common/lib/utils/important_event_service.ts create mode 100644 common/lib/utils/monitoring/monitor.ts create mode 100644 common/lib/utils/monitoring/monitor_service.ts create mode 100644 common/lib/utils/service_utils.ts create mode 100644 common/lib/utils/status_cache_item.ts delete mode 100644 tests/integration/container/tests/aurora_failover.test.ts delete mode 100644 tests/integration/container/tests/aurora_failover2.test.ts create mode 100644 tests/integration/container/tests/failover/aurora_failover.test.ts create mode 100644 tests/integration/container/tests/failover/aurora_failover2.test.ts create mode 100644 tests/integration/container/tests/failover/failover_tests.ts create mode 100644 tests/integration/container/tests/failover/gdb_failover.test.ts create mode 100644 tests/unit/batching_event_publisher.test.ts diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index ecd6a876..8b59c9ef 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -48,8 +48,8 @@ jobs: environment: integration-tests steps: - run: echo "Fork PR approved by maintainer" - run-integration-tests: - name: Run Integration Tests + run-integration-tests-default: + name: Run Integration Tests (Default) needs: [ approve, approve-fork ] if: | always() && @@ -59,8 +59,7 @@ jobs: strategy: fail-fast: false matrix: - versions: [ "default", "latest" ] - dbEngine: ["aurora-mysql", "aurora-postgres" ] + dbEngine: ["aurora-mysql", "aurora-postgres"] steps: - name: 'Clone repository' @@ -97,8 +96,75 @@ jobs: AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} - AURORA_MYSQL_DB_ENGINE_VERSION: ${{ matrix.dbEngine }} - AURORA_PG_DB_ENGINE_VERSION: ${{ matrix.versions }} + AURORA_MYSQL_DB_ENGINE_VERSION: default + AURORA_PG_DB_ENGINE_VERSION: default + + - name: "Get Github Action IP" + if: always() + id: ip + uses: haythem/public-ip@v1.3 + + - name: "Remove Github Action IP" + if: always() + run: | + aws ec2 revoke-security-group-ingress \ + --group-name default \ + --protocol -1 \ + --port -1 \ + --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ + 2>&1 > /dev/null; + + - name: Archive results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-report-default-${{ matrix.dbEngine }} + path: ./tests/integration/container/reports + retention-days: 5 + + run-integration-tests-latest: + name: Run Integration Tests (Latest) + runs-on: ubuntu-latest + needs: run-integration-tests-default + strategy: + fail-fast: false + matrix: + dbEngine: ["aurora-mysql", "aurora-postgres" ] + + steps: + - name: Clone repository + uses: actions/checkout@v4 + - name: "Set up JDK 8" + uses: actions/setup-java@v3 + with: + distribution: "corretto" + java-version: 8 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: "20.x" + - name: Install dependencies + run: npm install --no-save + + - name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} + role-session-name: nodejs_int_latest_tests + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + output-credentials: true + + - name: Run Integration Tests + run: | + ./gradlew --no-parallel --no-daemon test-${{ matrix.dbEngine }} --info + env: + RDS_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ steps.creds.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.creds.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.creds.outputs.aws-session-token }} + AURORA_MYSQL_DB_ENGINE_VERSION: latest + AURORA_PG_DB_ENGINE_VERSION: latest - name: "Get Github Action IP" if: always() @@ -119,6 +185,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: integration-report-default-${{ matrix.dbEngine }}-${{ matrix.versions}} + name: integration-report-latest-${{ matrix.dbEngine }} path: ./tests/integration/container/reports retention-days: 5 diff --git a/common/lib/authentication/aws_secrets_manager_plugin.ts b/common/lib/authentication/aws_secrets_manager_plugin.ts index c7f2e621..631b9f06 100644 --- a/common/lib/authentication/aws_secrets_manager_plugin.ts +++ b/common/lib/authentication/aws_secrets_manager_plugin.ts @@ -129,20 +129,18 @@ export class AwsSecretsManagerPlugin extends AbstractConnectionPlugin implements this.pluginService.updateConfigWithProperties(props); return await connectFunc(); } catch (error) { - if (error instanceof Error) { - if ((error.message.includes("password authentication failed") || error.message.includes("Access denied")) && !secretWasFetched) { - // Login unsuccessful with cached credentials - // Try to re-fetch credentials and try again - - secretWasFetched = await this.updateSecret(true); - if (secretWasFetched) { - WrapperProperties.USER.set(props, this.secret?.username ?? ""); - WrapperProperties.PASSWORD.set(props, this.secret?.password ?? ""); - return await connectFunc(); - } + if ((error.message.includes("password authentication failed") || error.message.includes("Access denied")) && !secretWasFetched) { + // Login unsuccessful with cached credentials + // Try to re-fetch credentials and try again + + secretWasFetched = await this.updateSecret(true); + if (secretWasFetched) { + WrapperProperties.USER.set(props, this.secret?.username ?? ""); + WrapperProperties.PASSWORD.set(props, this.secret?.password ?? ""); + return await connectFunc(); } - logger.debug(Messages.get("AwsSecretsManagerConnectionPlugin.unhandledError", error.name, error.message)); } + logger.debug(Messages.get("AwsSecretsManagerConnectionPlugin.unhandledError", error.name, error.message)); throw error; } } diff --git a/common/lib/authentication/aws_secrets_manager_plugin_factory.ts b/common/lib/authentication/aws_secrets_manager_plugin_factory.ts index eb0ad30d..6b3242cd 100644 --- a/common/lib/authentication/aws_secrets_manager_plugin_factory.ts +++ b/common/lib/authentication/aws_secrets_manager_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class AwsSecretsManagerPluginFactory extends ConnectionPluginFactory { private static awsSecretsManagerPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin) { AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin = await import("./aws_secrets_manager_plugin"); } - return new AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin.AwsSecretsManagerPlugin(pluginService, new Map(properties)); + return new AwsSecretsManagerPluginFactory.awsSecretsManagerPlugin.AwsSecretsManagerPlugin(servicesContainer.pluginService, new Map(properties)); } catch (error: any) { if (error.code === "MODULE_NOT_FOUND") { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AwsSecretsManagerPlugin")); diff --git a/common/lib/authentication/iam_authentication_plugin_factory.ts b/common/lib/authentication/iam_authentication_plugin_factory.ts index da5e0a32..6ececd67 100644 --- a/common/lib/authentication/iam_authentication_plugin_factory.ts +++ b/common/lib/authentication/iam_authentication_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class IamAuthenticationPluginFactory extends ConnectionPluginFactory { private static iamAuthenticationPlugin: any; - async getInstance(pluginService: PluginService, properties: object): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: object): Promise { try { if (!IamAuthenticationPluginFactory.iamAuthenticationPlugin) { IamAuthenticationPluginFactory.iamAuthenticationPlugin = await import("./iam_authentication_plugin"); } - return new IamAuthenticationPluginFactory.iamAuthenticationPlugin.IamAuthenticationPlugin(pluginService); + return new IamAuthenticationPluginFactory.iamAuthenticationPlugin.IamAuthenticationPlugin(servicesContainer.pluginService); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "IamAuthenticationPlugin")); } diff --git a/common/lib/aws_client.ts b/common/lib/aws_client.ts index 0140c847..9f772be1 100644 --- a/common/lib/aws_client.ts +++ b/common/lib/aws_client.ts @@ -14,8 +14,7 @@ limitations under the License. */ -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; -import { PluginService, PluginServiceImpl } from "./plugin_service"; +import { PluginService } from "./plugin_service"; import { DatabaseDialect, DatabaseType } from "./database_dialect/database_dialect"; import { ConnectionUrlParser } from "./utils/connection_url_parser"; import { HostListProvider } from "./host_list_provider/host_list_provider"; @@ -23,26 +22,31 @@ import { PluginManager } from "./plugin_manager"; import pkgStream from "stream"; import { ClientWrapper } from "./client_wrapper"; -import { ConnectionProviderManager } from "./connection_provider_manager"; import { DefaultTelemetryFactory } from "./utils/telemetry/default_telemetry_factory"; import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; import { DriverDialect } from "./driver_dialect/driver_dialect"; import { WrapperProperties } from "./wrapper_property"; import { DriverConfigurationProfiles } from "./profile/driver_configuration_profiles"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { AwsWrapperError, TransactionIsolationLevel, ConnectionProvider } from "./"; +import { AwsWrapperError, ConnectionProvider, TransactionIsolationLevel } from "./"; import { Messages } from "./utils/messages"; import { HostListProviderService } from "./host_list_provider_service"; import { SessionStateClient } from "./session_state_client"; -import { DriverConnectionProvider } from "./driver_connection_provider"; +import { ServiceUtils } from "./utils/service_utils"; import { StorageService } from "./utils/storage/storage_service"; +import { MonitorService } from "./utils/monitoring/monitor_service"; import { CoreServicesContainer } from "./utils/core_services_container"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { EventPublisher } from "./utils/events/event"; const { EventEmitter } = pkgStream; export abstract class AwsClient extends EventEmitter implements SessionStateClient { private _defaultPort: number = -1; + private readonly fullServiceContainer: FullServicesContainer; private readonly storageService: StorageService; + private readonly monitorService: MonitorService; + private readonly eventPublisher: EventPublisher; protected telemetryFactory: TelemetryFactory; protected pluginManager: PluginManager; protected pluginService: PluginService; @@ -67,7 +71,7 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie this.properties = new Map(Object.entries(config)); - this.storageService = CoreServicesContainer.getInstance().getStorageService(); + this.storageService = CoreServicesContainer.getInstance().storageService; const profileName = WrapperProperties.PROFILE_NAME.get(this.properties); if (profileName && profileName.length > 0) { @@ -103,22 +107,27 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie } } + const coreServicesContainer: CoreServicesContainer = CoreServicesContainer.getInstance(); + this.storageService = coreServicesContainer.storageService; + this.monitorService = coreServicesContainer.monitorService; + this.eventPublisher = coreServicesContainer.eventPublisher; this.telemetryFactory = new DefaultTelemetryFactory(this.properties); - const container = new PluginServiceManagerContainer(); - this.pluginService = new PluginServiceImpl( - container, + + this.fullServiceContainer = ServiceUtils.instance.createStandardServiceContainer( + this.storageService, + this.monitorService, + this.eventPublisher, this, + this.properties, dbType, knownDialectsByCode, - this.properties, - this._configurationProfile?.getDriverDialect() ?? driverDialect - ); - this.pluginManager = new PluginManager( - container, - this.properties, - new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(this.properties)), - this.telemetryFactory + this._configurationProfile?.getDriverDialect() ?? driverDialect, + this.telemetryFactory, + connectionProvider ); + + this.pluginService = this.fullServiceContainer.pluginService; + this.pluginManager = this.fullServiceContainer.pluginManager; } private async setup() { @@ -130,12 +139,12 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie await this.setup(); const hostListProvider: HostListProvider = this.pluginService .getDialect() - .getHostListProvider(this.properties, this.properties.get("host"), (this.pluginService)); + .getHostListProvider(this.properties, this.properties.get("host"), this.fullServiceContainer); this.pluginService.setHostListProvider(hostListProvider); await this.pluginService.refreshHostList(); const initialHostInfo = this.pluginService.getInitialConnectionHostInfo(); if (initialHostInfo != null) { - await this.pluginManager.initHostProvider(initialHostInfo, this.properties, (this.pluginService)); + await this.pluginManager.initHostProvider(initialHostInfo, this.properties, this.fullServiceContainer.hostListProviderService); await this.pluginService.refreshHostList(); } } @@ -159,23 +168,23 @@ export abstract class AwsClient extends EventEmitter implements SessionStateClie abstract setReadOnly(readOnly: boolean): Promise; - abstract isReadOnly(): boolean; + abstract isReadOnly(): boolean | undefined; abstract setAutoCommit(autoCommit: boolean): Promise; - abstract getAutoCommit(): boolean; + abstract getAutoCommit(): boolean | undefined; abstract setTransactionIsolation(level: TransactionIsolationLevel): Promise; - abstract getTransactionIsolation(): TransactionIsolationLevel; + abstract getTransactionIsolation(): TransactionIsolationLevel | undefined; abstract setSchema(schema: any): Promise; - abstract getSchema(): string; + abstract getSchema(): string | undefined; abstract setCatalog(catalog: string): Promise; - abstract getCatalog(): string; + abstract getCatalog(): string | undefined; abstract end(): Promise; diff --git a/common/lib/connection_info.ts b/common/lib/connection_info.ts index 2d195120..77b8e650 100644 --- a/common/lib/connection_info.ts +++ b/common/lib/connection_info.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { ClientWrapper } from "./client_wrapper"; diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 9c399dd7..1cad14c7 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -43,6 +43,8 @@ import { CustomEndpointPluginFactory } from "./plugins/custom_endpoint/custom_en import { ConfigurationProfile } from "./profile/configuration_profile"; import { HostMonitoring2PluginFactory } from "./plugins/efm2/host_monitoring2_plugin_factory"; import { BlueGreenPluginFactory } from "./plugins/bluegreen/blue_green_plugin_factory"; +import { GlobalDbFailoverPluginFactory } from "./plugins/gdb_failover/global_db_failover_plugin_factory"; +import { FullServicesContainer } from "./utils/full_services_container"; /* Type alias used for plugin factory sorting. It holds a reference to a plugin @@ -65,6 +67,7 @@ export class ConnectionPluginChainBuilder { ["readWriteSplitting", { factory: ReadWriteSplittingPluginFactory, weight: 600 }], ["failover", { factory: FailoverPluginFactory, weight: 700 }], ["failover2", { factory: Failover2PluginFactory, weight: 710 }], + ["gdbFailover", { factory: GlobalDbFailoverPluginFactory, weight: 720 }], ["efm", { factory: HostMonitoringPluginFactory, weight: 800 }], ["efm2", { factory: HostMonitoring2PluginFactory, weight: 810 }], ["fastestResponseStrategy", { factory: FastestResponseStrategyPluginFactory, weight: 900 }], @@ -86,6 +89,7 @@ export class ConnectionPluginChainBuilder { [ReadWriteSplittingPluginFactory, 600], [FailoverPluginFactory, 700], [Failover2PluginFactory, 710], + [GlobalDbFailoverPluginFactory, 720], [HostMonitoringPluginFactory, 800], [HostMonitoring2PluginFactory, 810], [LimitlessConnectionPluginFactory, 950], @@ -99,7 +103,7 @@ export class ConnectionPluginChainBuilder { ]); static async getPlugins( - pluginService: PluginService, + servicesContainer: FullServicesContainer, props: Map, connectionProviderManager: ConnectionProviderManager, configurationProfile: ConfigurationProfile | null @@ -162,10 +166,10 @@ export class ConnectionPluginChainBuilder { for (const pluginFactoryInfo of pluginFactoryInfoList) { const factoryObj = new pluginFactoryInfo.factory(); - plugins.push(await factoryObj.getInstance(pluginService, props)); + plugins.push(await factoryObj.getInstance(servicesContainer, props)); } - plugins.push(new DefaultPlugin(pluginService, connectionProviderManager)); + plugins.push(new DefaultPlugin(servicesContainer, connectionProviderManager)); return plugins; } diff --git a/common/lib/database_dialect/database_dialect.ts b/common/lib/database_dialect/database_dialect.ts index 22f2cf28..2077934a 100644 --- a/common/lib/database_dialect/database_dialect.ts +++ b/common/lib/database_dialect/database_dialect.ts @@ -21,6 +21,7 @@ import { FailoverRestriction } from "../plugins/failover/failover_restriction"; import { ErrorHandler } from "../error_handler"; import { TransactionIsolationLevel } from "../utils/transaction_isolation_level"; import { HostRole } from "../host_role"; +import { FullServicesContainer } from "../utils/full_services_container"; export enum DatabaseType { MYSQL, @@ -41,7 +42,7 @@ export interface DatabaseDialect { getErrorHandler(): ErrorHandler; getHostRole(targetClient: ClientWrapper): Promise; isDialect(targetClient: ClientWrapper): Promise; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider; + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider; isClientValid(targetClient: ClientWrapper): Promise; getDatabaseType(): DatabaseType; getDialectName(): string; diff --git a/common/lib/database_dialect/database_dialect_codes.ts b/common/lib/database_dialect/database_dialect_codes.ts index 4815f48e..4351a6cb 100644 --- a/common/lib/database_dialect/database_dialect_codes.ts +++ b/common/lib/database_dialect/database_dialect_codes.ts @@ -15,11 +15,13 @@ */ export class DatabaseDialectCodes { + static readonly GLOBAL_AURORA_MYSQL: string = "global-aurora-mysql"; static readonly AURORA_MYSQL: string = "aurora-mysql"; static readonly RDS_MYSQL: string = "rds-mysql"; static readonly MYSQL: string = "mysql"; // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html static readonly RDS_MULTI_AZ_MYSQL: string = "rds-multi-az-mysql"; + static readonly GLOBAL_AURORA_PG: string = "global-aurora-pg"; static readonly AURORA_PG: string = "aurora-pg"; static readonly RDS_PG: string = "rds-pg"; // https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/multi-az-db-clusters-concepts.html diff --git a/common/lib/database_dialect/database_dialect_manager.ts b/common/lib/database_dialect/database_dialect_manager.ts index e8127f2a..b462e60b 100644 --- a/common/lib/database_dialect/database_dialect_manager.ts +++ b/common/lib/database_dialect/database_dialect_manager.ts @@ -95,6 +95,14 @@ export class DatabaseDialectManager implements DatabaseDialectProvider { if (this.dbType === DatabaseType.MYSQL) { const type = this.rdsHelper.identifyRdsType(host); + if (type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + this.canUpdate = false; + this.dialectCode = DatabaseDialectCodes.GLOBAL_AURORA_MYSQL; + this.dialect = this.knownDialectsByCode.get(DatabaseDialectCodes.GLOBAL_AURORA_MYSQL); + this.logCurrentDialect(); + return this.dialect; + } + if (type.isRdsCluster) { this.canUpdate = true; this.dialectCode = DatabaseDialectCodes.AURORA_MYSQL; @@ -128,6 +136,14 @@ export class DatabaseDialectManager implements DatabaseDialectProvider { return this.dialect; } + if (type == RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + this.canUpdate = false; + this.dialectCode = DatabaseDialectCodes.GLOBAL_AURORA_PG; + this.dialect = this.knownDialectsByCode.get(DatabaseDialectCodes.GLOBAL_AURORA_PG); + this.logCurrentDialect(); + return this.dialect; + } + if (type.isRdsCluster) { this.canUpdate = true; this.dialectCode = DatabaseDialectCodes.AURORA_PG; diff --git a/common/lib/database_dialect/topology_aware_database_dialect.ts b/common/lib/database_dialect/topology_aware_database_dialect.ts index cdd1929c..df0553c8 100644 --- a/common/lib/database_dialect/topology_aware_database_dialect.ts +++ b/common/lib/database_dialect/topology_aware_database_dialect.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { HostRole } from "../host_role"; import { ClientWrapper } from "../client_wrapper"; @@ -34,3 +34,7 @@ export interface TopologyAwareDatabaseDialect { export interface GlobalAuroraTopologyDialect extends TopologyAwareDatabaseDialect { getRegionByInstanceId(targetClient: ClientWrapper, instanceId: string): Promise; } + +export function isDialectTopologyAware(dialect: any): dialect is TopologyAwareDatabaseDialect { + return dialect; +} diff --git a/common/lib/driver_dialect/driver_dialect.ts b/common/lib/driver_dialect/driver_dialect.ts index 34780544..05bf0591 100644 --- a/common/lib/driver_dialect/driver_dialect.ts +++ b/common/lib/driver_dialect/driver_dialect.ts @@ -30,7 +30,7 @@ export interface DriverDialect { setConnectTimeout(props: Map, wrapperConnectTimeout?: any): void; - setQueryTimeout(props: Map, sql?: any, wrapperConnectTimeout?: any): void; + setQueryTimeout(props: Map, sql?: any, wrapperQueryTimeout?: any): void; setKeepAliveProperties(props: Map, keepAliveProps: any): void; diff --git a/common/lib/highest_weight_host_selector.ts b/common/lib/highest_weight_host_selector.ts index 370d6877..75d42d48 100644 --- a/common/lib/highest_weight_host_selector.ts +++ b/common/lib/highest_weight_host_selector.ts @@ -26,7 +26,7 @@ export class HighestWeightHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => (hostA.weight > hostB.weight ? -1 : hostA.weight < hostB.weight ? 1 : 0)); if (eligibleHosts.length === 0) { diff --git a/common/lib/plugin_service_manager_container.ts b/common/lib/host_availability/host_availability_cache_item.ts similarity index 50% rename from common/lib/plugin_service_manager_container.ts rename to common/lib/host_availability/host_availability_cache_item.ts index e8272822..c068f2de 100644 --- a/common/lib/plugin_service_manager_container.ts +++ b/common/lib/host_availability/host_availability_cache_item.ts @@ -14,26 +14,12 @@ limitations under the License. */ -import { PluginService } from "./plugin_service"; -import { PluginManager } from "./plugin_manager"; +import { HostAvailability } from "./host_availability"; -export class PluginServiceManagerContainer { - private _pluginService?: PluginService | null; - private _pluginManager?: PluginManager | null; +export class HostAvailabilityCacheItem { + readonly availability: HostAvailability; - get pluginService(): PluginService | null { - return this._pluginService ?? null; - } - - set pluginService(service: PluginService | null) { - this._pluginService = service; - } - - get pluginManager(): PluginManager | null { - return this._pluginManager ?? null; - } - - set pluginManager(service: PluginManager | null) { - this._pluginManager = service; + constructor(availability: HostAvailability) { + this.availability = availability; } } diff --git a/common/lib/host_list_provider/aurora_topology_utils.ts b/common/lib/host_list_provider/aurora_topology_utils.ts new file mode 100644 index 00000000..5a16ac35 --- /dev/null +++ b/common/lib/host_list_provider/aurora_topology_utils.ts @@ -0,0 +1,68 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TopologyQueryResult, TopologyUtils } from "./topology_utils"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { HostInfo } from "../host_info"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; +import { Messages } from "../utils/messages"; + +/** + * TopologyUtils implementation for Aurora clusters using a single HostInfo template. + */ +export class AuroraTopologyUtils extends TopologyUtils { + async queryForTopology( + targetClient: ClientWrapper, + dialect: DatabaseDialect, + initialHost: HostInfo, + clusterInstanceTemplate: HostInfo + ): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + return await dialect + .queryForTopology(targetClient) + .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHosts(res, initialHost, clusterInstanceTemplate))); + } + + public createHosts(topologyQueryResults: TopologyQueryResult[], initialHost: HostInfo, clusterInstanceTemplate: HostInfo): HostInfo[] { + const hostsMap = new Map(); + topologyQueryResults.forEach((row) => { + const lastUpdateTime = row.lastUpdateTime ?? Date.now(); + + const host = this.createHost( + row.id, + row.host, + row.isWriter, + row.weight, + lastUpdateTime, + initialHost, + clusterInstanceTemplate, + row.endpoint, + row.port + ); + + const existing = hostsMap.get(host.host); + if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { + hostsMap.set(host.host, host); + } + }); + + return Array.from(hostsMap.values()); + } +} diff --git a/common/lib/host_list_provider/connection_string_host_list_provider.ts b/common/lib/host_list_provider/connection_string_host_list_provider.ts index 0885d9b3..a7973111 100644 --- a/common/lib/host_list_provider/connection_string_host_list_provider.ts +++ b/common/lib/host_list_provider/connection_string_host_list_provider.ts @@ -65,17 +65,12 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider this.isInitialized = true; } - refresh(): Promise; - refresh(client: ClientWrapper): Promise; - refresh(client?: ClientWrapper): Promise; - refresh(client?: ClientWrapper | undefined): Promise { + refresh(): Promise { this.init(); return Promise.resolve(this.hostList); } - forceRefresh(): Promise; - forceRefresh(client: ClientWrapper): Promise; - forceRefresh(client?: ClientWrapper): Promise { + forceRefresh(): Promise { this.init(); return Promise.resolve(this.hostList); } @@ -89,7 +84,7 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider return null; } const instance = await this.hostListProviderService.getDialect().getHostAliasAndParseResults(client.client); - const topology = await this.refresh(client.client); + const topology = await this.refresh(); if (!topology || topology.length == 0) { return null; } @@ -97,19 +92,6 @@ export class ConnectionStringHostListProvider implements StaticHostListProvider return topology.filter((hostInfo) => instance === hostInfo.hostId)[0]; } - createHost(host: string, isWriter: boolean, weight: number, lastUpdateTime: number, port?: number): HostInfo { - return this.hostListProviderService - .getHostInfoBuilder() - .withHost(host ?? "") - .withPort(port ?? this.initialPort) - .withRole(isWriter ? HostRole.WRITER : HostRole.READER) - .withAvailability(HostAvailability.AVAILABLE) - .withWeight(weight) - .withLastUpdateTime(lastUpdateTime) - .withHostId(host) - .build(); - } - getHostProviderType(): string { return this.constructor.name; } diff --git a/common/lib/host_list_provider/global_aurora_host_list_provider.ts b/common/lib/host_list_provider/global_aurora_host_list_provider.ts new file mode 100644 index 00000000..7b17e3a5 --- /dev/null +++ b/common/lib/host_list_provider/global_aurora_host_list_provider.ts @@ -0,0 +1,71 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RdsHostListProvider } from "./rds_host_list_provider"; +import { FullServicesContainer } from "../utils/full_services_container"; +import { HostInfo } from "../host_info"; +import { WrapperProperties } from "../wrapper_property"; +import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./monitoring/cluster_topology_monitor"; +import { GlobalAuroraTopologyMonitor } from "./monitoring/global_aurora_topology_monitor"; +import { MonitorInitializer } from "../utils/monitoring/monitor"; +import { ClientWrapper } from "../client_wrapper"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { parseInstanceTemplates } from "../utils/utils"; + +export class GlobalAuroraHostListProvider extends RdsHostListProvider { + protected instanceTemplatesByRegion: Map; + protected override initSettings(): void { + super.initSettings(); + + const instanceTemplates = WrapperProperties.GLOBAL_CLUSTER_INSTANCE_HOST_PATTERNS.get(this.properties); + this.instanceTemplatesByRegion = parseInstanceTemplates( + instanceTemplates, + (hostPattern: string) => this.validateHostPatternSetting(hostPattern), + () => this.hostListProviderService.getHostInfoBuilder() + ); + } + + protected override async getOrCreateMonitor(): Promise { + const initializer: MonitorInitializer = { + createMonitor: (servicesContainer: FullServicesContainer): ClusterTopologyMonitor => { + return new GlobalAuroraTopologyMonitor( + servicesContainer, + this.topologyUtils, + this.clusterId, + this.initialHost, + this.properties, + this.clusterInstanceTemplate, + this.refreshRateNano, + this.highRefreshRateNano, + this.instanceTemplatesByRegion + ); + } + }; + + return await this.servicesContainers.monitorService.runIfAbsent( + ClusterTopologyMonitorImpl, + this.clusterId, + this.servicesContainers, + this.properties, + initializer + ); + } + + override async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + this.init(); + return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.instanceTemplatesByRegion); + } +} diff --git a/common/lib/host_list_provider/global_topology_utils.ts b/common/lib/host_list_provider/global_topology_utils.ts index 7e9131db..7b197dc1 100644 --- a/common/lib/host_list_provider/global_topology_utils.ts +++ b/common/lib/host_list_provider/global_topology_utils.ts @@ -1,45 +1,40 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { TopologyQueryResult, TopologyUtils } from "./topology_utils"; import { ClientWrapper } from "../client_wrapper"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { HostInfo } from "../host_info"; -import { isDialectTopologyAware } from "../utils/utils"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; import { Messages } from "../utils/messages"; import { AwsWrapperError } from "../utils/errors"; -export class GlobalTopologyUtils extends TopologyUtils { - async queryForTopology( - targetClient: ClientWrapper, - dialect: DatabaseDialect, - initialHost: HostInfo, - clusterInstanceTemplate: HostInfo - ): Promise { - throw new AwsWrapperError("Not implemented"); - } +export interface GdbTopologyUtils { + getRegion(instanceId: string, targetClient: ClientWrapper, dialect: DatabaseDialect): Promise; +} - async queryForTopologyWithRegion( +export class GlobalTopologyUtils extends TopologyUtils implements GdbTopologyUtils { + async queryForTopology( targetClient: ClientWrapper, dialect: DatabaseDialect, initialHost: HostInfo, instanceTemplateByRegion: Map ): Promise { if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); + throw new AwsWrapperError(Messages.get("RdsHostListProvider.incorrectDialect")); } return await dialect @@ -47,6 +42,16 @@ export class GlobalTopologyUtils extends TopologyUtils { .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHostsWithTemplateMap(res, initialHost, instanceTemplateByRegion))); } + async getRegion(instanceId: string, targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + if (!isDialectTopologyAware(dialect)) { + throw new AwsWrapperError(Messages.get("RdsHostListProvider.incorrectDialect")); + } + + const results = await dialect.queryForTopology(targetClient); + const match = results.find((row) => row.id === instanceId); + return match?.awsRegion ?? null; + } + private createHostsWithTemplateMap( topologyQueryResults: TopologyQueryResult[], initialHost: HostInfo, diff --git a/common/lib/host_list_provider/host_list_provider.ts b/common/lib/host_list_provider/host_list_provider.ts index a8ea22f4..b7e1005a 100644 --- a/common/lib/host_list_provider/host_list_provider.ts +++ b/common/lib/host_list_provider/host_list_provider.ts @@ -19,25 +19,13 @@ import { HostRole } from "../host_role"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; -export type DynamicHostListProvider = HostListProvider; - export type StaticHostListProvider = HostListProvider; -export interface BlockingHostListProvider extends HostListProvider { - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; - - clearAll(): Promise; -} - export interface HostListProvider { refresh(): Promise; - refresh(client: ClientWrapper): Promise; - forceRefresh(): Promise; - forceRefresh(client: ClientWrapper): Promise; - getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise; identifyConnection(targetClient: ClientWrapper): Promise; @@ -46,3 +34,7 @@ export interface HostListProvider { getClusterId(): string; } + +export interface DynamicHostListProvider extends HostListProvider { + forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; +} diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index 2808c55a..e9b4dcfd 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -17,50 +17,76 @@ import { HostInfo } from "../../host_info"; import { PluginService } from "../../plugin_service"; import { HostAvailability } from "../../host_availability/host_availability"; -import { logTopology, sleep } from "../../utils/utils"; +import { convertMsToNanos, convertNanosToMs, getTimeInNanos, logTopology, sleep } from "../../utils/utils"; import { logger } from "../../../logutils"; import { HostRole } from "../../host_role"; import { ClientWrapper } from "../../client_wrapper"; -import { AwsWrapperError } from "../../utils/errors"; -import { MonitoringRdsHostListProvider } from "./monitoring_host_list_provider"; +import { AwsTimeoutError, AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; -import { CoreServicesContainer } from "../../utils/core_services_container"; import { Topology } from "../topology"; import { StorageService } from "../../utils/storage/storage_service"; import { TopologyUtils } from "../topology_utils"; import { RdsUtils } from "../../utils/rds_utils"; - -export interface ClusterTopologyMonitor { +import { AbstractMonitor, Monitor } from "../../utils/monitoring/monitor"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { HostListProviderService } from "../../host_list_provider_service"; +import { Event, EventSubscriber } from "../../utils/events/event"; +import { MonitorResetEvent } from "../../utils/events/monitor_reset_event"; +import { ServiceUtils } from "../../utils/service_utils"; +import { WrapperProperties } from "../../wrapper_property"; + +export interface ClusterTopologyMonitor extends Monitor, EventSubscriber { forceRefresh(client: ClientWrapper, timeoutMs: number): Promise; close(): Promise; - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; + /** + * Initiates a topology update. + * + * @param verifyTopology defines whether extra measures should be taken to verify the topology. If false, the + * method will return as soon as topology is successfully retrieved from any instance. If + * true, extra steps are taken to verify the topology is accurate. + * @param timeoutMs timeout in msec to wait until the topology gets refreshed (if verifyWriter has a value of + * false) or verified (if verifyTopology has a value of true). + * @return true if successful, false if unsuccessful or the timeout is reached + * @throws AwsWrapperError if wrapper timed out while fetching the topology. + */ + forceMonitoringRefresh(verifyTopology: boolean, timeoutMs: number): Promise; + + canDispose(): boolean; } -export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { - private static readonly TOPOLOGY_CACHE_EXPIRATION_NANOS: number = 5 * 60 * 1_000_000_000; // 5 minutes. - static readonly MONITORING_PROPERTY_PREFIX: string = "topology_monitoring_"; +export class ClusterTopologyMonitorImpl extends AbstractMonitor implements ClusterTopologyMonitor { + private static readonly MONITOR_TERMINATION_TIMEOUT_SEC: number = 30; + private static readonly STABLE_TOPOLOGIES_DURATION_NS: bigint = convertMsToNanos(15000); // 15 seconds. + protected static readonly DEFAULT_CONNECTION_TIMEOUT_MS: number = 5000; + protected static readonly DEFAULT_QUERY_TIMEOUT_MS: number = 5000; private readonly clusterId: string; - private readonly initialHostInfo: HostInfo; + protected readonly initialHostInfo: HostInfo; + private readonly servicesContainer: FullServicesContainer; private readonly _monitoringProperties: Map; private readonly _pluginService: PluginService; - private readonly _hostListProvider: MonitoringRdsHostListProvider; - private readonly refreshRateMs: number; - private readonly highRefreshRateMs: number; + protected readonly hostListProviderService: HostListProviderService; + private readonly refreshRateNs: number; + private readonly highRefreshRateNs: number; private readonly storageService: StorageService; - private readonly topologyUtils: TopologyUtils; private readonly rdsUtils: RdsUtils = new RdsUtils(); - private readonly instanceTemplate: HostInfo; + protected readonly instanceTemplate: HostInfo; - private writerHostInfo: HostInfo = null; + private writerHostInfo: HostInfo | null = null; private isVerifiedWriterConnection: boolean = false; - private monitoringClient: ClientWrapper = null; - private highRefreshRateEndTimeMs: number = -1; - private highRefreshPeriodAfterPanicMs: number = 30000; // 30 seconds. - private ignoreNewTopologyRequestsEndTimeMs: number = -1; - private ignoreTopologyRequestMs: number = 10000; // 10 seconds. + private monitoringClient: ClientWrapper | null = null; + private highRefreshRateEndTimeNs: bigint = BigInt(0); + + public readonly topologyUtils: TopologyUtils; + public readonly readerTopologiesById: Map = new Map(); + public readonly completedOneCycle: Map = new Map(); + // When comparing topologies, we don't want to check HostInfo.weight, which is used in HostInfo#equals. + // We use this function to compare the other fields. + protected readonly hostInfoExtractor = (host: HostInfo): string => { + return `${host.host}:${host.port}:${host.availability}:${host.role}`; + }; // Tracking of the host monitors. private hostMonitors: Map = new Map(); @@ -70,46 +96,50 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { public hostMonitorsLatestTopology: HostInfo[] = []; // Controls for stopping asynchronous monitoring tasks. - private stopMonitoring: boolean = false; public hostMonitorsStop: boolean = false; - private untrackedPromises: Promise[] = []; // Signals to other methods that asynchronous tasks have completed/should be completed. private requestToUpdateTopology: boolean = false; + private submittedHosts: Map> = new Map(); + private stableTopologiesStartNs: bigint; constructor( + servicesContainer: FullServicesContainer, topologyUtils: TopologyUtils, clusterId: string, initialHostInfo: HostInfo, props: Map, instanceTemplate: HostInfo, - pluginService: PluginService, - hostListProvider: MonitoringRdsHostListProvider, - refreshRateMs: number, - highRefreshRateMs: number + refreshRateNs: number, + highRefreshRateNs: number ) { + super(ClusterTopologyMonitorImpl.MONITOR_TERMINATION_TIMEOUT_SEC); this.topologyUtils = topologyUtils; this.clusterId = clusterId; - this.storageService = CoreServicesContainer.getInstance().getStorageService(); // TODO: store serviceContainer instead this.initialHostInfo = initialHostInfo; this.instanceTemplate = instanceTemplate; - this._pluginService = pluginService; - this._hostListProvider = hostListProvider; - this.refreshRateMs = refreshRateMs; - this.highRefreshRateMs = highRefreshRateMs; + this.servicesContainer = servicesContainer; + this.storageService = this.servicesContainer.storageService; + this._pluginService = this.servicesContainer.pluginService; + this.hostListProviderService = this.servicesContainer.hostListProviderService; + this.refreshRateNs = refreshRateNs; + this.highRefreshRateNs = highRefreshRateNs; this._monitoringProperties = new Map(props); for (const [key, val] of props) { - if (key.startsWith(ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX)) { - this._monitoringProperties.set(key.substring(ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX.length), val); + if (key.startsWith(WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX)) { + this._monitoringProperties.set(key.substring(WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX.length), val); this._monitoringProperties.delete(key); } } - this.untrackedPromises.push(this.run()); - } - get hostListProvider(): MonitoringRdsHostListProvider { - return this._hostListProvider; + const connectTimeout = + this._monitoringProperties.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name) ?? ClusterTopologyMonitorImpl.DEFAULT_CONNECTION_TIMEOUT_MS; + const queryTimeout = + this._monitoringProperties.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name) ?? ClusterTopologyMonitorImpl.DEFAULT_QUERY_TIMEOUT_MS; + const driverDialect = this._pluginService.getDriverDialect(); + driverDialect.setConnectTimeout(this._monitoringProperties, connectTimeout); + driverDialect.setQueryTimeout(this._monitoringProperties, undefined, queryTimeout); } get pluginService(): PluginService { @@ -121,27 +151,35 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } async close(): Promise { - this.stopMonitoring = true; this.hostMonitorsStop = true; this.requestToUpdateTopology = true; - await Promise.all(this.untrackedPromises); - await this.closeConnection(this.monitoringClient); - await this.closeConnection(this.hostMonitorsWriterClient); - await this.closeConnection(this.hostMonitorsReaderClient); - this.untrackedPromises = []; + await Promise.all(this.submittedHosts.values()); + + const monitoringClientToClose = this.monitoringClient; + const hostMonitorsWriterClientToClose = this.hostMonitorsWriterClient; + const hostMonitorsReaderClientToClose = this.hostMonitorsReaderClient; + + this.monitoringClient = null; + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + + await this.closeConnection(monitoringClientToClose); + if (hostMonitorsWriterClientToClose && hostMonitorsWriterClientToClose !== monitoringClientToClose) { + await this.closeConnection(hostMonitorsWriterClientToClose); + } + if ( + hostMonitorsReaderClientToClose && + hostMonitorsReaderClientToClose !== monitoringClientToClose && + hostMonitorsReaderClientToClose !== hostMonitorsWriterClientToClose + ) { + await this.closeConnection(hostMonitorsReaderClientToClose); + } + + this.submittedHosts.clear(); this.hostMonitors.clear(); } async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { - if (Date.now() < this.ignoreNewTopologyRequestsEndTimeMs) { - // Previous failover has just completed, use results without triggering new update. - const currentHosts = this.getStoredHosts(); - if (currentHosts !== null) { - logger.info(Messages.get("ClusterTopologyMonitoring.ignoringNewTopologyRequest")); - return currentHosts; - } - } - if (shouldVerifyWriter) { this.isVerifiedWriterConnection = false; if (this.monitoringClient) { @@ -157,15 +195,16 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { async forceRefresh(client: ClientWrapper, timeoutMs: number): Promise { if (this.isVerifiedWriterConnection) { + // Get the monitoring task to refresh the topology using a verified connection. return await this.waitTillTopologyGetsUpdated(timeoutMs); } - // Otherwise use provided unverified connection to update topology. + // Otherwise, use the provided unverified connection to update the topology. return await this.fetchTopologyAndUpdateCache(client); } async waitTillTopologyGetsUpdated(timeoutMs: number): Promise { - // Signal to any monitor that might be in delay, that topology should be updated. + // Notify the monitoring task, which may be sleeping, that topology should be refreshed immediately. this.requestToUpdateTopology = true; const currentHosts: HostInfo[] = this.getStoredHosts(); @@ -183,7 +222,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } if (Date.now() >= endTime) { - throw new AwsWrapperError(Messages.get("ClusterTopologyMonitor.timeoutError", timeoutMs.toString())); + throw new AwsTimeoutError(Messages.get("ClusterTopologyMonitor.timeoutError", timeoutMs.toString())); } return latestHosts; } @@ -194,7 +233,7 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } try { - const hosts: HostInfo[] = await this._hostListProvider.sqlQueryForTopology(client); + const hosts: HostInfo[] = await this.queryForTopology(client); if (hosts) { this.updateTopologyCache(hosts); } @@ -206,12 +245,11 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { } private async openAnyClientAndUpdateTopology(): Promise { - let writerVerifiedByThisTask = false; if (!this.monitoringClient) { let client: ClientWrapper; try { - client = await this._pluginService.forceConnect(this.initialHostInfo, this._monitoringProperties); - } catch { + client = await this.servicesContainer.pluginService.forceConnect(this.initialHostInfo, this._monitoringProperties); + } catch (connectError) { // Unable to connect to host; return null; } @@ -226,7 +264,6 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { if (this.rdsUtils.isRdsInstance(this.initialHostInfo.host)) { this.writerHostInfo = this.initialHostInfo; logger.info(Messages.get("ClusterTopologyMonitor.writerMonitoringConnection", this.writerHostInfo.host)); - writerVerifiedByThisTask = true; } else { const pair: [string, string] = await this.topologyUtils.getInstanceId(this.monitoringClient); const instanceTemplate: HostInfo = await this.getInstanceTemplate(pair[1], this.monitoringClient); @@ -238,20 +275,13 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { // Do nothing. logger.error(Messages.get("ClusterTopologyMonitor.invalidWriterQuery", error?.message)); } - } else { + } else if (client) { // Monitoring connection already set by another task, close the new connection. await this.closeConnection(client); } } const hosts: HostInfo[] = await this.fetchTopologyAndUpdateCache(this.monitoringClient); - if (writerVerifiedByThisTask) { - if (this.ignoreNewTopologyRequestsEndTimeMs === -1) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } else { - this.ignoreNewTopologyRequestsEndTimeMs = Date.now() + this.ignoreTopologyRequestMs; - } - } if (hosts === null) { this.isVerifiedWriterConnection = false; @@ -264,294 +294,530 @@ export class ClusterTopologyMonitorImpl implements ClusterTopologyMonitor { return Promise.resolve(this.instanceTemplate); } + queryForTopology(client: ClientWrapper): Promise { + return this.topologyUtils.queryForTopology(client, this.pluginService.getDialect(), this.initialHostInfo, this.instanceTemplate); + } + + updateHostsAvailability(hosts: HostInfo[]): void { + if (!hosts) { + return; + } + + hosts.forEach((host) => { + host.setAvailability(this.readerTopologiesById.has(host.hostId) ? HostAvailability.AVAILABLE : HostAvailability.NOT_AVAILABLE); + }); + } + updateTopologyCache(hosts: HostInfo[]): void { this.storageService.set(this.clusterId, new Topology(hosts)); this.requestToUpdateTopology = false; } - async getWriterHostIdIfConnected(client: ClientWrapper, hostId: string): Promise { - const writerHost: string = await this.hostListProvider.getWriterId(client); - // Returns the hostId of the writer if client is connected to that writer, otherwise returns null. - return writerHost === hostId ? writerHost : null; + protected clearTopologyCache(): void { + this.servicesContainer.storageService.remove(Topology, this.clusterId); } - async closeConnection(client: ClientWrapper): Promise { - if (client !== null) { - await client.abort(); - client = null; - } + async closeConnection(client: ClientWrapper | null): Promise { + await client?.abort(); } async updateMonitoringClient(newClient: ClientWrapper | null): Promise { const clientToClose = this.monitoringClient; this.monitoringClient = newClient; - if (clientToClose) { - await clientToClose.abort(); - } + await clientToClose?.abort(); } - private isInPanicMode(): boolean { - return !this.monitoringClient || !this.isVerifiedWriterConnection; + async stop(): Promise { + this._stop = true; + this.hostMonitorsStop = true; + + await Promise.all(this.submittedHosts.values()); + + await this.closeHostMonitors(); + + const hostMonitorsWriterClientToClose = this.hostMonitorsWriterClient; + const hostMonitorsReaderClientToClose = this.hostMonitorsReaderClient; + const monitoringClientToClose = this.monitoringClient; + + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + this.monitoringClient = null; + + await this.closeConnection(hostMonitorsWriterClientToClose); + if (hostMonitorsReaderClientToClose && hostMonitorsReaderClientToClose !== hostMonitorsWriterClientToClose) { + await this.closeConnection(hostMonitorsReaderClientToClose); + } + if ( + monitoringClientToClose && + monitoringClientToClose !== hostMonitorsWriterClientToClose && + monitoringClientToClose !== hostMonitorsReaderClientToClose + ) { + await this.closeConnection(monitoringClientToClose); + } + + this.submittedHosts.clear(); + + return super.stop(); } - async run(): Promise { - logger.debug(Messages.get("ClusterTopologyMonitor.startMonitoring")); + async monitor(): Promise { try { - while (!this.stopMonitoring) { + logger.debug(Messages.get("ClusterTopologyMonitor.startMonitoring", this.clusterId, this.initialHostInfo.host)); + this.servicesContainer.eventPublisher.subscribe(this, new Set([MonitorResetEvent])); + + while (!this._stop) { + this.lastActivityTimestampNanos = getTimeInNanos(); + if (this.isInPanicMode()) { - // Panic Mode: high refresh rate in effect. + if (this.submittedHosts.size === 0) { + logger.debug(Messages.get("ClusterTopologyMonitor.startingHostMonitoringTasks")); - if (this.hostMonitors.size === 0) { - // Initialize host tasks. - logger.debug(Messages.get("ClusterTopologyMonitor.startingHostMonitors")); + // Start host monitoring tasks. this.hostMonitorsStop = false; - if (this.hostMonitorsReaderClient !== null) { - await this.closeConnection(this.hostMonitorsReaderClient); - } - if (this.hostMonitorsWriterClient !== null) { - await this.closeConnection(this.hostMonitorsWriterClient); - } - this.hostMonitorsWriterClient = null; - this.hostMonitorsReaderClient = null; + await this.hostMonitorClientCleanUp(); this.hostMonitorsWriterInfo = null; this.hostMonitorsLatestTopology = []; - // Use any client to gather topology information. let hosts: HostInfo[] = this.getStoredHosts(); - if (!hosts) { + if (hosts === null) { + // Use any available connection to get the topology. hosts = await this.openAnyClientAndUpdateTopology(); } - // Set up host monitors. - if (hosts && !this.isVerifiedWriterConnection) { - for (const hostInfo of hosts) { - if (!this.hostMonitors.has(hostInfo.host)) { - const hostMonitor = new HostMonitor(this, hostInfo, this.writerHostInfo); - const hostRun = hostMonitor.run(); - this.hostMonitors.set(hostInfo.host, hostMonitor); - this.untrackedPromises.push(hostRun); - } + await this.closeHostMonitors(); + + if (!(hosts !== null && !this.isVerifiedWriterConnection)) { + await this.delay(true); + continue; + } + + for (const hostInfo of hosts) { + if (!this.submittedHosts.get(hostInfo.host)) { + const minimalServiceContainer = ServiceUtils.instance.createMinimalServiceContainerFrom( + this.servicesContainer, + this._monitoringProperties + ); + await minimalServiceContainer.pluginManager.init(); + const hostMonitor = new HostMonitor(minimalServiceContainer, this, hostInfo, this.writerHostInfo); + const promise = hostMonitor.run(); + this.submittedHosts.set(hostInfo.host, promise); } } - // If topology is not correctly updated, will try on the next round. + + // We will try again in the next iteration. } else { - // Host monitors already running, check if a writer has been detected. - const writerClient = this.hostMonitorsWriterClient; - const writerHostInfo = this.hostMonitorsWriterInfo; - if (writerClient && writerHostInfo && writerHostInfo !== this.writerHostInfo) { - // Writer detected, update monitoringClient. - logger.info(Messages.get("ClusterTopologyMonitor.writerPickedUpFromHostMonitors", writerHostInfo.hostId)); - await this.updateMonitoringClient(writerClient); - this.writerHostInfo = writerHostInfo; + // The host monitors are running, so we check if the writer has been detected. + const writerClient: ClientWrapper | null = this.hostMonitorsWriterClient; + const writerClientHostInfo: HostInfo | null = this.hostMonitorsWriterInfo; + + if (writerClient && writerClientHostInfo) { + logger.debug(Messages.get("ClusterTopologyMonitor.writerPickedUpFromHostMonitors", writerClientHostInfo.toString())); + + this.monitoringClient = writerClient; + this.writerHostInfo = writerClientHostInfo; this.isVerifiedWriterConnection = true; - if (this.ignoreNewTopologyRequestsEndTimeMs === -1) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } else { - this.ignoreNewTopologyRequestsEndTimeMs = Date.now() + this.ignoreTopologyRequestMs; - } - if (this.highRefreshRateEndTimeMs === -1) { - this.highRefreshRateEndTimeMs = 0; - } else { - this.highRefreshRateEndTimeMs = Date.now() + this.highRefreshPeriodAfterPanicMs; - } + this.highRefreshRateEndTimeNs = getTimeInNanos() + BigInt(this.highRefreshRateNs); - // Stop monitoring of each host, writer detected. this.hostMonitorsStop = true; - this.hostMonitors.clear(); + await this.closeHostMonitors(); + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); + + await this.delay(true); continue; } else { - // No writer detected, update host monitors with any new hosts in the topology. - const hosts: HostInfo[] = this.hostMonitorsLatestTopology; - if (hosts !== null && !this.hostMonitorsStop) { + // Update host monitors with the new instances in the topology. + const hosts: HostInfo[] | null = this.hostMonitorsLatestTopology; + if (hosts && !this.hostMonitorsStop) { for (const hostInfo of hosts) { - if (!this.hostMonitors.has(hostInfo.host)) { - const hostMonitor = new HostMonitor(this, hostInfo, this.writerHostInfo); - const hostRun = hostMonitor.run(); - this.hostMonitors.set(hostInfo.host, hostMonitor); - this.untrackedPromises.push(hostRun); + if (!this.submittedHosts.get(hostInfo.host)) { + const minimalServiceContainer = ServiceUtils.instance.createMinimalServiceContainerFrom( + this.servicesContainer, + this._monitoringProperties + ); + await minimalServiceContainer.pluginManager.init(); + // Intentionally not calling await on hostMonitor.run(). + const hostMonitor = new HostMonitor(minimalServiceContainer, this, hostInfo, this.writerHostInfo); + const promise = hostMonitor.run(); + this.submittedHosts.set(hostInfo.host, promise); } } } } } - // Trigger a delay before retrying. + + this.checkForStableReaderTopologies(); await this.delay(true); } else { - // Regular mode: lower refresh rate than panic mode. - if (this.hostMonitors.size !== 0) { - // Stop host monitors. - this.hostMonitorsStop = true; - this.hostMonitors.clear(); + // We are in regular mode. + if (this.submittedHosts.size !== 0) { + await this.closeHostMonitors(); + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); } - const hosts = await this.fetchTopologyAndUpdateCache(this.monitoringClient); + + const hosts: HostInfo[] = await this.fetchTopologyAndUpdateCache(this.monitoringClient); if (hosts === null) { - // Unable to gather topology, switch to panic mode. + // Attempt to fetch topology failed, so we switch to panic mode. + const clientToClose = this.monitoringClient; + this.monitoringClient = null; + await this.closeConnection(clientToClose); this.isVerifiedWriterConnection = false; - await this.updateMonitoringClient(null); + this.writerHostInfo = null; + await this.delay(false); continue; } - if (this.highRefreshRateEndTimeMs > 0 && Date.now() > this.highRefreshRateEndTimeMs) { - this.highRefreshRateEndTimeMs = 0; + + if (this.highRefreshRateEndTimeNs > 0 && getTimeInNanos() > this.highRefreshRateEndTimeNs) { + this.highRefreshRateEndTimeNs = BigInt(0); } - if (this.highRefreshRateEndTimeMs < 0) { - // Log topology when not in high refresh rate. - this.logTopology(`[clusterTopologyMonitor] `); + + // We avoid logging the topology while using the high refresh rate because it is too noisy. + if (this.highRefreshRateEndTimeNs === BigInt(0)) { + logger.debug(logTopology(this.getStoredHosts(), "")); } - // Set an easily interruptible delay between topology refreshes. + await this.delay(false); } - if (this.ignoreNewTopologyRequestsEndTimeMs > 0 && Date.now() > this.ignoreNewTopologyRequestsEndTimeMs) { - this.ignoreNewTopologyRequestsEndTimeMs = 0; - } } - } catch (error) { - logger.error(Messages.get("ClusterTopologyMonitor.errorDuringMonitoring", error?.message)); } finally { - this.stopMonitoring = true; - await this.updateMonitoringClient(null); - logger.debug(Messages.get("ClusterTopologyMonitor.endMonitoring")); + this._stop = true; + await this.closeHostMonitors(); + await this.hostMonitorClientCleanUp(); + + this.servicesContainer.eventPublisher.unsubscribe(this, new Set([MonitorResetEvent])); + + logger.debug(Messages.get("ClusterTopologyMonitor.stopHostMonitoringTask", this.initialHostInfo.host)); } + + return Promise.resolve(); + } + + protected checkForStableReaderTopologies(): void { + const latestHosts: HostInfo[] = this.getStoredHosts(); + if (!latestHosts || latestHosts.length === 0) { + this.stableTopologiesStartNs = BigInt(0); + return; + } + + const readerIds: string[] = latestHosts.map((host) => host.hostId); + for (const id of readerIds) { + const completedCycle = this.completedOneCycle.get(id) ?? false; + if (!completedCycle) { + // Not all reader monitors have completed a cycle. We shouldn't conclude that reader topologies are stable until + // each reader monitor has made at least one attempt to fetch topology information, even if unsuccessful. + this.stableTopologiesStartNs = BigInt(0); + return; + } + } + + const readerTopologyValues = Array.from(this.readerTopologiesById.values()); + const readerTopology: HostInfo[] | undefined = readerTopologyValues.length > 0 ? readerTopologyValues[0] : undefined; + if (!readerTopology) { + // readerTopologiesById has been cleared since checking its size. + this.stableTopologiesStartNs = BigInt(0); + return; + } + + // Check whether the topologies match. HostInfos are compared using their host, port, role, and availability fields. + // Using the first HostInfo in the topology as the reference. + // Note that monitors that encounter errors will remove their entry from the map, so only entries from + // successful monitors are checked. + const reference = JSON.stringify(readerTopology.map(this.hostInfoExtractor).sort()); + const allTopologiesMatch = readerTopologyValues.every((hosts) => JSON.stringify(hosts.map(this.hostInfoExtractor).sort()) === reference); + + if (!allTopologiesMatch) { + // The topologies detected by each reader do not match. + this.stableTopologiesStartNs = BigInt(0); + return; + } + + // All reader topologies match. + if (this.stableTopologiesStartNs === BigInt(0)) { + this.stableTopologiesStartNs = getTimeInNanos(); + } + + if (getTimeInNanos() > this.stableTopologiesStartNs + ClusterTopologyMonitorImpl.STABLE_TOPOLOGIES_DURATION_NS) { + // Reader topologies have been consistent for STABLE_TOPOLOGIES_DURATION_NS, so the topology should be accurate. + this.stableTopologiesStartNs = BigInt(0); + this.updateHostsAvailability(readerTopology); + logger.debug( + logTopology( + readerTopology, + Messages.get( + "ClusterTopologyMonitor.matchingReaderTopologies", + String(convertNanosToMs(ClusterTopologyMonitorImpl.STABLE_TOPOLOGIES_DURATION_NS)) + ) + ) + ); + this.updateTopologyCache(readerTopology); + } + } + + protected async reset(): Promise { + logger.debug(Messages.get("ClusterTopologyMonitor.reset", this.clusterId, this.initialHostInfo.host)); + + this.hostMonitorsStop = true; + await this.closeHostMonitors(); + await this.hostMonitorClientCleanUp(); + this.hostMonitorsStop = false; + this.submittedHosts.clear(); + this.stableTopologiesStartNs = BigInt(0); + this.readerTopologiesById.clear(); + this.completedOneCycle.clear(); + + this.hostMonitorsWriterInfo = null; + this.hostMonitorsLatestTopology = []; + + await this.updateMonitoringClient(null); + this.isVerifiedWriterConnection = false; + this.writerHostInfo = null; + this.highRefreshRateEndTimeNs = BigInt(0); + this.requestToUpdateTopology = false; + this.clearTopologyCache(); + + // This breaks any waiting/sleeping cycles in the monitoring task. + this.requestToUpdateTopology = true; + } + + async processEvent(event: Event): Promise { + if (event instanceof MonitorResetEvent) { + logger.debug(Messages.get("ClusterTopologyMonitor.resetEventReceived")); + const resetEvent = event as MonitorResetEvent; + if (resetEvent.clusterId === this.clusterId) { + await this.reset(); + } + } + } + + protected async hostMonitorClientCleanUp(): Promise { + const writerClientToClose = this.hostMonitorsWriterClient; + const readerClientToClose = this.hostMonitorsReaderClient; + + this.hostMonitorsWriterClient = null; + this.hostMonitorsReaderClient = null; + + if (writerClientToClose && this.monitoringClient !== writerClientToClose) { + try { + await this.closeConnection(writerClientToClose); + } catch (e: any) { + // Ignore + } + } + + if (readerClientToClose && this.monitoringClient !== readerClientToClose && writerClientToClose !== readerClientToClose) { + try { + await this.closeConnection(readerClientToClose); + } catch (e: any) { + // Ignore + } + } + } + + protected async closeHostMonitors(): Promise { + await Promise.all(this.submittedHosts.values()); + this.submittedHosts.clear(); + await this.hostMonitorClientCleanUp(); + } + + private isInPanicMode(): boolean { + return !this.monitoringClient || !this.isVerifiedWriterConnection; } private getStoredHosts(): HostInfo[] | null { - const topology = this.storageService.get(Topology, this.clusterId); - return topology == null ? null : topology.hosts; + return this.storageService.get(Topology, this.clusterId)?.hosts ?? null; } - private async delay(useHighRefreshRate: boolean) { - if (Date.now() < this.highRefreshRateEndTimeMs) { + private async delay(useHighRefreshRate: boolean): Promise { + if (getTimeInNanos() < this.highRefreshRateEndTimeNs) { useHighRefreshRate = true; } - const endTime = Date.now() + (useHighRefreshRate ? this.highRefreshRateMs : this.refreshRateMs); + const delayNs = useHighRefreshRate ? this.highRefreshRateNs : this.refreshRateNs; + const endTime: bigint = getTimeInNanos() + BigInt(delayNs); await sleep(50); - while (Date.now() < endTime && !this.requestToUpdateTopology) { + while (getTimeInNanos() < endTime && !this.requestToUpdateTopology && !this._stop) { await sleep(50); } } - - logTopology(msgPrefix: string) { - const hosts: HostInfo[] = this.getStoredHosts(); - if (hosts && hosts.length !== 0) { - logger.debug(logTopology(hosts, msgPrefix)); - } - } } export class HostMonitor { + private static readonly INITIAL_BACKOFF_MS = 100; + private static readonly MAX_BACKOFF_MS = 10000; + + protected readonly servicesContainer: FullServicesContainer; protected readonly monitor: ClusterTopologyMonitorImpl; protected readonly hostInfo: HostInfo; - protected readonly writerHostInfo: HostInfo; + protected readonly writerHostInfo: HostInfo | null; protected writerChanged: boolean = false; + protected connectionAttempts: number = 0; + protected client: ClientWrapper | null = null; - constructor(monitor: ClusterTopologyMonitorImpl, hostInfo: HostInfo, writerHostInfo: HostInfo) { + constructor(servicesContainer: FullServicesContainer, monitor: ClusterTopologyMonitorImpl, hostInfo: HostInfo, writerHostInfo: HostInfo | null) { + this.servicesContainer = servicesContainer; this.monitor = monitor; this.hostInfo = hostInfo; this.writerHostInfo = writerHostInfo; } async run() { - let client: ClientWrapper | null = null; let updateTopology: boolean = false; const startTime: number = Date.now(); logger.debug(Messages.get("HostMonitor.startMonitoring", this.hostInfo.hostId)); + const pluginService = this.servicesContainer.pluginService; try { while (!this.monitor.hostMonitorsStop) { - if (!client) { + if (!this.client) { try { - client = await this.monitor.pluginService.forceConnect(this.hostInfo, this.monitor.monitoringProperties); - this.monitor.pluginService.setAvailability(this.hostInfo.allAliases, HostAvailability.AVAILABLE); + this.client = await pluginService.forceConnect(this.hostInfo, this.monitor.monitoringProperties); + this.connectionAttempts = 0; } catch (error) { - this.monitor.pluginService.setAvailability(this.hostInfo.allAliases, HostAvailability.NOT_AVAILABLE); + // A problem occurred while connecting. + if (pluginService.isNetworkError(error)) { + // It's a network issue that's expected during a cluster failover. + // We will try again on the next iteration. + await sleep(100); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; + } else if (pluginService.isLoginError(error)) { + throw new AwsWrapperError(Messages.get("HostMonitor.loginErrorDuringMonitoring"), error); + } else { + // It might be some transient error. Let's try again. + // If the error repeats, we will try again after a longer delay. + const backoff = this.calculateBackoffWithJitter(this.connectionAttempts++); + await sleep(backoff); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; + } } } - if (client) { - let writerId = null; + if (this.client) { + let isWriter: boolean = false; try { - writerId = await this.monitor.getWriterHostIdIfConnected(client, this.hostInfo.hostId); + isWriter = await this.monitor.topologyUtils.isWriterInstance(this.client); } catch (error) { logger.error(Messages.get("ClusterTopologyMonitor.invalidWriterQuery", error?.message)); - await this.monitor.closeConnection(client); - client = null; + await this.monitor.closeConnection(this.client); + this.client = null; } - if (writerId) { - // First connection after failover may be stale. - if ((await this.monitor.pluginService.getHostRole(client)) !== HostRole.WRITER) { - logger.debug(Messages.get("HostMonitor.writerIsStale", writerId)); - writerId = null; + if (isWriter) { + try { + // First connection after failover may be stale. + const hostRole = await this.monitor.pluginService.getHostRole(this.client); + if (hostRole !== HostRole.WRITER) { + isWriter = false; + } + } catch (error: any) { + // Invalid connection, retry. + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + continue; } } - if (writerId) { + if (isWriter) { + // This prevents us from closing the connection in the finally block. if (this.monitor.hostMonitorsWriterClient) { - await this.monitor.closeConnection(client); + // The writer connection is already set up, probably by another host monitor. + await this.monitor.closeConnection(this.client); } else { - logger.debug(Messages.get("HostMonitor.detectedWriter", writerId, this.hostInfo.host)); - const updatedHosts: HostInfo[] = await this.monitor.fetchTopologyAndUpdateCache(client); - if (updatedHosts && this.monitor.hostMonitorsWriterClient === null) { - this.monitor.hostMonitorsWriterClient = client; - this.monitor.hostMonitorsWriterInfo = this.hostInfo; - this.monitor.hostMonitorsStop = true; - this.monitor.logTopology(`[hostMonitor ${this.hostInfo.hostId}] `); - } else { - await this.monitor.closeConnection(client); - } + // Successfully updated the host monitor writer connection. + logger.debug(Messages.get("HostMonitor.detectedWriter", this.hostInfo.hostId, this.hostInfo.url)); + + this.servicesContainer.importantEventService.registerEvent(() => + Messages.get("HostMonitor.detectedWriter", this.hostInfo.hostId, this.hostInfo.url) + ); + + await this.monitor.fetchTopologyAndUpdateCache(this.client); + this.hostInfo.setAvailability(HostAvailability.AVAILABLE); + this.monitor.hostMonitorsWriterClient = this.client; + this.monitor.hostMonitorsWriterInfo = this.hostInfo; + // Connection is already assigned to this.monitor.hostMonitorsWriterClient + // so we need to reset client without closing it. + this.client = null; + this.monitor.hostMonitorsStop = true; + logger.debug(logTopology(this.monitor.hostMonitorsLatestTopology, `[hostMonitor ${this.hostInfo.hostId}] `)); } - client = null; return; - } else if (client) { + } else if (this.client) { // Client is a reader. if (!this.monitor.hostMonitorsWriterClient) { - // While the writer hasn't been identified, reader client can update topology. + // We can use this reader connection to update the topology while we wait for the writer connection to + // be established. if (updateTopology) { - await this.readerTaskFetchTopology(client, this.writerHostInfo); - } else if (this.monitor.hostMonitorsReaderClient === null) { - this.monitor.hostMonitorsReaderClient = client; + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); + } else if (!this.monitor.hostMonitorsReaderClient) { + this.monitor.hostMonitorsReaderClient = this.client; updateTopology = true; - await this.readerTaskFetchTopology(client, this.writerHostInfo); + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); + } else { + await this.readerTaskFetchTopology(this.client, this.writerHostInfo); } } } } + + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); await sleep(100); } } catch (error) { // Close the monitor. } finally { - await this.monitor.closeConnection(client); + this.monitor.completedOneCycle.set(this.hostInfo.hostId, true); + this.monitor.readerTopologiesById.delete(this.hostInfo.hostId); + + await this.monitor.closeConnection(this.client); logger.debug(Messages.get("HostMonitor.endMonitoring", this.hostInfo.hostId, (Date.now() - startTime).toString())); } } - private async readerTaskFetchTopology(client: any, writerHostInfo: HostInfo) { + private async readerTaskFetchTopology(client: ClientWrapper, writerHostInfo: HostInfo | null) { if (!client) { return; } - let hosts: HostInfo[]; + let hosts: HostInfo[] | null; try { - hosts = await this.monitor.hostListProvider.sqlQueryForTopology(client); - if (hosts === null) { + hosts = await this.monitor.queryForTopology(client); + if (!hosts) { return; } - this.monitor.hostMonitorsLatestTopology = hosts; } catch (error) { return; } + // Share this topology so that the main monitoring task can adjust the node monitoring tasks. + this.monitor.hostMonitorsLatestTopology = hosts; + this.monitor.readerTopologiesById.set(this.hostInfo.hostId, hosts); + if (this.writerChanged) { + this.monitor.updateHostsAvailability(hosts); this.monitor.updateTopologyCache(hosts); logger.debug(logTopology(hosts, `[hostMonitor ${this.hostInfo.hostId}] `)); return; } - const latestWriterHostInfo: HostInfo = hosts.find((x) => x.role === HostRole.WRITER); + const latestWriterHostInfo = hosts.find((x) => x.role === HostRole.WRITER); if (latestWriterHostInfo && writerHostInfo && latestWriterHostInfo.hostAndPort !== writerHostInfo.hostAndPort) { this.writerChanged = true; logger.debug(Messages.get("HostMonitor.writerHostChanged", writerHostInfo.hostAndPort, latestWriterHostInfo.hostAndPort)); + this.monitor.updateHostsAvailability(hosts); this.monitor.updateTopologyCache(hosts); logger.debug(logTopology(hosts, `[hostMonitor ${this.hostInfo.hostId}] `)); } } + + private calculateBackoffWithJitter(attempt: number): number { + let backoff = HostMonitor.INITIAL_BACKOFF_MS * Math.round(Math.pow(2, Math.min(attempt, 6))); + backoff = Math.min(backoff, HostMonitor.MAX_BACKOFF_MS); + return Math.round(backoff * (0.5 + Math.random() * 0.5)); + } } diff --git a/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts b/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts new file mode 100644 index 00000000..9582c522 --- /dev/null +++ b/common/lib/host_list_provider/monitoring/global_aurora_topology_monitor.ts @@ -0,0 +1,69 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClusterTopologyMonitorImpl } from "./cluster_topology_monitor"; +import { GdbTopologyUtils, GlobalTopologyUtils } from "../global_topology_utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { HostInfo } from "../../host_info"; +import { ClientWrapper } from "../../client_wrapper"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { TopologyUtils } from "../topology_utils"; + +function isGdbTopologyUtils(utils: TopologyUtils): utils is TopologyUtils & GdbTopologyUtils { + return "getRegion" in utils && typeof (utils as unknown as GdbTopologyUtils).getRegion === "function"; +} + +export class GlobalAuroraTopologyMonitor extends ClusterTopologyMonitorImpl { + protected readonly instanceTemplatesByRegion: Map; + declare public readonly topologyUtils: TopologyUtils; + + constructor( + servicesContainer: FullServicesContainer, + topologyUtils: TopologyUtils, + clusterId: string, + initialHostInfo: HostInfo, + properties: Map, + instanceTemplate: HostInfo, + refreshRateNano: number, + highRefreshRateNano: number, + instanceTemplatesByRegion: Map + ) { + super(servicesContainer, topologyUtils, clusterId, initialHostInfo, properties, instanceTemplate, refreshRateNano, highRefreshRateNano); + + this.instanceTemplatesByRegion = instanceTemplatesByRegion; + this.topologyUtils = topologyUtils; + } + + protected override async getInstanceTemplate(hostId: string, targetClient: ClientWrapper): Promise { + if (!isGdbTopologyUtils(this.topologyUtils)) { + throw new AwsWrapperError(Messages.get("GlobalAuroraTopologyMonitor.invalidTopologyUtils")); + } + + const dialect = this.hostListProviderService.getDialect(); + const region = await this.topologyUtils.getRegion(hostId, targetClient, dialect); + + if (region) { + const instanceTemplate = this.instanceTemplatesByRegion.get(region); + if (!instanceTemplate) { + throw new AwsWrapperError(Messages.get("GlobalAuroraTopologyMonitor.cannotFindRegionTemplate", region)); + } + return instanceTemplate; + } + + return this.instanceTemplate; + } +} diff --git a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts b/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts deleted file mode 100644 index c3f9b87d..00000000 --- a/common/lib/host_list_provider/monitoring/monitoring_host_list_provider.ts +++ /dev/null @@ -1,112 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { RdsHostListProvider } from "../rds_host_list_provider"; -import { HostInfo, AwsWrapperError } from "../../"; -import { PluginService } from "../../plugin_service"; -import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./cluster_topology_monitor"; -import { HostListProviderService } from "../../host_list_provider_service"; -import { ClientWrapper } from "../../client_wrapper"; -import { DatabaseDialect } from "../../database_dialect/database_dialect"; -import { Messages } from "../../utils/messages"; -import { WrapperProperties } from "../../wrapper_property"; -import { BlockingHostListProvider } from "../host_list_provider"; -import { logger } from "../../../logutils"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; -import { isDialectTopologyAware } from "../../utils/utils"; -import { TopologyUtils } from "../topology_utils"; - -export class MonitoringRdsHostListProvider extends RdsHostListProvider implements BlockingHostListProvider { - static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(60_000_000_000); // 1 minute. - static readonly MONITOR_EXPIRATION_NANOS: bigint = BigInt(15 * 60_000_000_000); // 15 minutes. - static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds. - - private static monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitoringRdsHostListProvider.CACHE_CLEANUP_NANOS, - () => true, - async (item: ClusterTopologyMonitor) => { - try { - await item.close(); - } catch { - // Ignore. - } - }, - "MonitoringRdsHostListProvider.monitors" - ); - - private readonly pluginService: PluginService; - - constructor( - properties: Map, - originalUrl: string, - topologyUtils: TopologyUtils, - hostListProviderService: HostListProviderService, - pluginService: PluginService - ) { - super(properties, originalUrl, topologyUtils, hostListProviderService); - this.pluginService = pluginService; - } - - async clearAll(): Promise { - RdsHostListProvider.clearAll(); - await MonitoringRdsHostListProvider.monitors.clear(); - } - - async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { - const monitor: ClusterTopologyMonitor = this.initMonitor(); - - try { - return await monitor.forceRefresh(targetClient, MonitoringRdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); - } catch (error) { - logger.info(Messages.get("MonitoringHostListProvider.errorForceRefresh", error.message)); - return null; - } - } - - async sqlQueryForTopology(targetClient: ClientWrapper): Promise { - return await this.topologyUtils.queryForTopology(targetClient, this.pluginService.getDialect(), this.initialHost, this.clusterInstanceTemplate); - } - - async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { - const monitor: ClusterTopologyMonitor = this.initMonitor(); - - return await monitor.forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); - } - - protected initMonitor(): ClusterTopologyMonitor { - const monitor: ClusterTopologyMonitor = MonitoringRdsHostListProvider.monitors.computeIfAbsent( - this.clusterId, - () => - new ClusterTopologyMonitorImpl( - this.topologyUtils, - this.clusterId, - this.initialHost, - this.properties, - this.clusterInstanceTemplate, - this.pluginService, - this, - WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties), - WrapperProperties.CLUSTER_TOPOLOGY_HIGH_REFRESH_RATE_MS.get(this.properties) - ), - MonitoringRdsHostListProvider.MONITOR_EXPIRATION_NANOS - ); - - if (monitor === null) { - throw new AwsWrapperError(Messages.get("MonitoringHostListProvider.requiresMonitor")); - } - return monitor; - } -} diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index aae69e4f..9558e1aa 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -21,29 +21,36 @@ import { RdsUrlType } from "../utils/rds_url_type"; import { RdsUtils } from "../utils/rds_utils"; import { HostListProviderService } from "../host_list_provider_service"; import { ConnectionUrlParser } from "../utils/connection_url_parser"; -import { AwsWrapperError } from "../utils/errors"; +import { AwsTimeoutError, AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; import { WrapperProperties } from "../wrapper_property"; import { logger } from "../../logutils"; -import { isDialectTopologyAware, logTopology } from "../utils/utils"; +import { isDialectTopologyAware } from "../database_dialect/topology_aware_database_dialect"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { ClientWrapper } from "../client_wrapper"; import { CoreServicesContainer } from "../utils/core_services_container"; import { StorageService } from "../utils/storage/storage_service"; import { Topology } from "./topology"; import { TopologyUtils } from "./topology_utils"; +import { FullServicesContainer } from "../utils/full_services_container"; +import { PluginService } from "../plugin_service"; +import { ClusterTopologyMonitor, ClusterTopologyMonitorImpl } from "./monitoring/cluster_topology_monitor"; +import { MonitorInitializer } from "../utils/monitoring/monitor"; export class RdsHostListProvider implements DynamicHostListProvider { + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS: number = 5000; private readonly originalUrl: string; - private readonly rdsHelper: RdsUtils; + protected readonly rdsHelper: RdsUtils; + protected readonly servicesContainers: FullServicesContainer; + private readonly pluginService: PluginService; private readonly storageService: StorageService; protected readonly topologyUtils: TopologyUtils; protected readonly properties: Map; private rdsUrlType: RdsUrlType; private initialHostList: HostInfo[]; protected initialHost: HostInfo; - private refreshRateNano: number; - private hostList?: HostInfo[]; + protected refreshRateNano: number; + protected highRefreshRateNano: number; protected readonly connectionUrlParser: ConnectionUrlParser; protected readonly hostListProviderService: HostListProviderService; @@ -51,18 +58,34 @@ export class RdsHostListProvider implements DynamicHostListProvider { public isInitialized: boolean = false; public clusterInstanceTemplate?: HostInfo; - constructor(properties: Map, originalUrl: string, topologyUtils: TopologyUtils, hostListProviderService: HostListProviderService) { + constructor(properties: Map, originalUrl: string, topologyUtils: TopologyUtils, servicesContainers: FullServicesContainer) { this.rdsHelper = new RdsUtils(); this.topologyUtils = topologyUtils; - this.hostListProviderService = hostListProviderService; - this.connectionUrlParser = hostListProviderService.getConnectionUrlParser(); + this.servicesContainers = servicesContainers; + this.pluginService = this.servicesContainers.pluginService; + this.storageService = this.servicesContainers.storageService; + this.hostListProviderService = this.servicesContainers.hostListProviderService; + this.connectionUrlParser = this.hostListProviderService.getConnectionUrlParser(); this.originalUrl = originalUrl; this.properties = properties; - this.storageService = CoreServicesContainer.getInstance().getStorageService(); // TODO: store the service container instead. + this.refreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties) * 1000000; + this.highRefreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_HIGH_REFRESH_RATE_MS.get(this.properties) * 1000000; + } + + init(): void { + if (this.isInitialized) { + return; + } + + this.initSettings(); - let port = WrapperProperties.PORT.get(properties); + this.isInitialized = true; + } + + protected initSettings(): void { + let port = WrapperProperties.PORT.get(this.properties); if (port == null) { - port = hostListProviderService.getDialect().getDefaultPort(); + port = this.hostListProviderService.getDialect().getDefaultPort(); } this.initialHostList = this.connectionUrlParser.getHostsFromConnectionUrl(this.originalUrl, false, port, () => @@ -74,15 +97,8 @@ export class RdsHostListProvider implements DynamicHostListProvider { this.initialHost = this.initialHostList[0]; this.hostListProviderService.setInitialConnectionHostInfo(this.initialHost); - this.refreshRateNano = WrapperProperties.CLUSTER_TOPOLOGY_REFRESH_RATE_MS.get(this.properties) * 1000000; - this.rdsUrlType = this.rdsHelper.identifyRdsType(this.initialHost.host); - } - - init(): void { - if (this.isInitialized) { - return; - } + this.clusterId = WrapperProperties.CLUSTER_ID.get(this.properties); const hostInfoBuilder = this.hostListProviderService.getHostInfoBuilder(); this.clusterInstanceTemplate = hostInfoBuilder @@ -91,27 +107,51 @@ export class RdsHostListProvider implements DynamicHostListProvider { .build(); this.validateHostPatternSetting(this.clusterInstanceTemplate.host); + this.rdsUrlType = this.rdsHelper.identifyRdsType(this.initialHost.host); + } - this.clusterId = WrapperProperties.CLUSTER_ID.get(this.properties); + protected async getOrCreateMonitor(): Promise { + const initializer: MonitorInitializer = { + createMonitor: (servicesContainer: FullServicesContainer): ClusterTopologyMonitor => { + return new ClusterTopologyMonitorImpl( + servicesContainer, + this.topologyUtils, + this.clusterId, + this.initialHost, + this.properties, + this.clusterInstanceTemplate, + this.refreshRateNano, + this.highRefreshRateNano + ); + } + }; + + return await this.servicesContainers.monitorService.runIfAbsent( + ClusterTopologyMonitorImpl, + this.clusterId, + this.servicesContainers, + this.properties, + initializer + ); + } - this.isInitialized = true; + async forceRefresh(): Promise { + return this.forceMonitoringRefresh(false, RdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); } - async forceRefresh(): Promise; - async forceRefresh(targetClient: ClientWrapper): Promise; - async forceRefresh(targetClient?: ClientWrapper): Promise { + async forceMonitoringRefresh(verifyTopology: boolean, timeoutMs: number): Promise { this.init(); - const currentClient = targetClient ?? this.hostListProviderService.getCurrentClient().targetClient; - if (currentClient) { - const results: FetchTopologyResult = await this.getTopology(currentClient, true); - this.hostList = results.hosts; - return Array.from(this.hostList); + if (!this.pluginService.isDialectConfirmed()) { + // We need to confirm the dialect before creating a topology monitor so that it uses the correct SQL queries. + // We will return the original hosts parsed from the connections string until the dialect has been confirmed. + return this.initialHostList; } - throw new AwsWrapperError("Could not retrieve targetClient."); + + return await this.forceRefreshMonitor(verifyTopology, timeoutMs); } - async getHostRole(client: ClientWrapper, dialect: DatabaseDialect): Promise { + async getHostRole(client: ClientWrapper, _dialect: DatabaseDialect): Promise { return this.topologyUtils.getHostRole(client); } @@ -134,7 +174,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { return null; } - let topology = await this.refresh(targetClient); + let topology = await this.refresh(); let isForcedRefresh = false; if (!topology) { @@ -161,47 +201,37 @@ export class RdsHostListProvider implements DynamicHostListProvider { return matches.length === 0 ? null : matches[0]; } - async refresh(): Promise; - async refresh(targetClient: ClientWrapper): Promise; - async refresh(targetClient?: ClientWrapper): Promise { + async refresh(): Promise { this.init(); - const currentClient = targetClient ?? this.hostListProviderService.getCurrentClient().targetClient; - const results: FetchTopologyResult = await this.getTopology(currentClient, false); - logger.debug(logTopology(results.hosts, results.isCachedData ? "[From cache] " : "")); - this.hostList = results.hosts; - return this.hostList; + const results: FetchTopologyResult = await this.getTopology(); + return results.hosts; } - async getTopology(targetClient: ClientWrapper | undefined, forceUpdate: boolean): Promise { + async getTopology(): Promise { this.init(); - if (!this.clusterId) { - throw new AwsWrapperError(Messages.get("RdsHostListProvider.noClusterId")); - } - - const cachedHosts: HostInfo[] | null = this.getStoredTopology(); + const storedTopology: HostInfo[] | null = this.getStoredTopology(); - // This clusterId is a primary one and is about to create a new entry in the cache. - // When a primary entry is created it needs to be suggested for other (non-primary) entries. - // Remember a flag to do suggestion after cache is updated. - if (!cachedHosts || forceUpdate) { + if (!storedTopology) { // need to re-fetch the topology. - if (!targetClient || !(await this.hostListProviderService.isClientValid(targetClient))) { + + if (!this.pluginService.isDialectConfirmed()) { + // We need to confirm the dialect before creating a topology monitor so that it uses the correct SQL queries. + // We will return the original hosts parsed from the connections string until the dialect has been confirmed. return new FetchTopologyResult(false, this.initialHostList); } - const hosts = await this.getCurrentTopology(targetClient, this.hostListProviderService.getDialect()); + const hosts = await this.forceRefreshMonitor(false, RdsHostListProvider.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); if (hosts && hosts.length > 0) { - this.storageService.set(this.clusterId, new Topology(hosts)); return new FetchTopologyResult(false, hosts); } } - if (!cachedHosts) { + if (!storedTopology) { return new FetchTopologyResult(false, this.initialHostList); } else { - return new FetchTopologyResult(true, cachedHosts); + return new FetchTopologyResult(true, storedTopology); } } @@ -209,12 +239,16 @@ export class RdsHostListProvider implements DynamicHostListProvider { return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.clusterInstanceTemplate); } - private getHostEndpoint(hostName: string): string | null { - if (!this.clusterInstanceTemplate || !this.clusterInstanceTemplate.host) { - return null; + protected async forceRefreshMonitor(verifyTopology: boolean, timeoutMs: number): Promise { + const monitor = await this.getOrCreateMonitor(); + try { + return await monitor.forceMonitoringRefresh(verifyTopology, timeoutMs); + } catch (error) { + if (error instanceof AwsTimeoutError) { + return null; + } + throw error; } - const host = this.clusterInstanceTemplate.host; - return host.replace("?", hostName); } getStoredTopology(): HostInfo[] | null { @@ -227,18 +261,13 @@ export class RdsHostListProvider implements DynamicHostListProvider { return topology == null ? null : topology.hosts; } - static clearAll(): void { - // No-op - // TODO: remove if still not used after full service container refactoring - } - clear(): void { if (this.clusterId) { - CoreServicesContainer.getInstance().getStorageService().remove(Topology, this.clusterId); + this.servicesContainers.storageService.remove(Topology, this.clusterId); } } - private validateHostPatternSetting(hostPattern: string) { + protected validateHostPatternSetting(hostPattern: string) { if (!this.rdsHelper.isDnsPatternValid(hostPattern)) { const message: string = Messages.get("RdsHostListProvider.invalidPattern.suggestedClusterId"); logger.error(message); @@ -246,7 +275,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { } const rdsUrlType: RdsUrlType = this.rdsHelper.identifyRdsType(hostPattern); - if (rdsUrlType == RdsUrlType.RDS_PROXY) { + if (rdsUrlType == RdsUrlType.RDS_PROXY || rdsUrlType == RdsUrlType.RDS_PROXY_ENDPOINT) { const message: string = Messages.get("RdsHostListProvider.clusterInstanceHostPatternNotSupportedForRDSProxy"); logger.error(message); throw new AwsWrapperError(message); diff --git a/common/lib/host_list_provider/topology.ts b/common/lib/host_list_provider/topology.ts index 35a53460..b7d4e344 100644 --- a/common/lib/host_list_provider/topology.ts +++ b/common/lib/host_list_provider/topology.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { HostInfo } from "../host_info"; diff --git a/common/lib/host_list_provider/topology_utils.ts b/common/lib/host_list_provider/topology_utils.ts index 2542c63f..500f3046 100644 --- a/common/lib/host_list_provider/topology_utils.ts +++ b/common/lib/host_list_provider/topology_utils.ts @@ -1,23 +1,22 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { ClientWrapper } from "../client_wrapper"; import { DatabaseDialect } from "../database_dialect/database_dialect"; import { HostInfo } from "../host_info"; -import { isDialectTopologyAware } from "../utils/utils"; import { Messages } from "../utils/messages"; import { HostRole } from "../host_role"; import { HostAvailability } from "../host_availability/host_availability"; @@ -25,6 +24,11 @@ import { HostInfoBuilder } from "../host_info_builder"; import { AwsWrapperError } from "../utils/errors"; import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; +/** + * Type representing an instance template - either a single HostInfo or a Map of region to HostInfo. + */ +export type InstanceTemplate = HostInfo | Map; + /** * Options for creating a TopologyQueryResult instance. */ @@ -66,11 +70,11 @@ export class TopologyQueryResult { } /** - * A class defining utility methods that can be used to retrieve and process a variety of database topology + * An abstract class defining utility methods that can be used to retrieve and process a variety of database topology * information. This class can be overridden to define logic specific to various database engine deployments * (e.g. Aurora, Multi-AZ, Global Aurora etc.). */ -export class TopologyUtils { +export abstract class TopologyUtils { protected readonly dialect: TopologyAwareDatabaseDialect; protected readonly hostInfoBuilder: HostInfoBuilder; @@ -84,25 +88,17 @@ export class TopologyUtils { * * @param targetClient the client wrapper to use to query the database. * @param dialect the database dialect to use for the topology query. - * @param clusterInstanceTemplate the template {@link HostInfo} to use when constructing new {@link HostInfo} objects from - * the data returned by the topology query. + * @param initialHost the initial host info. + * @param instanceTemplate the template for constructing host info objects. * @returns a list of {@link HostInfo} objects representing the results of the topology query. * @throws TypeError if the dialect is not topology-aware. */ - async queryForTopology( + abstract queryForTopology( targetClient: ClientWrapper, dialect: DatabaseDialect, initialHost: HostInfo, - clusterInstanceTemplate: HostInfo - ): Promise { - if (!isDialectTopologyAware(dialect)) { - throw new TypeError(Messages.get("RdsHostListProvider.incorrectDialect")); - } - - return await dialect - .queryForTopology(targetClient) - .then((res: TopologyQueryResult[]) => this.verifyWriter(this.createHosts(res, initialHost, clusterInstanceTemplate))); - } + instanceTemplate: InstanceTemplate + ): Promise; public createHost( instanceId: string | undefined, @@ -123,7 +119,6 @@ export class TopologyUtils { } const finalEndpoint = endpoint ?? this.getHostEndpoint(hostname, instanceTemplate) ?? ""; - const finalPort = port ?? (instanceTemplate?.isPortSpecified() ? instanceTemplate?.port : initialHost?.port); const host: HostInfo = this.hostInfoBuilder @@ -139,46 +134,8 @@ export class TopologyUtils { return host; } - /** - * Creates {@link HostInfo} objects from the given topology query results. - * - * @param topologyQueryResults the result set returned by the topology query describing the cluster topology - * @param initialHost the {@link HostInfo} describing the initial connection. - * @param clusterInstanceTemplate the template used to construct the new {@link HostInfo} objects. - * @returns a list of {@link HostInfo} objects representing the topology. - */ - public createHosts(topologyQueryResults: TopologyQueryResult[], initialHost: HostInfo, clusterInstanceTemplate: HostInfo): HostInfo[] { - const hostsMap = new Map(); - topologyQueryResults.forEach((row) => { - const lastUpdateTime = row.lastUpdateTime ?? Date.now(); - - const host = this.createHost( - row.id, - row.host, - row.isWriter, - row.weight, - lastUpdateTime, - initialHost, - clusterInstanceTemplate, - row.endpoint, - row.port - ); - - const existing = hostsMap.get(host.host); - if (!existing || existing.lastUpdateTime < host.lastUpdateTime) { - hostsMap.set(host.host, host); - } - }); - - return Array.from(hostsMap.values()); - } - /** * Gets the host endpoint by replacing the placeholder in the cluster instance template. - * - * @param hostName the host name to use in the endpoint. - * @param clusterInstanceTemplate the template containing the endpoint pattern. - * @returns the constructed endpoint, or null if the template is invalid. */ protected getHostEndpoint(hostName: string, clusterInstanceTemplate: HostInfo): string | null { if (!clusterInstanceTemplate || !clusterInstanceTemplate.host) { @@ -191,9 +148,6 @@ export class TopologyUtils { /** * Verifies that the topology contains exactly one writer instance. * If multiple writers are found, selects the most recently updated one. - * - * @param allHosts the list of all hosts from the topology query. - * @returns the verified list of hosts with exactly one writer, or null if no writer is found. */ protected async verifyWriter(allHosts: HostInfo[]): Promise { if (allHosts === null || allHosts.length === 0) { diff --git a/common/lib/host_list_provider_service.ts b/common/lib/host_list_provider_service.ts index 094750d9..15537772 100644 --- a/common/lib/host_list_provider_service.ts +++ b/common/lib/host_list_provider_service.ts @@ -14,7 +14,7 @@ limitations under the License. */ -import { BlockingHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { HostListProvider } from "./host_list_provider/host_list_provider"; import { HostInfo } from "./host_info"; import { AwsClient } from "./aws_client"; import { DatabaseDialect } from "./database_dialect/database_dialect"; @@ -28,7 +28,7 @@ export interface HostListProviderService { setHostListProvider(hostListProvider: HostListProvider): void; - isStaticHostListProvider(): boolean; + isDynamicHostListProvider(): boolean; setInitialConnectionHostInfo(initialConnectionHostInfo: HostInfo): void; @@ -53,6 +53,4 @@ export interface HostListProviderService { getTelemetryFactory(): TelemetryFactory; setAllowedAndBlockedHosts(allowedAndBlockedHosts: AllowedAndBlockedHosts): void; - - isBlockingHostListProvider(arg: any): arg is BlockingHostListProvider; } diff --git a/common/lib/host_selector.ts b/common/lib/host_selector.ts index ac995cf3..e32fbb93 100644 --- a/common/lib/host_selector.ts +++ b/common/lib/host_selector.ts @@ -18,5 +18,5 @@ import { HostInfo } from "./host_info"; import { HostRole } from "./host_role"; export interface HostSelector { - getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo; + getHost(hosts: HostInfo[], role: HostRole | null, props?: Map): HostInfo; } diff --git a/common/lib/internal_pooled_connection_provider.ts b/common/lib/internal_pooled_connection_provider.ts index aa9846aa..a104bfe2 100644 --- a/common/lib/internal_pooled_connection_provider.ts +++ b/common/lib/internal_pooled_connection_provider.ts @@ -27,7 +27,7 @@ import { lookup, LookupAddress } from "dns"; import { promisify } from "util"; import { HostInfoBuilder } from "./host_info_builder"; import { RdsUrlType } from "./utils/rds_url_type"; -import { AwsWrapperError } from "./index"; +import { AwsWrapperError } from "./utils/errors"; import { Messages } from "./utils/messages"; import { HostSelector } from "./host_selector"; import { RandomHostSelector } from "./random_host_selector"; diff --git a/common/lib/least_connections_host_selector.ts b/common/lib/least_connections_host_selector.ts index 58a6ed86..f02f355e 100644 --- a/common/lib/least_connections_host_selector.ts +++ b/common/lib/least_connections_host_selector.ts @@ -32,7 +32,7 @@ export class LeastConnectionsHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => { const hostACount = this.getNumConnections(hostA, LeastConnectionsHostSelector.databasePools); const hostBCount = this.getNumConnections(hostB, LeastConnectionsHostSelector.databasePools); diff --git a/common/lib/mysql_client_wrapper.ts b/common/lib/mysql_client_wrapper.ts index eb438d53..52367053 100644 --- a/common/lib/mysql_client_wrapper.ts +++ b/common/lib/mysql_client_wrapper.ts @@ -66,7 +66,7 @@ export class MySQLClientWrapper implements ClientWrapper { async abort(): Promise { try { - return await ClientUtils.queryWithTimeout(this.client?.destroy(), this.properties); + this.client?.destroy(); } catch (error: any) { // ignore } diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts new file mode 100644 index 00000000..2097d093 --- /dev/null +++ b/common/lib/partial_plugin_service.ts @@ -0,0 +1,550 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { PluginService } from "./plugin_service"; +import { HostInfo } from "./host_info"; +import { AwsClient } from "./aws_client"; +import { DynamicHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { ConnectionUrlParser } from "./utils/connection_url_parser"; +import { DatabaseDialect } from "./database_dialect/database_dialect"; +import { HostInfoBuilder } from "./host_info_builder"; +import { AwsTimeoutError, AwsWrapperError, UnsupportedMethodError } from "./"; +import { HostAvailability } from "./host_availability/host_availability"; +import { HostAvailabilityCacheItem } from "./host_availability/host_availability_cache_item"; +import { HostChangeOptions } from "./host_change_options"; +import { HostRole } from "./host_role"; +import { SessionStateService } from "./session_state_service"; +import { HostAvailabilityStrategyFactory } from "./host_availability/host_availability_strategy_factory"; +import { ClientWrapper } from "./client_wrapper"; +import { logger } from "../logutils"; +import { Messages } from "./utils/messages"; +import { getWriter, logTopology } from "./utils/utils"; +import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; +import { DriverDialect } from "./driver_dialect/driver_dialect"; +import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; +import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { HostListProviderService } from "./host_list_provider_service"; +import { StorageService } from "./utils/storage/storage_service"; +import { CoreServicesContainer } from "./utils/core_services_container"; + +/** + * A PluginService containing some methods that are not intended to be called. This class is intended to be used + * by monitors, which require a PluginService, but are not expected to need or use some of the methods defined + * by the PluginService interface. The methods that are not expected to be called will throw an + * UnsupportedOperationException when called. + */ +export class PartialPluginService implements PluginService, HostListProviderService { + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds + + protected readonly servicesContainer: FullServicesContainer; + protected readonly storageService: StorageService; + protected readonly props: Map; + protected hostListProvider: HostListProvider | null = null; + protected hosts: HostInfo[] = []; + protected currentHostInfo: HostInfo | null = null; + protected initialConnectionHostInfo: HostInfo | null = null; + protected isInTransactionFlag: boolean = false; + protected readonly dialect: DatabaseDialect; + protected readonly driverDialect: DriverDialect; + protected allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; + private _isPooledClient: boolean = false; + private connectionUrlParser: ConnectionUrlParser; + + constructor( + servicesContainer: FullServicesContainer, + props: Map, + dialect: DatabaseDialect, + driverDialect: DriverDialect, + connectionUrlParser: ConnectionUrlParser + ) { + this.servicesContainer = servicesContainer; + this.storageService = servicesContainer.storageService; + this.servicesContainer.hostListProviderService = this; + this.servicesContainer.pluginService = this; + + this.props = props; + this.dialect = dialect; + this.driverDialect = driverDialect; + this.connectionUrlParser = connectionUrlParser; + + this.hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get("host"), this.servicesContainer); + } + + getCurrentClient(): AwsClient { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getCurrentClient")); + } + + getCurrentHostInfo(): HostInfo | null { + if (!this.currentHostInfo) { + this.currentHostInfo = this.initialConnectionHostInfo; + + if (!this.currentHostInfo) { + if (this.getAllHosts().length === 0) { + throw new AwsWrapperError(Messages.get("PluginService.hostListEmpty")); + } + + const writerHost = getWriter(this.getAllHosts()); + if (writerHost) { + this.currentHostInfo = writerHost; + const allowedHosts = this.getHosts(); + if (!allowedHosts.some((hostInfo: HostInfo) => hostInfo.host === writerHost.host && hostInfo.port === writerHost.port)) { + throw new AwsWrapperError( + Messages.get( + "PluginService.currentHostNotAllowed", + this.currentHostInfo ? this.currentHostInfo.host : "", + logTopology(allowedHosts, "") + ) + ); + } + } + + if (!this.currentHostInfo) { + const hosts = this.getHosts(); + if (hosts.length > 0) { + this.currentHostInfo = hosts[0]; + } + } + } + + if (!this.currentHostInfo) { + throw new AwsWrapperError(Messages.get("PluginService.currentHostNotDefined")); + } + + logger.debug(`Set current host to: ${this.currentHostInfo.host}`); + } + + return this.currentHostInfo; + } + + setCurrentHostInfo(value: HostInfo): void { + this.currentHostInfo = value; + } + + setInitialConnectionHostInfo(initialConnectionHostInfo: HostInfo): void { + this.initialConnectionHostInfo = initialConnectionHostInfo; + } + + getInitialConnectionHostInfo(): HostInfo | null { + return this.initialConnectionHostInfo; + } + + acceptsStrategy(role: HostRole, strategy: string): boolean { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "acceptsStrategy")); + } + + getHostInfoByStrategy(role: HostRole, strategy: string, hosts?: HostInfo[]): HostInfo | undefined { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getHostInfoByStrategy")); + } + + getHostRole(client: any): Promise | undefined { + return this.dialect.getHostRole(client); + } + + getDriverDialect(): DriverDialect { + return this.driverDialect; + } + + getConnectionUrlParser(): ConnectionUrlParser { + return this.connectionUrlParser; + } + + setCurrentClient(newClient: ClientWrapper, hostInfo: HostInfo): Promise> { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setCurrentClient")); + } + + protected compare(hostInfoA: HostInfo, hostInfoB: HostInfo): Set { + const changes: Set = new Set(); + + if (hostInfoA.host !== hostInfoB.host || hostInfoA.port !== hostInfoB.port) { + changes.add(HostChangeOptions.HOSTNAME); + } + + if (hostInfoA.role !== hostInfoB.role) { + if (hostInfoB.role === HostRole.WRITER) { + changes.add(HostChangeOptions.PROMOTED_TO_WRITER); + } else if (hostInfoB.role === HostRole.READER) { + changes.add(HostChangeOptions.PROMOTED_TO_READER); + } + } + + if (hostInfoA.availability !== hostInfoB.availability) { + if (hostInfoB.availability === HostAvailability.AVAILABLE) { + changes.add(HostChangeOptions.WENT_UP); + } else if (hostInfoB.availability === HostAvailability.NOT_AVAILABLE) { + changes.add(HostChangeOptions.WENT_DOWN); + } + } + + if (changes.size > 0) { + changes.add(HostChangeOptions.HOST_CHANGED); + } + + return changes; + } + + getAllHosts(): HostInfo[] { + return this.hosts; + } + + getHosts(): HostInfo[] { + const hostPermissions = this.allowedAndBlockedHosts; + if (!hostPermissions) { + return this.hosts; + } + + let hosts = this.hosts; + const allowedHostIds = hostPermissions.getAllowedHostIds(); + const blockedHostIds = hostPermissions.getBlockedHostIds(); + + if (allowedHostIds && allowedHostIds.size > 0) { + hosts = hosts.filter((host: HostInfo) => allowedHostIds.has(host.hostId)); + } + + if (blockedHostIds && blockedHostIds.size > 0) { + hosts = hosts.filter((host: HostInfo) => !blockedHostIds.has(host.hostId)); + } + + return hosts; + } + + setAvailability(hostAliases: Set, availability: HostAvailability): void { + if (hostAliases.size === 0) { + return; + } + + const hostsToChange = [ + ...new Set( + this.getAllHosts().filter( + (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + ) + ) + ]; + + if (hostsToChange.length === 0) { + return; + } + + const changes = new Map>(); + for (const host of hostsToChange) { + const currentAvailability = host.getAvailability(); + host.availability = availability; + this.storageService.set(host.url, new HostAvailabilityCacheItem(availability)); + if (currentAvailability !== availability) { + let hostChanges: Set; + if (availability === HostAvailability.AVAILABLE) { + hostChanges = new Set([HostChangeOptions.WENT_UP, HostChangeOptions.HOST_CHANGED]); + } else { + hostChanges = new Set([HostChangeOptions.WENT_DOWN, HostChangeOptions.HOST_CHANGED]); + } + changes.set(host.url, hostChanges); + } + } + + if (changes.size > 0) { + this.servicesContainer.pluginManager?.notifyHostListChanged(changes); + } + } + + isInTransaction(): boolean { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "isInTransaction")); + } + + isDialectConfirmed(): boolean { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "isDialectConfirmed")); + } + + setInTransaction(inTransaction: boolean): void { + this.isInTransactionFlag = inTransaction; + } + + getHostListProvider(): HostListProvider | null { + return this.hostListProvider; + } + + async refreshHostList(): Promise { + const updatedHostList = await this.getHostListProvider()?.refresh(); + if (updatedHostList && updatedHostList !== this.hosts) { + this.updateHostAvailability(updatedHostList); + this.setHostList(this.hosts, updatedHostList); + } + } + + async forceRefreshHostList(): Promise { + await this.forceMonitoringRefresh(false, PartialPluginService.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); + } + + async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { + const hostListProvider = this.getHostListProvider(); + + if (!this.isDynamicHostListProvider()) { + const providerName = hostListProvider?.constructor.name ?? "null"; + throw new UnsupportedMethodError(Messages.get("PluginService.requiredDynamicHostListProvider", providerName)); + } + + try { + const updatedHostList = await (hostListProvider as DynamicHostListProvider).forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); + if (updatedHostList) { + this.updateHostAvailability(updatedHostList); + this.setHostList(this.hosts, updatedHostList); + return true; + } + } catch (err) { + if (err instanceof AwsTimeoutError) { + logger.debug(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + } + } + + return false; + } + + protected setHostList(oldHosts: HostInfo[] | null, newHosts: HostInfo[] | null): void { + const oldHostMap: Map = oldHosts ? new Map(oldHosts.map((e) => [e.url, e])) : new Map(); + + const newHostMap: Map = newHosts ? new Map(newHosts.map((e) => [e.url, e])) : new Map(); + + const changes: Map> = new Map(); + + oldHostMap.forEach((value, key) => { + const correspondingNewHost = newHostMap.get(key); + if (!correspondingNewHost) { + changes.set(key, new Set([HostChangeOptions.HOST_DELETED])); + } else { + const hostChanges = this.compare(value, correspondingNewHost); + if (hostChanges.size > 0) { + changes.set(key, hostChanges); + } + } + }); + + newHostMap.forEach((value, key) => { + if (!oldHostMap.has(key)) { + changes.set(key, new Set([HostChangeOptions.HOST_ADDED])); + } + }); + + if (changes.size > 0) { + this.hosts = newHosts ? newHosts : []; + this.servicesContainer.pluginManager?.notifyHostListChanged(changes); + } + } + + isDynamicHostListProvider(): boolean { + const provider = this.getHostListProvider(); + return provider !== null && typeof (provider as any).forceMonitoringRefresh === "function"; + } + + setHostListProvider(hostListProvider: HostListProvider): void { + this.hostListProvider = hostListProvider; + } + + connect(hostInfo: HostInfo, props: Map): Promise; + connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; + connect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin | null): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "connect")); + } + + forceConnect(hostInfo: HostInfo, props: Map): Promise; + forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; + forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin | null): Promise { + const pluginManager = this.servicesContainer.pluginManager; + if (!pluginManager) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "forceConnect")); + } + return pluginManager.forceConnect(hostInfo, props, true, pluginToSkip ?? null); + } + + protected updateHostAvailability(hosts: HostInfo[]): void { + hosts.forEach((host) => { + const cacheItem = this.storageService.get(HostAvailabilityCacheItem, host.url); + if (cacheItem != null) { + host.availability = cacheItem.availability; + } + }); + } + + // Error handler methods + isLoginError(e: Error): boolean { + return this.dialect.getErrorHandler().isLoginError(e); + } + + isNetworkError(e: Error): boolean { + return this.dialect.getErrorHandler().isNetworkError(e); + } + + isSyntaxError(e: Error): boolean { + return this.dialect.getErrorHandler().isSyntaxError(e); + } + + hasLoginError(): boolean { + return this.dialect.getErrorHandler().hasLoginError(); + } + + hasNetworkError(): boolean { + return this.dialect.getErrorHandler().hasNetworkError(); + } + + getUnexpectedError(): Error | null { + return this.dialect.getErrorHandler().getUnexpectedError(); + } + + attachErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().attachErrorListener(clientWrapper); + } + + attachNoOpErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().attachNoOpErrorListener(clientWrapper); + } + + removeErrorListener(clientWrapper: ClientWrapper | undefined): void { + this.dialect.getErrorHandler().removeErrorListener(clientWrapper); + } + + getDialect(): DatabaseDialect { + return this.dialect; + } + + async updateDialect(targetClient: ClientWrapper): Promise { + // Do nothing. This method is called after connecting in DefaultConnectionPlugin but the dialect passed to the + // constructor should already be updated and verified. + } + + async identifyConnection(targetClient: ClientWrapper): Promise { + const provider = this.getHostListProvider(); + if (!provider) { + return Promise.reject(new AwsWrapperError(Messages.get("PluginService.errorIdentifyConnection"))); + } + return provider.identifyConnection(targetClient); + } + + async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise { + if (!hostInfo) { + return; + } + + if (hostInfo.aliases.size > 0) { + logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); + return; + } + + hostInfo.addAlias(hostInfo.asAlias); + + try { + const res = await this.dialect.getHostAliasAndParseResults(targetClient); + if (res) { + hostInfo.addAlias(res); + } + } catch (error) { + logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); + } + + try { + const host = await this.identifyConnection(targetClient); + if (host && host.allAliases) { + hostInfo.addAlias(...host.allAliases); + } + } catch (error) { + // Ignore errors from identifyConnection + logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); + } + } + + getHostInfoBuilder(): HostInfoBuilder { + return new HostInfoBuilder({ hostAvailabilityStrategy: new HostAvailabilityStrategyFactory().create(this.props) }); + } + + getProperties(): Map { + return this.props; + } + + getTelemetryFactory(): TelemetryFactory { + const pluginManager = this.servicesContainer.pluginManager; + if (!pluginManager) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getTelemetryFactory")); + } + return pluginManager.getTelemetryFactory(); + } + + getSessionStateService(): SessionStateService { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getSessionStateService")); + } + + async updateState(sql: string): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateState")); + } + + updateInTransaction(sql: string): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateInTransaction")); + } + + async isClientValid(targetClient: ClientWrapper): Promise { + return await this.getDialect().isClientValid(targetClient); + } + + async abortCurrentClient(): Promise { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "abortCurrentClient")); + } + + async abortTargetClient(targetClient: ClientWrapper | undefined | null): Promise { + if (targetClient) { + await targetClient.abort(); + } + } + + updateConfigWithProperties(props: Map): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "updateConfigWithProperties")); + } + + setAllowedAndBlockedHosts(allowedAndBlockedHosts: AllowedAndBlockedHosts): void { + this.allowedAndBlockedHosts = allowedAndBlockedHosts; + } + + setStatus(clazz: any, status: T | null, clusterBound: boolean): void; + setStatus(clazz: any, status: T | null, key: string): void; + setStatus(clazz: any, status: T | null, clusterBoundOrKey: boolean | string): void { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setStatus")); + } + + getStatus(clazz: any, clusterBound: boolean): T; + getStatus(clazz: any, key: string): T; + getStatus(clazz: any, clusterBoundOrKey: boolean | string): T { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getStatus")); + } + + isPluginInUse(plugin: any): boolean { + try { + return this.servicesContainer.pluginManager?.isPluginInUse(plugin) ?? false; + } catch (e) { + return false; + } + } + + getPlugin(pluginClazz: new (...args: any[]) => T): T | null { + return this.servicesContainer.pluginManager?.unwrapPlugin(pluginClazz) ?? null; + } + + static clearCache(): void { + CoreServicesContainer.getInstance().storageService.clear(HostAvailabilityCacheItem); + } + + isPooledClient(): boolean { + return this._isPooledClient; + } + + setIsPooledClient(isPooledClient: boolean): void { + this._isPooledClient = isPooledClient; + } +} diff --git a/common/lib/plugin_factory.ts b/common/lib/plugin_factory.ts index 7360ba1d..1e69580b 100644 --- a/common/lib/plugin_factory.ts +++ b/common/lib/plugin_factory.ts @@ -16,9 +16,10 @@ import { PluginService } from "./plugin_service"; import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; export class ConnectionPluginFactory { - getInstance(pluginService: PluginService, properties: Map): Promise { + getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { return; } } diff --git a/common/lib/plugin_manager.ts b/common/lib/plugin_manager.ts index 410b4cf9..1edffdf5 100644 --- a/common/lib/plugin_manager.ts +++ b/common/lib/plugin_manager.ts @@ -19,7 +19,6 @@ import { HostInfo } from "./host_info"; import { ConnectionPluginChainBuilder } from "./connection_plugin_chain_builder"; import { AwsWrapperError } from "./utils/errors"; import { Messages } from "./utils/messages"; -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; import { HostListProviderService } from "./host_list_provider_service"; import { HostChangeOptions } from "./host_change_options"; import { OldConnectionSuggestionAction } from "./old_connection_suggestion_action"; @@ -32,6 +31,7 @@ import { TelemetryTraceLevel } from "./utils/telemetry/telemetry_trace_level"; import { ConnectionProvider } from "./connection_provider"; import { ConnectionPluginFactory } from "./plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; +import { FullServicesContainer } from "./utils/full_services_container"; import { CoreServicesContainer } from "./utils/core_services_container"; type PluginFunc = (plugin: ConnectionPlugin, targetFunc: () => Promise) => Promise; @@ -80,17 +80,16 @@ export class PluginManager { private readonly props: Map; private _plugins: ConnectionPlugin[] = []; private readonly connectionProviderManager: ConnectionProviderManager; - private pluginServiceManagerContainer: PluginServiceManagerContainer; + private fullServicesContainer: FullServicesContainer; protected telemetryFactory: TelemetryFactory; constructor( - pluginServiceManagerContainer: PluginServiceManagerContainer, + fullServicesContainer: FullServicesContainer, props: Map, connectionProviderManager: ConnectionProviderManager, telemetryFactory: TelemetryFactory ) { - this.pluginServiceManagerContainer = pluginServiceManagerContainer; - this.pluginServiceManagerContainer.pluginManager = this; + this.fullServicesContainer = fullServicesContainer; this.connectionProviderManager = connectionProviderManager; this.props = props; this.telemetryFactory = telemetryFactory; @@ -99,17 +98,15 @@ export class PluginManager { async init(configurationProfile?: ConfigurationProfile | null): Promise; async init(configurationProfile: ConfigurationProfile | null, plugins: ConnectionPlugin[]): Promise; async init(configurationProfile: ConfigurationProfile | null, plugins?: ConnectionPlugin[]) { - if (this.pluginServiceManagerContainer.pluginService != null) { - if (plugins) { - this._plugins = plugins; - } else { - this._plugins = await ConnectionPluginChainBuilder.getPlugins( - this.pluginServiceManagerContainer.pluginService, - this.props, - this.connectionProviderManager, - configurationProfile - ); - } + if (plugins) { + this._plugins = plugins; + } else { + this._plugins = await ConnectionPluginChainBuilder.getPlugins( + this.fullServicesContainer, + this.props, + this.connectionProviderManager, + configurationProfile + ); } for (const plugin of this._plugins) { PluginManager.PLUGINS.add(plugin); @@ -129,8 +126,8 @@ export class PluginManager { } const telemetryContext = this.telemetryFactory.openTelemetryContext(methodName, TelemetryTraceLevel.NESTED); - const currentClient: ClientWrapper = this.pluginServiceManagerContainer.pluginService.getCurrentClient().targetClient; - this.pluginServiceManagerContainer.pluginService.attachNoOpErrorListener(currentClient); + const currentClient: ClientWrapper = this.fullServicesContainer.pluginService.getCurrentClient().targetClient; + this.fullServicesContainer.pluginService.attachNoOpErrorListener(currentClient); try { return await telemetryContext.start(() => { return this.executeWithSubscribedPlugins( @@ -143,7 +140,7 @@ export class PluginManager { ); }); } finally { - this.pluginServiceManagerContainer.pluginService.attachErrorListener(currentClient); + this.fullServicesContainer.pluginService.attachErrorListener(currentClient); } } @@ -396,7 +393,7 @@ export class PluginManager { } PluginManager.STRATEGY_PLUGIN_CHAIN_CACHE.clear(); - CoreServicesContainer.releaseResources(); + await CoreServicesContainer.releaseResources(); PluginManager.PLUGINS = new Set(); } @@ -445,7 +442,7 @@ export class PluginManager { for (const p of this._plugins) { if (p instanceof iface) { - return p as any; + return p as T; } } return null; diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 18144455..68f69c97 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -14,18 +14,18 @@ limitations under the License. */ -import { PluginServiceManagerContainer } from "./plugin_service_manager_container"; import { ErrorHandler } from "./error_handler"; import { HostInfo } from "./host_info"; import { AwsClient } from "./aws_client"; import { HostListProviderService } from "./host_list_provider_service"; -import { BlockingHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; +import { DynamicHostListProvider, HostListProvider } from "./host_list_provider/host_list_provider"; import { ConnectionUrlParser } from "./utils/connection_url_parser"; import { DatabaseDialect, DatabaseType } from "./database_dialect/database_dialect"; import { HostInfoBuilder } from "./host_info_builder"; -import { AwsWrapperError } from "./"; +import { AwsTimeoutError, AwsWrapperError, UnsupportedMethodError } from "./utils/errors"; import { HostAvailability } from "./host_availability/host_availability"; -import { CacheMap } from "./utils/cache_map"; +import { HostAvailabilityCacheItem } from "./host_availability/host_availability_cache_item"; +import { StatusCacheItem } from "./utils/status_cache_item"; import { HostChangeOptions } from "./host_change_options"; import { HostRole } from "./host_role"; import { WrapperProperties } from "./wrapper_property"; @@ -45,6 +45,9 @@ import { TelemetryFactory } from "./utils/telemetry/telemetry_factory"; import { DriverDialect } from "./driver_dialect/driver_dialect"; import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; import { ConnectionPlugin } from "./connection_plugin"; +import { FullServicesContainer } from "./utils/full_services_container"; +import { StorageService } from "./utils/storage/storage_service"; +import { CoreServicesContainer } from "./utils/core_services_container"; export interface PluginService extends ErrorHandler { isInTransaction(): boolean; @@ -73,28 +76,22 @@ export interface PluginService extends ErrorHandler { getDialect(): DatabaseDialect; + isDialectConfirmed(): boolean; + getDriverDialect(): DriverDialect; getHostInfoBuilder(): HostInfoBuilder; - isStaticHostListProvider(): boolean; + isDynamicHostListProvider(): boolean; acceptsStrategy(role: HostRole, strategy: string): boolean; forceRefreshHostList(): Promise; - forceRefreshHostList(targetClient: ClientWrapper): Promise; - - forceRefreshHostList(targetClient?: ClientWrapper): Promise; - forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise; refreshHostList(): Promise; - refreshHostList(targetClient: ClientWrapper): Promise; - - refreshHostList(targetClient?: ClientWrapper): Promise; - getAllHosts(): HostInfo[]; getHosts(): HostInfo[]; @@ -157,29 +154,28 @@ export interface PluginService extends ErrorHandler { } export class PluginServiceImpl implements PluginService, HostListProviderService { - private static readonly DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO = 5 * 60_000_000_000; // 5 minutes + private static readonly DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS = 5000; // 5 seconds. private readonly _currentClient: AwsClient; private _currentHostInfo?: HostInfo; private _hostListProvider?: HostListProvider; private _initialConnectionHostInfo?: HostInfo; private _isInTransaction: boolean = false; - private pluginServiceManagerContainer: PluginServiceManagerContainer; + private servicesContainer: FullServicesContainer; protected hosts: HostInfo[] = []; private dbDialectProvider: DatabaseDialectProvider; private readonly initialHost: string; private dialect: DatabaseDialect; + private _isDialectConfirmed: boolean = false; private readonly driverDialect: DriverDialect; protected readonly sessionStateService: SessionStateService; - protected static readonly hostAvailabilityExpiringCache: CacheMap = new CacheMap(); + protected storageService: StorageService; readonly props: Map; private allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; - protected static readonly statusesExpiringCache: CacheMap = new CacheMap(); - protected static readonly DEFAULT_STATUS_CACHE_EXPIRE_NANO: number = 3_600_000_000_000; // 60 minutes protected _isPooledClient: boolean = false; constructor( - container: PluginServiceManagerContainer, + container: FullServicesContainer, client: AwsClient, dbType: DatabaseType, knownDialectsByCode: Map, @@ -187,12 +183,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService driverDialect: DriverDialect ) { this._currentClient = client; - this.pluginServiceManagerContainer = container; + this.servicesContainer = container; + this.storageService = container.storageService; this.props = props; this.dbDialectProvider = new DatabaseDialectManager(knownDialectsByCode, dbType, this.props); this.driverDialect = driverDialect; this.initialHost = props.get(WrapperProperties.HOST.name); - container.pluginService = this; this.dialect = WrapperProperties.CUSTOM_DATABASE_DIALECT.get(this.props) ?? this.dbDialectProvider.getDialect(this.props); this.sessionStateService = new SessionStateServiceImpl(this, this.props); @@ -223,7 +219,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } getHostInfoByStrategy(role: HostRole, strategy: string, hosts?: HostInfo[]): HostInfo | undefined { - const pluginManager = this.pluginServiceManagerContainer.pluginManager; + const pluginManager = this.servicesContainer.pluginManager; return pluginManager?.getHostInfoByStrategy(role, strategy, hosts); } @@ -285,6 +281,10 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return this.dialect; } + isDialectConfirmed(): boolean { + return this._isDialectConfirmed; + } + getDriverDialect(): DriverDialect { return this.driverDialect; } @@ -293,58 +293,46 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return new HostInfoBuilder({ hostAvailabilityStrategy: new HostAvailabilityStrategyFactory().create(this.props) }); } - isStaticHostListProvider(): boolean { - return false; + isDynamicHostListProvider(): boolean { + const provider = this.getHostListProvider(); + return provider !== null && typeof (provider as any).forceMonitoringRefresh === "function"; } acceptsStrategy(role: HostRole, strategy: string): boolean { - return this.pluginServiceManagerContainer.pluginManager?.acceptsStrategy(role, strategy) ?? false; + return this.servicesContainer.pluginManager?.acceptsStrategy(role, strategy) ?? false; } - async forceRefreshHostList(): Promise; - async forceRefreshHostList(targetClient: ClientWrapper): Promise; - async forceRefreshHostList(targetClient?: ClientWrapper): Promise { - const updatedHostList = targetClient - ? await this.getHostListProvider()?.forceRefresh(targetClient) - : await this.getHostListProvider()?.forceRefresh(); - if (updatedHostList && updatedHostList !== this.hosts) { - this.updateHostAvailability(updatedHostList); - await this.setHostList(this.hosts, updatedHostList); - } + async forceRefreshHostList(): Promise { + await this.forceMonitoringRefresh(false, PluginServiceImpl.DEFAULT_TOPOLOGY_QUERY_TIMEOUT_MS); } async forceMonitoringRefresh(shouldVerifyWriter: boolean, timeoutMs: number): Promise { const hostListProvider: HostListProvider = this.getHostListProvider(); - if (!this.isBlockingHostListProvider(hostListProvider)) { - logger.info(Messages.get("PluginService.requiredBlockingHostListProvider", typeof hostListProvider)); - throw new AwsWrapperError(Messages.get("PluginService.requiredBlockingHostListProvider", typeof hostListProvider)); + + if (!this.isDynamicHostListProvider()) { + const providerName = hostListProvider?.constructor.name ?? "null"; + throw new UnsupportedMethodError(Messages.get("PluginService.requiredDynamicHostListProvider", providerName)); } try { - const updatedHostList: HostInfo[] = await hostListProvider.forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); + const updatedHostList: HostInfo[] = await (hostListProvider as DynamicHostListProvider).forceMonitoringRefresh(shouldVerifyWriter, timeoutMs); if (updatedHostList) { - if (updatedHostList !== this.hosts) { - this.updateHostAvailability(updatedHostList); - await this.setHostList(this.hosts, updatedHostList); - } + this.updateHostAvailability(updatedHostList); + await this.setHostList(this.hosts, updatedHostList); return true; } } catch (err) { - // Do nothing. - logger.info(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + if (err instanceof AwsTimeoutError) { + // Do nothing. + logger.info(Messages.get("PluginService.forceMonitoringRefreshTimeout", timeoutMs.toString())); + } } return false; } - isBlockingHostListProvider(arg: any): arg is BlockingHostListProvider { - return arg != null && typeof arg.clearAll === "function" && typeof arg.forceMonitoringRefresh === "function"; - } - - async refreshHostList(): Promise; - async refreshHostList(targetClient: ClientWrapper): Promise; - async refreshHostList(targetClient?: ClientWrapper): Promise { - const updatedHostList = targetClient ? await this.getHostListProvider()?.refresh(targetClient) : await this.getHostListProvider()?.refresh(); + async refreshHostList(): Promise { + const updatedHostList = await this.getHostListProvider()?.refresh(); if (updatedHostList && updatedHostList !== this.hosts) { this.updateHostAvailability(updatedHostList); await this.setHostList(this.hosts, updatedHostList); @@ -353,9 +341,9 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private updateHostAvailability(hosts: HostInfo[]) { hosts.forEach((host) => { - const availability = PluginServiceImpl.hostAvailabilityExpiringCache.get(host.url); - if (availability != null) { - host.availability = availability; + const cacheItem = this.storageService.get(HostAvailabilityCacheItem, host.url); + if (cacheItem != null) { + host.availability = cacheItem.availability; } }); } @@ -421,7 +409,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (changes.size > 0) { this.hosts = newHosts ? newHosts : []; - await this.pluginServiceManagerContainer.pluginManager!.notifyHostListChanged(changes); + await this.servicesContainer.pluginManager!.notifyHostListChanged(changes); } } @@ -464,14 +452,13 @@ export class PluginServiceImpl implements PluginService, HostListProviderService ]; if (hostsToChange.length === 0) { - logger.debug(Messages.get("PluginService.hostsChangeListEmpty")); return; } const changes = new Map>(); for (const host of hostsToChange) { const currentAvailability = host.getAvailability(); - PluginServiceImpl.hostAvailabilityExpiringCache.put(host.url, availability, PluginServiceImpl.DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO); + this.storageService.set(host.url, new HostAvailabilityCacheItem(availability)); if (currentAvailability !== availability) { let hostChanges = new Set(); if (availability === HostAvailability.AVAILABLE) { @@ -525,13 +512,13 @@ export class PluginServiceImpl implements PluginService, HostListProviderService connect(hostInfo: HostInfo, props: Map): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin): Promise { - return this.pluginServiceManagerContainer.pluginManager!.connect(hostInfo, props, false, pluginToSkip); + return this.servicesContainer.pluginManager!.connect(hostInfo, props, false, pluginToSkip); } forceConnect(hostInfo: HostInfo, props: Map): Promise; forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin): Promise; forceConnect(hostInfo: HostInfo, props: Map, pluginToSkip?: ConnectionPlugin): Promise { - return this.pluginServiceManagerContainer.pluginManager!.forceConnect(hostInfo, props, false, pluginToSkip); + return this.servicesContainer.pluginManager!.forceConnect(hostInfo, props, false, pluginToSkip); } async setCurrentClient(newClient: ClientWrapper, hostInfo: HostInfo): Promise> { @@ -541,8 +528,8 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.sessionStateService.reset(); const changes = new Set([HostChangeOptions.INITIAL_CONNECTION]); - if (this.pluginServiceManagerContainer.pluginManager) { - await this.pluginServiceManagerContainer.pluginManager.notifyConnectionChanged(changes, null); + if (this.servicesContainer.pluginManager) { + await this.servicesContainer.pluginManager.notifyConnectionChanged(changes, null); } return changes; @@ -569,8 +556,10 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } } - const pluginOpinions: Set = - await this.pluginServiceManagerContainer.pluginManager!.notifyConnectionChanged(changes, null); + const pluginOpinions: Set = await this.servicesContainer.pluginManager!.notifyConnectionChanged( + changes, + null + ); const shouldCloseConnection = changes.has(HostChangeOptions.CONNECTION_OBJECT_CHANGED) && @@ -644,11 +633,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService const originalDialect = this.dialect; this.dialect = await this.dbDialectProvider.getDialectForUpdate(targetClient, this.initialHost, this.props.get(WrapperProperties.HOST.name)); + this._isDialectConfirmed = true; if (originalDialect === this.dialect) { return; } - this._hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get(WrapperProperties.HOST.name), this); + this._hostListProvider = this.dialect.getHostListProvider(this.props, this.props.get(WrapperProperties.HOST.name), this.servicesContainer); } private async updateReadOnly(statements: string[]) { @@ -691,7 +681,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } getTelemetryFactory(): TelemetryFactory { - return this.pluginServiceManagerContainer.pluginManager!.getTelemetryFactory(); + return this.servicesContainer.pluginManager!.getTelemetryFactory(); } /* Error Handler interface implementation */ @@ -736,15 +726,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.allowedAndBlockedHosts = allowedAndBlockedHosts; } - static clearHostAvailabilityCache(): void { - PluginServiceImpl.hostAvailabilityExpiringCache.clear(); - } - getStatus(clazz: any, clusterBound: boolean): T; getStatus(clazz: any, key: string): T; getStatus(clazz: any, clusterBound: boolean | string): T { if (typeof clusterBound === "string") { - return PluginServiceImpl.statusesExpiringCache.get(this.getStatusCacheKey(clazz, clusterBound)); + const cacheItem = this.storageService.get(StatusCacheItem, this.getStatusCacheKey(clazz, clusterBound)); + return cacheItem ? cacheItem.status : null; } let clusterId: string = null; if (clusterBound) { @@ -767,9 +754,9 @@ export class PluginServiceImpl implements PluginService, HostListProviderService if (typeof clusterBound === "string") { const cacheKey: string = this.getStatusCacheKey(clazz, clusterBound); if (!status) { - PluginServiceImpl.statusesExpiringCache.delete(cacheKey); + this.storageService.remove(StatusCacheItem, cacheKey); } else { - PluginServiceImpl.statusesExpiringCache.put(cacheKey, status, PluginServiceImpl.DEFAULT_STATUS_CACHE_EXPIRE_NANO); + this.storageService.set(cacheKey, new StatusCacheItem(status)); } return; } @@ -786,7 +773,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService } isPluginInUse(plugin: any) { - return this.pluginServiceManagerContainer.pluginManager!.isPluginInUse(plugin); + return this.servicesContainer.pluginManager!.isPluginInUse(plugin); } isPooledClient(): boolean { diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts index 2a56e606..7d7891a6 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts @@ -110,7 +110,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (writerCandidate === null || this.rdsUtils.isRdsClusterDns(writerCandidate.host)) { // Writer is not found. It seems that topology is outdated. writerCandidateClient = await connectFunc(); - await this.pluginService.forceRefreshHostList(writerCandidateClient); + await this.pluginService.forceRefreshHostList(); writerCandidate = await this.pluginService.identifyConnection(writerCandidateClient); if (writerCandidate) { @@ -132,7 +132,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if ((await this.pluginService.getHostRole(writerCandidateClient)) !== HostRole.WRITER) { // If the new connection resolves to a reader instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(writerCandidateClient); + await this.pluginService.forceRefreshHostList(); await this.pluginService.abortTargetClient(writerCandidateClient); await sleep(retryDelayMs); continue; @@ -177,7 +177,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (readerCandidate === null || this.rdsUtils.isRdsClusterDns(readerCandidate.host)) { // Reader is not found. It seems that topology is outdated. readerCandidateClient = await connectFunc(); - await this.pluginService.forceRefreshHostList(readerCandidateClient); + await this.pluginService.forceRefreshHostList(); readerCandidate = await this.pluginService.identifyConnection(readerCandidateClient); if (readerCandidate) { @@ -209,7 +209,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if ((await this.pluginService.getHostRole(readerCandidateClient)) !== HostRole.READER) { // If the new connection resolves to a writer instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(readerCandidateClient); + await this.pluginService.forceRefreshHostList(); if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts index d13d287f..b12e4d7b 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin_factory.ts @@ -15,19 +15,21 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class AuroraInitialConnectionStrategyFactory extends ConnectionPluginFactory { private static auroraInitialConnectionStrategyPlugin: any; - async getInstance(pluginService: PluginService, props: Map) { + async getInstance(servicesContainer: FullServicesContainer, props: Map) { try { if (!AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin) { AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin = await import("./aurora_initial_connection_strategy_plugin"); } - return new AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin.AuroraInitialConnectionStrategyPlugin(pluginService); + return new AuroraInitialConnectionStrategyFactory.auroraInitialConnectionStrategyPlugin.AuroraInitialConnectionStrategyPlugin( + servicesContainer.pluginService + ); } catch (error: any) { throw new AwsWrapperError( Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AuroraInitialConnectionStrategyPlugin") diff --git a/common/lib/plugins/bluegreen/blue_green_plugin.ts b/common/lib/plugins/bluegreen/blue_green_plugin.ts index 4f7b90ec..17cacb8d 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin.ts @@ -176,7 +176,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele this.startTimeNano = getTimeInNanos(); - while (routing && result && !result.isPresent()) { + while (routing && (!result || !result.isPresent())) { result = await routing.apply(this, methodName, methodFunc, methodArgs, this.properties, this.pluginService); if (!result?.isPresent()) { this.bgStatus = this.pluginService.getStatus(BlueGreenStatus, this.bgdId); diff --git a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts index f5ba48a3..d20e00b6 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class BlueGreenPluginFactory extends ConnectionPluginFactory { private static blueGreenPlugin: any; - async getInstance(pluginService: PluginService, props: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, props: Map): Promise { try { if (!BlueGreenPluginFactory.blueGreenPlugin) { BlueGreenPluginFactory.blueGreenPlugin = await import("./blue_green_plugin"); } - return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(pluginService, props); + return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(servicesContainer.pluginService, props); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "BlueGreenPluginFactory")); } diff --git a/common/lib/plugins/bluegreen/blue_green_status.ts b/common/lib/plugins/bluegreen/blue_green_status.ts index 1c17463f..783e2840 100644 --- a/common/lib/plugins/bluegreen/blue_green_status.ts +++ b/common/lib/plugins/bluegreen/blue_green_status.ts @@ -19,7 +19,6 @@ import { ConnectRouting } from "./routing/connect_routing"; import { ExecuteRouting } from "./routing/execute_routing"; import { BlueGreenRole } from "./blue_green_role"; import { HostInfo } from "../../host_info"; -import { Pair } from "../../utils/utils"; export class BlueGreenStatus { private readonly bgdId: string; @@ -28,7 +27,7 @@ export class BlueGreenStatus { private readonly _unmodifiableExecuteRouting: readonly ExecuteRouting[]; private readonly _roleByHost: Map; - private readonly _correspondingHosts: Map>; + private readonly _correspondingHosts: Map; constructor( bgdId: string, @@ -36,7 +35,7 @@ export class BlueGreenStatus { unmodifiableConnectRouting?: ConnectRouting[], unmodifiableExecuteRouting?: ExecuteRouting[], roleByHost?: Map, - correspondingHosts?: Map> + correspondingHosts?: Map ) { this.bgdId = bgdId; this._currentPhase = phase; @@ -62,7 +61,7 @@ export class BlueGreenStatus { return this._roleByHost; } - get correspondingHosts(): Map> { + get correspondingHosts(): Map { return this._correspondingHosts; } diff --git a/common/lib/plugins/bluegreen/blue_green_status_monitor.ts b/common/lib/plugins/bluegreen/blue_green_status_monitor.ts index f320c84f..0bd69858 100644 --- a/common/lib/plugins/bluegreen/blue_green_status_monitor.ts +++ b/common/lib/plugins/bluegreen/blue_green_status_monitor.ts @@ -36,6 +36,7 @@ import { HostListProviderService } from "../../host_list_provider_service"; import { StatusInfo } from "./status_info"; import { DatabaseDialect } from "../../database_dialect/database_dialect"; import { AwsWrapperError } from "../../utils/errors"; +import { FullServicesContainer } from "../../utils/full_services_container"; export interface OnBlueGreenStatusChange { onBlueGreenStatusChanged(role: BlueGreenRole, interimStatus: BlueGreenInterimStatus): void; @@ -50,6 +51,7 @@ export class BlueGreenStatusMonitor { protected static readonly knownVersions: Set = new Set([BlueGreenStatusMonitor.latestKnownVersion]); protected readonly blueGreenDialect: BlueGreenDialect; + protected readonly servicesContainer: FullServicesContainer; protected readonly pluginService: PluginService; protected readonly bgdId: string; protected readonly props: Map; @@ -99,7 +101,7 @@ export class BlueGreenStatusMonitor { role: BlueGreenRole, bgdId: string, initialHostInfo: HostInfo, - pluginService: PluginService, + servicesContainer: FullServicesContainer, props: Map, statusCheckIntervalMap: Map, onBlueGreenStatusChangeFunc: OnBlueGreenStatusChange @@ -107,7 +109,8 @@ export class BlueGreenStatusMonitor { this.role = role; this.bgdId = bgdId; this.initialHostInfo = initialHostInfo; - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = this.servicesContainer.pluginService; this.props = props; this.statusCheckIntervalMap = statusCheckIntervalMap; this.onBlueGreenStatusChangeFunc = onBlueGreenStatusChangeFunc; @@ -296,12 +299,7 @@ export class BlueGreenStatusMonitor { return; } - const client: ClientWrapper = this.clientWrapper; - if (await this.isConnectionClosed(client)) { - return; - } - - this.currentTopology = await this.hostListProvider.forceRefresh(client); + this.currentTopology = await this.hostListProvider.forceRefresh(); if (this.collectedTopology) { this.startTopology = this.currentTopology; } @@ -518,7 +516,7 @@ export class BlueGreenStatusMonitor { if (connectionHostInfoCopy) { this.hostListProvider = this.pluginService .getDialect() - .getHostListProvider(hostListProperties, connectionHostInfoCopy.host, this.pluginService as unknown as HostListProviderService); + .getHostListProvider(hostListProperties, connectionHostInfoCopy.host, this.servicesContainer); } else { logger.warn(Messages.get("Bgd.hostInfoNull")); } diff --git a/common/lib/plugins/bluegreen/blue_green_status_provider.ts b/common/lib/plugins/bluegreen/blue_green_status_provider.ts index 00f88db0..5a80116d 100644 --- a/common/lib/plugins/bluegreen/blue_green_status_provider.ts +++ b/common/lib/plugins/bluegreen/blue_green_status_provider.ts @@ -20,7 +20,7 @@ import { SimpleHostAvailabilityStrategy } from "../../host_availability/simple_h import { BlueGreenStatusMonitor } from "./blue_green_status_monitor"; import { BlueGreenInterimStatus } from "./blue_green_interim_status"; import { HostInfo } from "../../host_info"; -import { convertMsToNanos, getTimeInNanos, Pair } from "../../utils/utils"; +import { convertMsToNanos, getTimeInNanos } from "../../utils/utils"; import { BlueGreenRole } from "./blue_green_role"; import { BlueGreenStatus } from "./blue_green_status"; import { BlueGreenPhase } from "./blue_green_phase"; @@ -41,9 +41,10 @@ import { SuspendExecuteRouting } from "./routing/suspend_execute_routing"; import { SuspendUntilCorrespondingHostFoundConnectRouting } from "./routing/suspend_until_corresponding_host_found_connect_routing"; import { RejectConnectRouting } from "./routing/reject_connect_routing"; import { getValueHash } from "./blue_green_utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { StorageService } from "../../utils/storage/storage_service"; export class BlueGreenStatusProvider { - static readonly MONITORING_PROPERTY_PREFIX = "blue_green_monitoring_"; private static readonly DEFAULT_CONNECT_TIMEOUT_MS = 10_000; // 10 seconds private static readonly DEFAULT_QUERY_TIMEOUT_MS = 10_000; // 10 seconds @@ -54,8 +55,8 @@ export class BlueGreenStatusProvider { protected interimStatuses: BlueGreenInterimStatus[] = [null, null]; protected hostIpAddresses: Map = new Map(); - // The second parameter of Pair is null when no corresponding host is found. - protected readonly correspondingHosts: Map> = new Map(); + // The second element is null when no corresponding host is found. + protected readonly correspondingHosts: Map = new Map(); // all known host names; host with no port protected readonly roleByHost: Map = new Map(); @@ -76,14 +77,18 @@ export class BlueGreenStatusProvider { protected readonly switchoverTimeoutNanos: bigint; protected readonly suspendNewBlueConnectionsWhenInProgress: boolean; + protected readonly servicesContainer: FullServicesContainer; + protected readonly storageService: StorageService; protected readonly pluginService: PluginService; protected readonly properties: Map; protected readonly bgdId: string; protected phaseTimeNanos: Map = new Map(); protected readonly rdsUtils: RdsUtils = new RdsUtils(); - constructor(pluginService: PluginService, properties: Map, bgdId: string) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, properties: Map, bgdId: string) { + this.servicesContainer = servicesContainer; + this.pluginService = this.servicesContainer.pluginService; + this.storageService = this.servicesContainer.storageService; this.properties = properties; this.bgdId = bgdId; @@ -107,7 +112,7 @@ export class BlueGreenStatusProvider { BlueGreenRole.SOURCE, this.bgdId, this.pluginService.getCurrentHostInfo(), - this.pluginService, + this.servicesContainer, this.getMonitoringProperties(), this.statusCheckIntervalMap, { onBlueGreenStatusChanged: (role, status) => this.prepareStatus(role, status) } @@ -117,7 +122,7 @@ export class BlueGreenStatusProvider { BlueGreenRole.TARGET, this.bgdId, this.pluginService.getCurrentHostInfo(), - this.pluginService, + this.servicesContainer, this.getMonitoringProperties(), this.statusCheckIntervalMap, { onBlueGreenStatusChanged: (role, status) => this.prepareStatus(role, status) } @@ -128,7 +133,7 @@ export class BlueGreenStatusProvider { const monitoringConnProperties: Map = new Map(this.properties); for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(BlueGreenStatusProvider.MONITORING_PROPERTY_PREFIX)) { + if (!key.startsWith(WrapperProperties.BG_MONITORING_PROPERTY_PREFIX)) { continue; } @@ -234,19 +239,19 @@ export class BlueGreenStatusProvider { if (blueWriterHostInfo) { // greenWriterHostInfo can be null but that will be handled properly by corresponding routing. - this.correspondingHosts.set(blueWriterHostInfo.host, new Pair(blueWriterHostInfo, greenWriterHostInfo)); + this.correspondingHosts.set(blueWriterHostInfo.host, [blueWriterHostInfo, greenWriterHostInfo]); } if (sortedBlueReaderHostInfos?.length > 0) { if (sortedGreenReaderHostInfos?.length > 0) { let greenIndex: number = 0; sortedBlueReaderHostInfos.forEach((blueHostInfo) => { - this.correspondingHosts.set(blueHostInfo.host, new Pair(blueHostInfo, sortedGreenReaderHostInfos.at(greenIndex++))); + this.correspondingHosts.set(blueHostInfo.host, [blueHostInfo, sortedGreenReaderHostInfos.at(greenIndex++)]); greenIndex %= sortedGreenReaderHostInfos.length; }); } else { sortedBlueReaderHostInfos.forEach((blueHostInfo) => { - this.correspondingHosts.set(blueHostInfo.host, new Pair(blueHostInfo, greenWriterHostInfo)); + this.correspondingHosts.set(blueHostInfo.host, [blueHostInfo, greenWriterHostInfo]); }); } } @@ -268,10 +273,10 @@ export class BlueGreenStatusProvider { if (blueClusterHost !== null && greenClusterHost !== null) { if (!this.correspondingHosts.has(blueClusterHost)) { - this.correspondingHosts.set( - blueClusterHost, - new Pair(this.hostInfoBuilder.withHost(blueClusterHost).build(), this.hostInfoBuilder.withHost(greenClusterHost).build()) - ); + this.correspondingHosts.set(blueClusterHost, [ + this.hostInfoBuilder.withHost(blueClusterHost).build(), + this.hostInfoBuilder.withHost(greenClusterHost).build() + ]); } } @@ -288,10 +293,10 @@ export class BlueGreenStatusProvider { if (blueClusterReaderHost !== null && greenClusterReaderHost !== null) { if (!this.correspondingHosts.has(blueClusterReaderHost)) { - this.correspondingHosts.set( - blueClusterReaderHost, - new Pair(this.hostInfoBuilder.withHost(blueClusterReaderHost).build(), this.hostInfoBuilder.withHost(greenClusterReaderHost).build()) - ); + this.correspondingHosts.set(blueClusterReaderHost, [ + this.hostInfoBuilder.withHost(blueClusterReaderHost).build(), + this.hostInfoBuilder.withHost(greenClusterReaderHost).build() + ]); } } @@ -308,10 +313,10 @@ export class BlueGreenStatusProvider { }); if (greenHost) { if (!this.correspondingHosts.has(blueHost)) { - this.correspondingHosts.set( - blueHost, - new Pair(this.hostInfoBuilder.withHost(blueHost).build(), this.hostInfoBuilder.withHost(greenHost).build()) - ); + this.correspondingHosts.set(blueHost, [ + this.hostInfoBuilder.withHost(blueHost).build(), + this.hostInfoBuilder.withHost(greenHost).build() + ]); } } } @@ -483,7 +488,7 @@ export class BlueGreenStatusProvider { Array.from(this.roleByHost.entries()) .filter(([host, role]) => role === BlueGreenRole.SOURCE && this.correspondingHosts.has(host)) .forEach(([host, role]) => { - const hostSpec = this.correspondingHosts.get(host).left; + const hostSpec = this.correspondingHosts.get(host)[0]; const blueIp = this.hostIpAddresses.get(hostSpec.host); const substituteHostSpecWithIp = !blueIp ? hostSpec : this.hostInfoBuilder.copyFrom(hostSpec).withHost(blueIp).build(); @@ -627,9 +632,9 @@ export class BlueGreenStatusProvider { .forEach(([host, role]) => { const blueHost: string = host; const isBlueHostInstance: boolean = this.rdsUtils.isRdsInstance(blueHost); - const pair: Pair | undefined = this.correspondingHosts?.get(host); - const blueHostInfo: HostInfo | undefined = pair?.left; - const greenHostInfo: HostInfo | undefined = pair?.right; + const pair: [HostInfo, HostInfo | null] | undefined = this.correspondingHosts?.get(host); + const blueHostInfo: HostInfo | undefined = pair?.[0]; + const greenHostInfo: HostInfo | undefined = pair?.[1]; if (!greenHostInfo) { // A corresponding host is not found. We need to suspend this call. @@ -872,7 +877,7 @@ export class BlueGreenStatusProvider { logger.debug( "Corresponding hosts:\n" + Array.from(this.correspondingHosts.entries()) - .map(([key, value]) => ` ${key} -> ${value.right == null ? "" : value.right.hostAndPort}`) + .map(([key, value]) => ` ${key} -> ${value[1] == null ? "" : value[1].hostAndPort}`) .join("\n") ); diff --git a/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts b/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts index 7ce13cb1..ecf7cab2 100644 --- a/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/suspend_until_corresponding_host_found_connect_routing.ts @@ -26,7 +26,7 @@ import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; import { TelemetryContext } from "../../../utils/telemetry/telemetry_context"; import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; import { BlueGreenStatus } from "../blue_green_status"; -import { convertMsToNanos, convertNanosToMs, getTimeInNanos, Pair } from "../../../utils/utils"; +import { convertMsToNanos, convertNanosToMs, getTimeInNanos } from "../../../utils/utils"; import { WrapperProperties } from "../../../wrapper_property"; import { BlueGreenPhase } from "../blue_green_phase"; import { AwsWrapperError } from "../../../utils/errors"; @@ -60,7 +60,7 @@ export class SuspendUntilCorrespondingHostFoundConnectRouting extends BaseConnec return await telemetryContext.start(async () => { let bgStatus: BlueGreenStatus = pluginService.getStatus(BlueGreenStatus, this.bgdId); - let correspondingPair: Pair = bgStatus?.correspondingHosts.get(hostInfo.host); + let correspondingPair: [HostInfo, HostInfo] | undefined = bgStatus?.correspondingHosts.get(hostInfo.host); const timeoutNanos: bigint = convertMsToNanos(WrapperProperties.BG_CONNECT_TIMEOUT_MS.get(properties)); const suspendStartTime: bigint = getTimeInNanos(); @@ -70,7 +70,7 @@ export class SuspendUntilCorrespondingHostFoundConnectRouting extends BaseConnec getTimeInNanos() <= endTime && bgStatus != null && bgStatus.currentPhase !== BlueGreenPhase.COMPLETED && - (!correspondingPair || !correspondingPair.right) + (!correspondingPair || !correspondingPair[1]) ) { await this.delay(SuspendUntilCorrespondingHostFoundConnectRouting.SLEEP_TIME_MS, bgStatus, pluginService, this.bgdId); diff --git a/common/lib/plugins/connect_time_plugin_factory.ts b/common/lib/plugins/connect_time_plugin_factory.ts index bc333ccb..f2362458 100644 --- a/common/lib/plugins/connect_time_plugin_factory.ts +++ b/common/lib/plugins/connect_time_plugin_factory.ts @@ -15,15 +15,15 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class ConnectTimePluginFactory extends ConnectionPluginFactory { private static connectTimePlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ConnectTimePluginFactory.connectTimePlugin) { ConnectTimePluginFactory.connectTimePlugin = await import("./connect_time_plugin"); diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts index 4eb2b6e1..b7ce5b35 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class AuroraConnectionTrackerPluginFactory extends ConnectionPluginFactory { private static auroraConnectionTrackerPlugin: any; - async getInstance(pluginService: PluginService, props: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, props: Map): Promise { try { if (!AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin) { AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin = await import("./aurora_connection_tracker_plugin"); } - return new AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin.AuroraConnectionTrackerPlugin(pluginService); + return new AuroraConnectionTrackerPluginFactory.auroraConnectionTrackerPlugin.AuroraConnectionTrackerPlugin(servicesContainer.pluginService); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "AuroraConnectionTrackerPlugin")); } diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts index 0e945c2e..23a3ee50 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_plugin_factory.ts @@ -15,19 +15,19 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class CustomEndpointPluginFactory extends ConnectionPluginFactory { private static customEndpointPlugin: any; - async getInstance(pluginService: PluginService, props: Map) { + async getInstance(servicesContainer: FullServicesContainer, props: Map) { try { if (!CustomEndpointPluginFactory.customEndpointPlugin) { CustomEndpointPluginFactory.customEndpointPlugin = await import("./custom_endpoint_plugin"); } - return new CustomEndpointPluginFactory.customEndpointPlugin.CustomEndpointPlugin(pluginService, props); + return new CustomEndpointPluginFactory.customEndpointPlugin.CustomEndpointPlugin(servicesContainer.pluginService, props); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "CustomEndpointPlugin")); } diff --git a/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts b/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts index b4e2abec..5b89d8d5 100644 --- a/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts +++ b/common/lib/plugins/custom_endpoint/custom_endpoint_role_type.ts @@ -15,17 +15,16 @@ */ export enum CustomEndpointRoleType { - ANY, - WRITER, - READER + ANY = "ANY", + WRITER = "WRITER", + READER = "READER", + UNKNOWN = "UNKNOWN" } -const nameToValue = new Map([ - ["ANY", CustomEndpointRoleType.ANY], - ["WRITER", CustomEndpointRoleType.WRITER], - ["READER", CustomEndpointRoleType.READER] -]); - -export function customEndpointRoleTypeFromValue(name: string): CustomEndpointRoleType { - return nameToValue.get(name.toUpperCase()) ?? CustomEndpointRoleType.ANY; +export function customEndpointRoleTypeFromValue(value: string | null | undefined): CustomEndpointRoleType { + if (!value) { + return CustomEndpointRoleType.UNKNOWN; + } + const normalized = value.toUpperCase(); + return Object.values(CustomEndpointRoleType).find((v) => v === normalized) ?? CustomEndpointRoleType.UNKNOWN; } diff --git a/common/lib/plugins/default_plugin.ts b/common/lib/plugins/default_plugin.ts index 3af9b3fc..2c3b45d8 100644 --- a/common/lib/plugins/default_plugin.ts +++ b/common/lib/plugins/default_plugin.ts @@ -30,15 +30,18 @@ import { HostAvailability } from "../host_availability/host_availability"; import { ClientWrapper } from "../client_wrapper"; import { TelemetryTraceLevel } from "../utils/telemetry/telemetry_trace_level"; import { ConnectionInfo } from "../connection_info"; +import { FullServicesContainer } from "../utils/full_services_container"; export class DefaultPlugin extends AbstractConnectionPlugin { id: string = uniqueId("_defaultPlugin"); + private readonly servicesContainer: FullServicesContainer; private readonly pluginService: PluginService; private readonly connectionProviderManager: ConnectionProviderManager; - constructor(pluginService: PluginService, connectionProviderManager: ConnectionProviderManager) { + constructor(servicesContainer: FullServicesContainer, connectionProviderManager: ConnectionProviderManager) { super(); - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.connectionProviderManager = connectionProviderManager; } diff --git a/common/lib/plugins/dev/developer_connection_plugin_factory.ts b/common/lib/plugins/dev/developer_connection_plugin_factory.ts index ef942d67..e03dfa32 100644 --- a/common/lib/plugins/dev/developer_connection_plugin_factory.ts +++ b/common/lib/plugins/dev/developer_connection_plugin_factory.ts @@ -15,21 +15,25 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { Messages } from "../../utils/messages"; import { AwsWrapperError } from "../../utils/errors"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class DeveloperConnectionPluginFactory extends ConnectionPluginFactory { private static developerPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!DeveloperConnectionPluginFactory.developerPlugin) { DeveloperConnectionPluginFactory.developerPlugin = await import("./developer_connection_plugin"); } - return new DeveloperConnectionPluginFactory.developerPlugin.DeveloperConnectionPlugin(pluginService, properties, new RdsUtils()); + return new DeveloperConnectionPluginFactory.developerPlugin.DeveloperConnectionPlugin( + servicesContainer.pluginService, + properties, + new RdsUtils() + ); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "DeveloperConnectionPlugin")); } diff --git a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts b/common/lib/plugins/efm/host_monitoring_plugin_factory.ts index 62b59cd6..d6b43095 100644 --- a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts +++ b/common/lib/plugins/efm/host_monitoring_plugin_factory.ts @@ -15,21 +15,25 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class HostMonitoringPluginFactory extends ConnectionPluginFactory { private static hostMonitoringPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!HostMonitoringPluginFactory.hostMonitoringPlugin) { HostMonitoringPluginFactory.hostMonitoringPlugin = await import("./host_monitoring_connection_plugin"); } - return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin(pluginService, properties, new RdsUtils()); + return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin( + servicesContainer.pluginService, + properties, + new RdsUtils() + ); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts b/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts index 6763b028..0ac0319f 100644 --- a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts +++ b/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts @@ -15,21 +15,25 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { private static hostMonitoring2Plugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!HostMonitoring2PluginFactory.hostMonitoring2Plugin) { HostMonitoring2PluginFactory.hostMonitoring2Plugin = await import("./host_monitoring2_connection_plugin"); } - return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin(pluginService, properties, new RdsUtils()); + return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin( + servicesContainer.pluginService, + properties, + new RdsUtils() + ); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/execute_time_plugin_factory.ts b/common/lib/plugins/execute_time_plugin_factory.ts index 36e2885e..3087ad20 100644 --- a/common/lib/plugins/execute_time_plugin_factory.ts +++ b/common/lib/plugins/execute_time_plugin_factory.ts @@ -15,15 +15,15 @@ */ import { ConnectionPluginFactory } from "../plugin_factory"; -import { PluginService } from "../plugin_service"; import { ConnectionPlugin } from "../connection_plugin"; import { AwsWrapperError } from "../utils/errors"; import { Messages } from "../utils/messages"; +import { FullServicesContainer } from "../utils/full_services_container"; export class ExecuteTimePluginFactory extends ConnectionPluginFactory { private static executeTimePlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ExecuteTimePluginFactory.executeTimePlugin) { ExecuteTimePluginFactory.executeTimePlugin = await import("./execute_time_plugin"); diff --git a/common/lib/plugins/failover/failover_mode.ts b/common/lib/plugins/failover/failover_mode.ts index e6522335..42444f6c 100644 --- a/common/lib/plugins/failover/failover_mode.ts +++ b/common/lib/plugins/failover/failover_mode.ts @@ -15,19 +15,16 @@ */ export enum FailoverMode { - STRICT_WRITER, - STRICT_READER, - READER_OR_WRITER, - UNKNOWN + STRICT_WRITER = "strict-writer", + STRICT_READER = "strict-reader", + READER_OR_WRITER = "reader-or-writer", + UNKNOWN = "unknown" } -const nameToValue = new Map([ - ["strict-writer", FailoverMode.STRICT_WRITER], - ["strict-reader", FailoverMode.STRICT_READER], - ["reader-or-writer", FailoverMode.READER_OR_WRITER], - ["unknown", FailoverMode.UNKNOWN] -]); - -export function failoverModeFromValue(name: string): FailoverMode { - return nameToValue.get(name.toLowerCase()) ?? FailoverMode.UNKNOWN; +export function failoverModeFromValue(value: string | null | undefined): FailoverMode { + if (!value) { + return FailoverMode.UNKNOWN; + } + const normalized = value.toLowerCase(); + return Object.values(FailoverMode).find((v) => v === normalized) ?? FailoverMode.UNKNOWN; } diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index 4145e519..bbb81e73 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -205,6 +205,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { return ( this.enableFailoverSetting && this._rdsUrlType !== RdsUrlType.RDS_PROXY && + this._rdsUrlType !== RdsUrlType.RDS_PROXY_ENDPOINT && this.pluginService.getAllHosts() && this.pluginService.getAllHosts().length > 0 ); @@ -541,7 +542,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._readerFailoverHandler.setEnableFailoverStrictReader(this.failoverMode === FailoverMode.STRICT_READER); - logger.debug(Messages.get("Failover.parameterValue", "failoverMode", FailoverMode[this.failoverMode])); + logger.debug(Messages.get("Failover.parameterValue", "failoverMode", String(this.failoverMode))); } } } diff --git a/common/lib/plugins/failover/failover_plugin_factory.ts b/common/lib/plugins/failover/failover_plugin_factory.ts index 5f8a5c41..cab67b40 100644 --- a/common/lib/plugins/failover/failover_plugin_factory.ts +++ b/common/lib/plugins/failover/failover_plugin_factory.ts @@ -15,21 +15,21 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FailoverPluginFactory extends ConnectionPluginFactory { private static failoverPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FailoverPluginFactory.failoverPlugin) { FailoverPluginFactory.failoverPlugin = await import("./failover_plugin"); } - return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(pluginService, properties, new RdsUtils()); + return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(servicesContainer.pluginService, properties, new RdsUtils()); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FailoverPlugin")); } diff --git a/common/lib/plugins/failover/writer_failover_handler.ts b/common/lib/plugins/failover/writer_failover_handler.ts index 70d46eab..b8780b88 100644 --- a/common/lib/plugins/failover/writer_failover_handler.ts +++ b/common/lib/plugins/failover/writer_failover_handler.ts @@ -236,7 +236,7 @@ class ReconnectToWriterHandlerTask { const props = new Map(this.initialConnectionProps); props.set(WrapperProperties.HOST.name, this.originalWriterHost.host); this.currentClient = await this.pluginService.forceConnect(this.originalWriterHost, props); - await this.pluginService.forceRefreshHostList(this.currentClient); + await this.pluginService.forceRefreshHostList(); latestTopology = this.pluginService.getAllHosts(); } catch (error) { // Propagate errors that are not caused by network errors. @@ -382,7 +382,7 @@ class WaitForNewWriterHandlerTask { while (this.pluginService.getCurrentClient() && Date.now() < this.endTime && !this.failoverCompleted) { try { if (this.currentReaderTargetClient) { - await this.pluginService.forceRefreshHostList(this.currentReaderTargetClient); + await this.pluginService.forceRefreshHostList(); } const topology = this.pluginService.getAllHosts(); diff --git a/common/lib/plugins/failover2/failover2_plugin.ts b/common/lib/plugins/failover2/failover2_plugin.ts index 3b4a7f04..f9e04fe6 100644 --- a/common/lib/plugins/failover2/failover2_plugin.ts +++ b/common/lib/plugins/failover2/failover2_plugin.ts @@ -38,46 +38,47 @@ import { ClientWrapper } from "../../client_wrapper"; import { HostAvailability } from "../../host_availability/host_availability"; import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; import { HostRole } from "../../host_role"; -import { CanReleaseResources } from "../../can_release_resources"; import { ReaderFailoverResult } from "../failover/reader_failover_result"; -import { BlockingHostListProvider, HostListProvider } from "../../host_list_provider/host_list_provider"; import { logTopology } from "../../utils/utils"; +import { FullServicesContainer } from "../../utils/full_services_container"; -export class Failover2Plugin extends AbstractConnectionPlugin implements CanReleaseResources { +export class Failover2Plugin extends AbstractConnectionPlugin { private static readonly TELEMETRY_WRITER_FAILOVER = "failover to writer instance"; private static readonly TELEMETRY_READER_FAILOVER = "failover to reader"; private static readonly METHOD_END = "end"; private static readonly SUBSCRIBED_METHODS: Set = new Set(["initHostProvider", "connect", "query"]); private readonly _staleDnsHelper: StaleDnsHelper; - private readonly _properties: Map; - private readonly pluginService: PluginService; - private readonly _rdsHelper: RdsUtils; - private readonly failoverWriterTriggeredCounter: TelemetryCounter; - private readonly failoverWriterSuccessCounter: TelemetryCounter; - private readonly failoverWriterFailedCounter: TelemetryCounter; - private readonly failoverReaderTriggeredCounter: TelemetryCounter; - private readonly failoverReaderSuccessCounter: TelemetryCounter; - private readonly failoverReaderFailedCounter: TelemetryCounter; - private telemetryFailoverAdditionalTopTraceSetting: boolean = false; - private _rdsUrlType: RdsUrlType | null = null; - private _isInTransaction: boolean = false; + protected readonly properties: Map; + private readonly servicesContainer: FullServicesContainer; + protected readonly pluginService: PluginService; + protected readonly rdsHelper: RdsUtils; + protected readonly failoverWriterTriggeredCounter: TelemetryCounter; + protected readonly failoverWriterSuccessCounter: TelemetryCounter; + protected readonly failoverWriterFailedCounter: TelemetryCounter; + protected readonly failoverReaderTriggeredCounter: TelemetryCounter; + protected readonly failoverReaderSuccessCounter: TelemetryCounter; + protected readonly failoverReaderFailedCounter: TelemetryCounter; + protected telemetryFailoverAdditionalTopTraceSetting: boolean = false; + protected rdsUrlType: RdsUrlType | null = null; + protected _isInTransaction: boolean = false; private _lastError: any; failoverMode: FailoverMode = FailoverMode.UNKNOWN; - private hostListProviderService?: HostListProviderService; + protected hostListProviderService?: HostListProviderService; protected enableFailoverSetting: boolean = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.defaultValue; - private readonly failoverTimeoutSettingMs: number = WrapperProperties.FAILOVER_TIMEOUT_MS.defaultValue; - private readonly failoverReaderHostSelectorStrategy: string = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.defaultValue; + protected readonly failoverTimeoutSettingMs: number = WrapperProperties.FAILOVER_TIMEOUT_MS.defaultValue; + protected readonly failoverReaderHostSelectorStrategy: string = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.defaultValue; - constructor(pluginService: PluginService, properties: Map, rdsHelper: RdsUtils) { + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils) { super(); - this._properties = properties; - this.pluginService = pluginService; - this._rdsHelper = rdsHelper; + this.properties = properties; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; + this.rdsHelper = rdsHelper; this._staleDnsHelper = new StaleDnsHelper(this.pluginService); - this.enableFailoverSetting = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.get(this._properties); - this.failoverTimeoutSettingMs = WrapperProperties.FAILOVER_TIMEOUT_MS.get(this._properties); - this.failoverReaderHostSelectorStrategy = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.get(this._properties); + this.enableFailoverSetting = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.get(this.properties); + this.failoverTimeoutSettingMs = WrapperProperties.FAILOVER_TIMEOUT_MS.get(this.properties); + this.failoverReaderHostSelectorStrategy = WrapperProperties.FAILOVER_READER_HOST_SELECTOR_STRATEGY.get(this.properties); const telemetryFactory = this.pluginService.getTelemetryFactory(); this.failoverWriterTriggeredCounter = telemetryFactory.createCounter("writerFailover.triggered.count"); @@ -104,21 +105,13 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } initHostProviderFunc(); - - this.failoverMode = failoverModeFromValue(WrapperProperties.FAILOVER_MODE.get(props)); - this._rdsUrlType = this._rdsHelper.identifyRdsType(hostInfo.host); - - if (this.failoverMode === FailoverMode.UNKNOWN) { - this.failoverMode = this._rdsUrlType === RdsUrlType.RDS_READER_CLUSTER ? FailoverMode.READER_OR_WRITER : FailoverMode.STRICT_WRITER; - } - - logger.debug(Messages.get("Failover.parameterValue", "failoverMode", FailoverMode[this.failoverMode])); } - private isFailoverEnabled(): boolean { + protected isFailoverEnabled(): boolean { return ( this.enableFailoverSetting && - this._rdsUrlType !== RdsUrlType.RDS_PROXY && + this.rdsUrlType !== RdsUrlType.RDS_PROXY && + this.rdsUrlType !== RdsUrlType.RDS_PROXY_ENDPOINT && this.pluginService.getAllHosts() && this.pluginService.getAllHosts().length > 0 ); @@ -130,6 +123,8 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele isInitialConnection: boolean, connectFunc: () => Promise ): Promise { + this.initFailoverMode(); + if ( // Failover is not enabled, does not require additional processing. !this.enableFailoverSetting || @@ -188,7 +183,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } if (isInitialConnection) { - await this.pluginService.refreshHostList(client); + await this.pluginService.refreshHostList(); } return client; @@ -231,6 +226,10 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele await this.failoverReader(); } + this.throwFailoverSuccessException(); + } + + protected throwFailoverSuccessException(): void { if (this._isInTransaction || this.pluginService.isInTransaction()) { // "Transaction resolution unknown. Please re-configure session state if required and try // restarting transaction." @@ -243,7 +242,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } } - async failoverReader() { + private async failoverReader() { const telemetryFactory = this.pluginService.getTelemetryFactory(); const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); @@ -367,7 +366,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele throw new InternalQueryTimeoutError(Messages.get("Failover.timeoutError")); } - async failoverWriter() { + private async failoverWriter() { const telemetryFactory = this.pluginService.getTelemetryFactory(); const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_WRITER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverWriterTriggeredCounter.inc(); @@ -430,7 +429,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele } private async createConnectionForHost(hostInfo: HostInfo): Promise { - const copyProps = new Map(this._properties); + const copyProps = new Map(this.properties); copyProps.set(WrapperProperties.HOST.name, hostInfo.host); return await this.pluginService.connect(hostInfo, copyProps, this); } @@ -464,6 +463,22 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele return methodName === Failover2Plugin.METHOD_END; } + protected initFailoverMode(): void { + if (this.rdsUrlType) { + return; + } + + this.failoverMode = failoverModeFromValue(WrapperProperties.FAILOVER_MODE.get(this.properties)); + const initialHostInfo: HostInfo | undefined | null = this.hostListProviderService?.getInitialConnectionHostInfo(); + this.rdsUrlType = this.rdsHelper.identifyRdsType(initialHostInfo?.host); + + if (this.failoverMode === FailoverMode.UNKNOWN) { + this.failoverMode = this.rdsUrlType === RdsUrlType.RDS_READER_CLUSTER ? FailoverMode.READER_OR_WRITER : FailoverMode.STRICT_WRITER; + } + + logger.debug(Messages.get("Failover.parameterValue", "failoverMode", String(this.failoverMode))); + } + private shouldErrorTriggerClientSwitch(error: any): boolean { if (!this.isFailoverEnabled()) { logger.debug(Messages.get("Failover.failoverDisabled")); @@ -486,11 +501,4 @@ export class Failover2Plugin extends AbstractConnectionPlugin implements CanRele this.failoverWriterFailedCounter.inc(); throw new FailoverFailedError(errorMessage); } - - async releaseResources(): Promise { - const hostListProvider: HostListProvider = this.pluginService.getHostListProvider(); - if (this.hostListProviderService.isBlockingHostListProvider(hostListProvider)) { - await (hostListProvider as BlockingHostListProvider).clearAll(); - } - } } diff --git a/common/lib/plugins/failover2/failover2_plugin_factory.ts b/common/lib/plugins/failover2/failover2_plugin_factory.ts index 64b2bcdb..d6068760 100644 --- a/common/lib/plugins/failover2/failover2_plugin_factory.ts +++ b/common/lib/plugins/failover2/failover2_plugin_factory.ts @@ -15,21 +15,21 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { RdsUtils } from "../../utils/rds_utils"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class Failover2PluginFactory extends ConnectionPluginFactory { private static failover2Plugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!Failover2PluginFactory.failover2Plugin) { Failover2PluginFactory.failover2Plugin = await import("./failover2_plugin"); } - return new Failover2PluginFactory.failover2Plugin.Failover2Plugin(pluginService, properties, new RdsUtils()); + return new Failover2PluginFactory.failover2Plugin.Failover2Plugin(servicesContainer, properties, new RdsUtils()); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "Failover2Plugin")); } diff --git a/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts b/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts index c7789502..ac391d64 100644 --- a/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts +++ b/common/lib/plugins/federated_auth/federated_auth_plugin_factory.ts @@ -15,16 +15,16 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FederatedAuthPluginFactory extends ConnectionPluginFactory { private static federatedAuthPlugin: any; private static adfsCredentialsProvider: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FederatedAuthPluginFactory.federatedAuthPlugin) { FederatedAuthPluginFactory.federatedAuthPlugin = await import("./federated_auth_plugin"); @@ -34,6 +34,7 @@ export class FederatedAuthPluginFactory extends ConnectionPluginFactory { FederatedAuthPluginFactory.adfsCredentialsProvider = await import("./adfs_credentials_provider_factory"); } + const pluginService = servicesContainer.pluginService; const adfsCredentialsProviderFactory = new FederatedAuthPluginFactory.adfsCredentialsProvider.AdfsCredentialsProviderFactory(pluginService); return new FederatedAuthPluginFactory.federatedAuthPlugin.FederatedAuthPlugin(pluginService, adfsCredentialsProviderFactory); } catch (error: any) { diff --git a/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts b/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts index c4b80147..b3fc5332 100644 --- a/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts +++ b/common/lib/plugins/federated_auth/okta_auth_plugin_factory.ts @@ -15,16 +15,16 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class OktaAuthPluginFactory extends ConnectionPluginFactory { private static oktaAuthPlugin: any; private static oktaCredentialsProviderFactory: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!OktaAuthPluginFactory.oktaAuthPlugin) { OktaAuthPluginFactory.oktaAuthPlugin = await import("./okta_auth_plugin"); @@ -33,6 +33,7 @@ export class OktaAuthPluginFactory extends ConnectionPluginFactory { OktaAuthPluginFactory.oktaCredentialsProviderFactory = await import("./okta_credentials_provider_factory"); } + const pluginService = servicesContainer.pluginService; const oktaCredentialsProviderFactory = new OktaAuthPluginFactory.oktaCredentialsProviderFactory.OktaCredentialsProviderFactory(pluginService); return new OktaAuthPluginFactory.oktaAuthPlugin.OktaAuthPlugin(pluginService, oktaCredentialsProviderFactory); } catch (error: any) { diff --git a/common/lib/plugins/gdb_failover/global_db_failover_mode.ts b/common/lib/plugins/gdb_failover/global_db_failover_mode.ts new file mode 100644 index 00000000..975372ec --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_mode.ts @@ -0,0 +1,34 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export enum GlobalDbFailoverMode { + STRICT_WRITER = "strict-writer", + STRICT_HOME_READER = "strict-home-reader", + STRICT_OUT_OF_HOME_READER = "strict-out-of-home-reader", + STRICT_ANY_READER = "strict-any-reader", + HOME_READER_OR_WRITER = "home-reader-or-writer", + OUT_OF_HOME_READER_OR_WRITER = "out-of-home-reader-or-writer", + ANY_READER_OR_WRITER = "any-reader-or-writer", + UNKNOWN = "unknown" +} + +export function globalDbFailoverModeFromValue(value: string | null | undefined): GlobalDbFailoverMode { + if (!value) { + return GlobalDbFailoverMode.UNKNOWN; + } + const normalized = value.toLowerCase(); + return Object.values(GlobalDbFailoverMode).find((v) => v === normalized) ?? GlobalDbFailoverMode.UNKNOWN; +} diff --git a/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts b/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts new file mode 100644 index 00000000..dd366c1c --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_plugin.ts @@ -0,0 +1,367 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { RdsUtils } from "../../utils/rds_utils"; +import { GlobalDbFailoverMode, globalDbFailoverModeFromValue } from "./global_db_failover_mode"; +import { HostInfo } from "../../host_info"; +import { WrapperProperties } from "../../wrapper_property"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { logger } from "../../../logutils"; +import { Messages } from "../../utils/messages"; +import { AwsTimeoutError, AwsWrapperError, FailoverFailedError, FailoverSuccessError, UnsupportedMethodError } from "../../utils/errors"; +import { ClientWrapper } from "../../client_wrapper"; +import { HostAvailability } from "../../host_availability/host_availability"; +import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; +import { HostRole } from "../../host_role"; +import { ReaderFailoverResult } from "../failover/reader_failover_result"; +import { containsHostAndPort, convertNanosToMs, equalsIgnoreCase, getTimeInNanos, getWriter, logTopology, sleep } from "../../utils/utils"; +import { Failover2Plugin } from "../failover2/failover2_plugin"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GlobalDbFailoverPlugin extends Failover2Plugin { + private static readonly TELEMETRY_FAILOVER = "failover"; + + protected activeHomeFailoverMode: GlobalDbFailoverMode = GlobalDbFailoverMode.UNKNOWN; + protected inactiveHomeFailoverMode: GlobalDbFailoverMode = GlobalDbFailoverMode.UNKNOWN; + protected homeRegion: string | null = null; + + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils) { + super(servicesContainer, properties, rdsHelper); + } + + protected initFailoverMode(): void { + if (this.rdsUrlType !== null) { + return; + } + + const initialHostInfo = this.hostListProviderService?.getInitialConnectionHostInfo(); + if (!initialHostInfo) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingInitialHost")); + } + + this.rdsUrlType = this.rdsHelper.identifyRdsType(initialHostInfo.host); + + this.homeRegion = WrapperProperties.FAILOVER_HOME_REGION.get(this.properties) ?? null; + if (!this.homeRegion) { + if (!this.rdsUrlType.hasRegion) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingHomeRegion")); + } + this.homeRegion = this.rdsHelper.getRdsRegion(initialHostInfo.host); + if (!this.homeRegion) { + throw new AwsWrapperError(Messages.get("GlobalDbFailoverPlugin.missingHomeRegion")); + } + } + + logger.debug(Messages.get("Failover.parameterValue", "failoverHomeRegion", this.homeRegion)); + + const activeHomeMode = WrapperProperties.ACTIVE_HOME_FAILOVER_MODE.get(this.properties); + const inactiveHomeMode = WrapperProperties.INACTIVE_HOME_FAILOVER_MODE.get(this.properties); + + this.activeHomeFailoverMode = globalDbFailoverModeFromValue(activeHomeMode); + this.inactiveHomeFailoverMode = globalDbFailoverModeFromValue(inactiveHomeMode); + + if (this.activeHomeFailoverMode === GlobalDbFailoverMode.UNKNOWN) { + switch (this.rdsUrlType) { + case RdsUrlType.RDS_WRITER_CLUSTER: + case RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER: + this.activeHomeFailoverMode = GlobalDbFailoverMode.STRICT_WRITER; + break; + default: + this.activeHomeFailoverMode = GlobalDbFailoverMode.HOME_READER_OR_WRITER; + } + } + + if (this.inactiveHomeFailoverMode === GlobalDbFailoverMode.UNKNOWN) { + switch (this.rdsUrlType) { + case RdsUrlType.RDS_WRITER_CLUSTER: + case RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER: + this.inactiveHomeFailoverMode = GlobalDbFailoverMode.STRICT_WRITER; + break; + default: + this.inactiveHomeFailoverMode = GlobalDbFailoverMode.HOME_READER_OR_WRITER; + } + } + + logger.debug(Messages.get("Failover.parameterValue", "activeHomeFailoverMode", this.activeHomeFailoverMode)); + logger.debug(Messages.get("Failover.parameterValue", "inactiveHomeFailoverMode", this.inactiveHomeFailoverMode)); + } + + override async failover(): Promise { + const telemetryFactory = this.pluginService.getTelemetryFactory(); + const telemetryContext = telemetryFactory.openTelemetryContext(GlobalDbFailoverPlugin.TELEMETRY_FAILOVER, TelemetryTraceLevel.NESTED); + + const failoverStartTimeNs = getTimeInNanos(); + const failoverEndTimeNs = failoverStartTimeNs + BigInt(this.failoverTimeoutSettingMs) * BigInt(1_000_000); + + try { + await telemetryContext.start(async () => { + logger.info(Messages.get("GlobalDbFailoverPlugin.startFailover")); + + // Force refresh host list and wait for topology to stabilize + const refreshResult = await this.pluginService.forceMonitoringRefresh(true, this.failoverTimeoutSettingMs); + if (!refreshResult) { + this.failoverWriterTriggeredCounter.inc(); + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unableToRefreshHostList")); + throw new FailoverFailedError(Messages.get("Failover.unableToRefreshHostList")); + } + + const updatedHosts = this.pluginService.getAllHosts(); + const writerCandidate = getWriter(updatedHosts); + + if (!writerCandidate) { + this.failoverWriterTriggeredCounter.inc(); + this.failoverWriterFailedCounter.inc(); + const message = logTopology(updatedHosts, Messages.get("Failover.unableToDetermineWriter")); + logger.error(message); + throw new FailoverFailedError(message); + } + + // Check writer region to determine failover mode + const writerRegion = this.rdsHelper.getRdsRegion(writerCandidate.host); + const isHomeRegion = equalsIgnoreCase(this.homeRegion, writerRegion); + logger.debug(Messages.get("GlobalDbFailoverPlugin.isHomeRegion", String(isHomeRegion))); + + const currentFailoverMode = isHomeRegion ? this.activeHomeFailoverMode : this.inactiveHomeFailoverMode; + logger.debug(Messages.get("GlobalDbFailoverPlugin.currentFailoverMode", String(currentFailoverMode))); + + switch (currentFailoverMode) { + case GlobalDbFailoverMode.STRICT_WRITER: + await this.failoverToWriter(writerCandidate); + break; + case GlobalDbFailoverMode.STRICT_HOME_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER && this.isHostInHomeRegion(x)), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.STRICT_OUT_OF_HOME_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER && !this.isHostInHomeRegion(x)), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.STRICT_ANY_READER: + await this.failoverToAllowedHost( + () => this.pluginService.getHosts().filter((x) => x.role === HostRole.READER), + HostRole.READER, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.HOME_READER_OR_WRITER: + await this.failoverToAllowedHost( + () => + this.pluginService.getHosts().filter((x) => x.role === HostRole.WRITER || (x.role === HostRole.READER && this.isHostInHomeRegion(x))), + null, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.OUT_OF_HOME_READER_OR_WRITER: + await this.failoverToAllowedHost( + () => + this.pluginService + .getHosts() + .filter((x) => x.role === HostRole.WRITER || (x.role === HostRole.READER && !this.isHostInHomeRegion(x))), + null, + failoverEndTimeNs + ); + break; + case GlobalDbFailoverMode.ANY_READER_OR_WRITER: + await this.failoverToAllowedHost(() => [...this.pluginService.getHosts()], null, failoverEndTimeNs); + break; + case GlobalDbFailoverMode.UNKNOWN: + default: + throw new UnsupportedMethodError(`Unsupported failover mode: ${currentFailoverMode}`); + } + + logger.debug(Messages.get("Failover.establishedConnection", this.pluginService.getCurrentHostInfo()?.host ?? "unknown")); + this.throwFailoverSuccessException(); + }); + } finally { + logger.debug(Messages.get("GlobalDbFailoverPlugin.failoverElapsed", String(convertNanosToMs(getTimeInNanos() - failoverStartTimeNs)))); + + if (this.telemetryFailoverAdditionalTopTraceSetting && telemetryContext) { + await telemetryFactory.postCopy(telemetryContext, TelemetryTraceLevel.FORCE_TOP_LEVEL); + } + } + } + + private isHostInHomeRegion(host: HostInfo): boolean { + const hostRegion = this.rdsHelper.getRdsRegion(host.host); + return equalsIgnoreCase(hostRegion, this.homeRegion); + } + + protected async failoverToWriter(writerCandidate: HostInfo): Promise { + this.failoverWriterTriggeredCounter.inc(); + let writerCandidateConn: ClientWrapper | null = null; + + try { + const allowedHosts = this.pluginService.getHosts(); + if (!containsHostAndPort(allowedHosts, writerCandidate.hostAndPort)) { + this.failoverWriterFailedCounter.inc(); + const topologyString = logTopology(allowedHosts, ""); + logger.error(Messages.get("Failover.newWriterNotAllowed", writerCandidate.url, topologyString)); + throw new FailoverFailedError(Messages.get("Failover.newWriterNotAllowed", writerCandidate.url, topologyString)); + } + + try { + writerCandidateConn = await this.pluginService.connect(writerCandidate, this.properties, this); + } catch (error) { + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unableToConnectToWriterDueToError", writerCandidate.host, error.message)); + throw new FailoverFailedError(Messages.get("Failover.unableToConnectToWriterDueToError", writerCandidate.host, error.message)); + } + + const role = await this.pluginService.getHostRole(writerCandidateConn); + if (role !== HostRole.WRITER) { + await writerCandidateConn?.abort(); + writerCandidateConn = null; + this.failoverWriterFailedCounter.inc(); + logger.error(Messages.get("Failover.unexpectedReaderRole", writerCandidate.host)); + throw new FailoverFailedError(Messages.get("Failover.unexpectedReaderRole", writerCandidate.host)); + } + + await this.pluginService.setCurrentClient(writerCandidateConn, writerCandidate); + writerCandidateConn = null; // Prevent connection from being closed in finally block + + this.failoverWriterSuccessCounter.inc(); + } catch (ex) { + if (!(ex instanceof FailoverFailedError)) { + // Counter has already been incremented in Failover2Plugin before throwing the FailoverFailedError. + // So no need to increment again here. + this.failoverWriterFailedCounter.inc(); + } + throw ex; + } finally { + if (writerCandidateConn && this.pluginService.getCurrentClient().targetClient !== writerCandidateConn) { + await writerCandidateConn.abort(); + } + } + } + + protected async failoverToAllowedHost(getAllowedHosts: () => HostInfo[], verifyRole: HostRole | null, failoverEndTimeNs: bigint): Promise { + this.failoverReaderTriggeredCounter.inc(); + + let result: ReaderFailoverResult | null = null; + try { + try { + result = await this.getAllowedFailoverConnection(getAllowedHosts, verifyRole, failoverEndTimeNs); + await this.pluginService.setCurrentClient(result.client!, result.newHost!); + result = null; + } catch (e) { + if (e instanceof AwsTimeoutError) { + logger.error(Messages.get("Failover.unableToConnectToReader")); + throw new FailoverFailedError(Messages.get("Failover.unableToConnectToReader")); + } + throw e; + } + + logger.info(Messages.get("Failover.establishedConnection", this.pluginService.getCurrentHostInfo()?.host ?? "unknown")); + this.throwFailoverSuccessException(); + } catch (ex) { + if (ex instanceof FailoverSuccessError) { + this.failoverReaderSuccessCounter.inc(); + } else { + this.failoverReaderFailedCounter.inc(); + } + throw ex; + } finally { + if (result?.client && result?.client !== this.pluginService.getCurrentClient().targetClient) { + await result?.client.abort(); + } + } + } + + protected async getAllowedFailoverConnection( + getAllowedHosts: () => HostInfo[], + verifyRole: HostRole | null, + failoverEndTimeNs: bigint + ): Promise { + do { + await this.pluginService.refreshHostList(); + let updatedAllowedHosts = getAllowedHosts(); + + // Make a copy of hosts and set their availability + updatedAllowedHosts = updatedAllowedHosts.map((x) => + this.pluginService.getHostInfoBuilder().copyFrom(x).withAvailability(HostAvailability.AVAILABLE).build() + ); + + const remainingAllowedHosts = [...updatedAllowedHosts]; + + if (remainingAllowedHosts.length === 0) { + await sleep(100); + continue; + } + + while (remainingAllowedHosts.length > 0 && getTimeInNanos() < failoverEndTimeNs) { + let candidateHost: HostInfo | undefined; + try { + candidateHost = this.pluginService.getHostInfoByStrategy(verifyRole, this.failoverReaderHostSelectorStrategy, remainingAllowedHosts); + } catch { + // Strategy can't get a host according to requested conditions. + // Do nothing + } + + if (!candidateHost) { + logger.debug( + logTopology( + remainingAllowedHosts, + `${Messages.get("GlobalDbFailoverPlugin.unableToFindCandidateWithMatchingRole", String(verifyRole), this.failoverReaderHostSelectorStrategy)}` + ) + ); + await sleep(100); + break; + } + + let candidateConn: ClientWrapper | null = null; + try { + candidateConn = await this.pluginService.connect(candidateHost, this.properties, this); + // Since the roles in the host list might not be accurate, we execute a query to check the instance's role + const role = verifyRole === null ? null : await this.pluginService.getHostRole(candidateConn); + + if (verifyRole === null || verifyRole === role) { + const updatedHostSpec = this.pluginService + .getHostInfoBuilder() + .copyFrom(candidateHost) + .withRole(role ?? candidateHost.role) + .build(); + return new ReaderFailoverResult(candidateConn, updatedHostSpec, true); + } + + // The role is not as expected, so the connection is not valid + const index = remainingAllowedHosts.findIndex((h) => h.hostAndPort === candidateHost!.hostAndPort); + if (index !== -1) { + remainingAllowedHosts.splice(index, 1); + } + await candidateConn.abort(); + candidateConn = null; + } catch { + const index = remainingAllowedHosts.findIndex((h) => h.hostAndPort === candidateHost!.hostAndPort); + if (index !== -1) { + remainingAllowedHosts.splice(index, 1); + } + if (candidateConn) { + await candidateConn.abort(); + } + } + } + } while (getTimeInNanos() < failoverEndTimeNs); // All hosts failed. Keep trying until we hit the timeout. + + throw new AwsTimeoutError(Messages.get("Failover.failoverReaderTimeout")); + } +} diff --git a/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts b/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts new file mode 100644 index 00000000..25950509 --- /dev/null +++ b/common/lib/plugins/gdb_failover/global_db_failover_plugin_factory.ts @@ -0,0 +1,38 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { PluginService } from "../../plugin_service"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { RdsUtils } from "../../utils/rds_utils"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GlobalDbFailoverPluginFactory extends ConnectionPluginFactory { + private static globalDbFailoverPlugin: any; + + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + try { + if (!GlobalDbFailoverPluginFactory.globalDbFailoverPlugin) { + GlobalDbFailoverPluginFactory.globalDbFailoverPlugin = await import("./global_db_failover_plugin"); + } + return new GlobalDbFailoverPluginFactory.globalDbFailoverPlugin.GlobalDbFailoverPlugin(servicesContainer, properties, new RdsUtils()); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "GlobalDbFailoverPlugin")); + } + } +} diff --git a/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts b/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts index 330c02e6..b3267766 100644 --- a/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts +++ b/common/lib/plugins/limitless/limitless_connection_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class LimitlessConnectionPluginFactory implements ConnectionPluginFactory { private static limitlessPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!LimitlessConnectionPluginFactory.limitlessPlugin) { LimitlessConnectionPluginFactory.limitlessPlugin = await import("./limitless_connection_plugin"); } - return new LimitlessConnectionPluginFactory.limitlessPlugin.LimitlessConnectionPlugin(pluginService, properties); + return new LimitlessConnectionPluginFactory.limitlessPlugin.LimitlessConnectionPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "LimitlessConnectionPlugin")); } diff --git a/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts index 65482211..14008e4a 100644 --- a/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts +++ b/common/lib/plugins/read_write_splitting/abstract_read_write_splitting_plugin.ts @@ -282,7 +282,6 @@ export abstract class AbstractReadWriteSplittingPlugin extends AbstractConnectio } async closeIdleClients() { - logger.debug(Messages.get("ReadWriteSplittingPlugin.closingInternalClients")); await this.closeReaderClientIfIdle(); await this.closeWriterClientIfIdle(); } diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts index 397d5dfb..6ade242f 100644 --- a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts @@ -61,7 +61,7 @@ export class ReadWriteSplittingPlugin extends AbstractReadWriteSplittingPlugin { } const result = await connectFunc(); - if (!isInitialConnection || this._hostListProviderService?.isStaticHostListProvider()) { + if (!isInitialConnection || !this._hostListProviderService?.isDynamicHostListProvider()) { return result; } const currentRole = this.pluginService.getCurrentHostInfo()?.role; diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts index 62485db1..ac7fd53b 100644 --- a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class ReadWriteSplittingPluginFactory extends ConnectionPluginFactory { private static readWriteSplittingPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!ReadWriteSplittingPluginFactory.readWriteSplittingPlugin) { ReadWriteSplittingPluginFactory.readWriteSplittingPlugin = await import("./read_write_splitting_plugin"); } - return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(pluginService, properties); + return new ReadWriteSplittingPluginFactory.readWriteSplittingPlugin.ReadWriteSplittingPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "readWriteSplittingPlugin")); } diff --git a/common/lib/plugins/stale_dns/stale_dns_helper.ts b/common/lib/plugins/stale_dns/stale_dns_helper.ts index 501fd5bf..d1b71535 100644 --- a/common/lib/plugins/stale_dns/stale_dns_helper.ts +++ b/common/lib/plugins/stale_dns/stale_dns_helper.ts @@ -14,28 +14,26 @@ limitations under the License. */ -import { logger } from "../../../logutils"; +import { levels, logger } from "../../../logutils"; import { HostInfo } from "../../host_info"; import { HostListProviderService } from "../../host_list_provider_service"; import { HostRole } from "../../host_role"; import { PluginService } from "../../plugin_service"; import { Messages } from "../../utils/messages"; import { RdsUtils } from "../../utils/rds_utils"; -import { lookup, LookupAddress } from "dns"; -import { promisify } from "util"; -import { AwsWrapperError } from "../../utils/errors"; import { HostChangeOptions } from "../../host_change_options"; import { WrapperProperties } from "../../wrapper_property"; import { ClientWrapper } from "../../client_wrapper"; -import { getWriter, logTopology } from "../../utils/utils"; +import { containsHostAndPort, getWriter, logTopology } from "../../utils/utils"; import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; +import { RdsUrlType } from "../../utils/rds_url_type"; +import { AwsWrapperError } from "../../utils/errors"; export class StaleDnsHelper { private readonly pluginService: PluginService; private readonly rdsUtils: RdsUtils = new RdsUtils(); private writerHostInfo: HostInfo | null = null; - private writerHostAddress: string = ""; private readonly telemetryFactory: TelemetryFactory; private readonly staleDNSDetectedCounter: TelemetryCounter; @@ -53,39 +51,44 @@ export class StaleDnsHelper { props: Map, connectFunc: () => Promise ): Promise { - if (!this.rdsUtils.isWriterClusterDns(host)) { - return connectFunc(); - } + const type: RdsUrlType = this.rdsUtils.identifyRdsType(host); - const currentTargetClient = await connectFunc(); - - let clusterInetAddress = ""; - try { - const lookupResult = await this.lookupResult(host); - clusterInetAddress = lookupResult.address; - } catch (error) { - // ignore + if (type !== RdsUrlType.RDS_WRITER_CLUSTER && type !== RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER) { + return connectFunc(); } - const hostInetAddress = clusterInetAddress; - logger.debug(Messages.get("StaleDnsHelper.clusterEndpointDns", hostInetAddress)); - - if (!clusterInetAddress) { - return currentTargetClient; + if (type === RdsUrlType.RDS_WRITER_CLUSTER) { + const writer = getWriter(this.pluginService.getAllHosts()); + if (writer != null && this.rdsUtils.isRdsInstance(writer.host)) { + if ( + isInitialConnection && + WrapperProperties.SKIP_INACTIVE_WRITER_CLUSTER_CHECK.get(props) && + !this.rdsUtils.isSameRegion(writer.host, host) + ) { + // The cluster writer endpoint belongs to a different region than the current writer region. + // It means that the cluster is Aurora Global Database and cluster writer endpoint is in secondary region. + // In this case the cluster writer endpoint is in inactive state and doesn't represent the current writer + // so any connection check should be skipped. + // Continue with a normal workflow. + return connectFunc(); + } + } else { + // No writer is available. It could be the case with the first connection when topology isn't yet available. + // Continue with a normal workflow. + return connectFunc(); + } } - const currentHostInfo = this.pluginService.getCurrentHostInfo(); - if (!currentHostInfo) { - throw new AwsWrapperError("Stale DNS Helper: Current hostInfo was null."); - } + const currentTargetClient = await connectFunc(); - if (currentHostInfo && currentHostInfo.role === HostRole.READER) { + const isConnectedToReader: boolean = (await this.pluginService.getHostRole(currentTargetClient)) === HostRole.READER; + if (isConnectedToReader) { // This is if-statement is only reached if the connection url is a writer cluster endpoint. // If the new connection resolves to a reader instance, this means the topology is outdated. // Force refresh to update the topology. - await this.pluginService.forceRefreshHostList(currentTargetClient); + await this.pluginService.forceRefreshHostList(); } else { - await this.pluginService.refreshHostList(currentTargetClient); + await this.pluginService.refreshHostList(); } logger.debug(logTopology(this.pluginService.getAllHosts(), "[StaleDnsHelper.getVerifiedConnection] ")); @@ -104,27 +107,18 @@ export class StaleDnsHelper { return currentTargetClient; } - if (!this.writerHostAddress) { - try { - const lookupResult = await this.lookupResult(this.writerHostInfo.host); - this.writerHostAddress = lookupResult.address; - } catch (error) { - // ignore - } - } - - logger.debug(Messages.get("StaleDnsHelper.writerInetAddress", this.writerHostAddress)); - - if (!this.writerHostAddress) { - return currentTargetClient; - } + if (isConnectedToReader) { + // Reconnect to writer host if current connection is reader. - if (this.writerHostAddress !== clusterInetAddress) { - // DNS resolves a cluster endpoint to a wrong writer - // opens a connection to a proper writer host logger.debug(Messages.get("StaleDnsHelper.staleDnsDetected", this.writerHostInfo.host)); this.staleDNSDetectedCounter.inc(); + const allowedHosts: HostInfo[] = this.pluginService.getHosts(); + + if (!containsHostAndPort(allowedHosts, this.writerHostInfo.hostAndPort)) { + throw new AwsWrapperError(Messages.get("StaleDnsHelper.currentWriterNotAllowed", this.writerHostInfo.host, logTopology(allowedHosts, ""))); + } + let targetClient = null; try { const newProps = new Map(props); @@ -149,7 +143,7 @@ export class StaleDnsHelper { } for (const [key, values] of changes.entries()) { - if (logger.level === "debug") { + if (levels[logger.level] <= levels.debug) { const valStr = Array.from(values) .map((x) => HostChangeOptions[x]) .join(", "); @@ -159,14 +153,9 @@ export class StaleDnsHelper { if (key === this.writerHostInfo.url && values.has(HostChangeOptions.PROMOTED_TO_READER)) { logger.debug(Messages.get("StaleDnsHelper.reset")); this.writerHostInfo = null; - this.writerHostAddress = ""; } } } return Promise.resolve(); } - - lookupResult(host: string): Promise { - return promisify(lookup)(host, {}); - } } diff --git a/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts b/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts index 8669e59d..2c6d6740 100644 --- a/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts +++ b/common/lib/plugins/stale_dns/stale_dns_plugin_factory.ts @@ -15,20 +15,20 @@ */ import { ConnectionPluginFactory } from "../../plugin_factory"; -import { PluginService } from "../../plugin_service"; import { ConnectionPlugin } from "../../connection_plugin"; import { AwsWrapperError } from "../../utils/errors"; import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class StaleDnsPluginFactory extends ConnectionPluginFactory { private static staleDnsPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!StaleDnsPluginFactory.staleDnsPlugin) { StaleDnsPluginFactory.staleDnsPlugin = await import("./stale_dns_plugin"); } - return new StaleDnsPluginFactory.staleDnsPlugin.StaleDnsPlugin(pluginService, properties); + return new StaleDnsPluginFactory.staleDnsPlugin.StaleDnsPlugin(servicesContainer.pluginService, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "StaleDnsPlugin")); } diff --git a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts index 05146d74..9518fdfd 100644 --- a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts +++ b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts @@ -15,20 +15,23 @@ */ import { ConnectionPluginFactory } from "../../../plugin_factory"; -import { PluginService } from "../../../plugin_service"; import { ConnectionPlugin } from "../../../connection_plugin"; import { AwsWrapperError } from "../../../utils/errors"; import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class FastestResponseStrategyPluginFactory extends ConnectionPluginFactory { private static fastestResponseStrategyPlugin: any; - async getInstance(pluginService: PluginService, properties: Map): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { try { if (!FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin) { FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin = await import("./fastest_response_strategy_plugin"); } - return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin(pluginService, properties); + return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin( + servicesContainer.pluginService, + properties + ); } catch (error: any) { throw new AwsWrapperError( Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FastestResponseStrategyPluginFactory") diff --git a/common/lib/random_host_selector.ts b/common/lib/random_host_selector.ts index d38d985c..5d35f0eb 100644 --- a/common/lib/random_host_selector.ts +++ b/common/lib/random_host_selector.ts @@ -25,7 +25,9 @@ export class RandomHostSelector implements HostSelector { public static STRATEGY_NAME = "random"; getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { - const eligibleHosts = hosts.filter((hostInfo: HostInfo) => hostInfo.role === role && hostInfo.getAvailability() === HostAvailability.AVAILABLE); + const eligibleHosts = hosts.filter( + (hostInfo: HostInfo) => (role === null || hostInfo.role === role) && hostInfo.getAvailability() === HostAvailability.AVAILABLE + ); if (eligibleHosts.length === 0) { throw new AwsWrapperError(Messages.get("HostSelector.noHostsMatchingRole", role)); } diff --git a/common/lib/round_robin_host_selector.ts b/common/lib/round_robin_host_selector.ts index bc53a251..d6fcd621 100644 --- a/common/lib/round_robin_host_selector.ts +++ b/common/lib/round_robin_host_selector.ts @@ -31,7 +31,7 @@ export class RoundRobinHostSelector implements HostSelector { getHost(hosts: HostInfo[], role: HostRole, props?: Map): HostInfo { const eligibleHosts: HostInfo[] = hosts - .filter((host: HostInfo) => host.role === role && host.availability === HostAvailability.AVAILABLE) + .filter((host: HostInfo) => (role === null || host.role === role) && host.availability === HostAvailability.AVAILABLE) .sort((hostA: HostInfo, hostB: HostInfo) => { const hostAHostName = hostA.host.toLowerCase(); const hostBHostName = hostB.host.toLowerCase(); diff --git a/common/lib/session_state_client.ts b/common/lib/session_state_client.ts index 8c978f22..cd660158 100644 --- a/common/lib/session_state_client.ts +++ b/common/lib/session_state_client.ts @@ -19,11 +19,11 @@ import { TransactionIsolationLevel } from "./utils/transaction_isolation_level"; export interface SessionStateClient { setReadOnly(readOnly: boolean): Promise; - isReadOnly(): boolean; + isReadOnly(): boolean | undefined; setAutoCommit(autoCommit: boolean): Promise; - getAutoCommit(): boolean; + getAutoCommit(): boolean | undefined; setTransactionIsolation(level: TransactionIsolationLevel): Promise; @@ -31,9 +31,9 @@ export interface SessionStateClient { setSchema(schema: any): Promise; - getSchema(): string; + getSchema(): string | undefined; setCatalog(catalog: string): Promise; - getCatalog(): string; + getCatalog(): string | undefined; } diff --git a/common/lib/types.ts b/common/lib/types.ts index 3332b0c9..a89e6af4 100644 --- a/common/lib/types.ts +++ b/common/lib/types.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ /** * Type representing a constructor for any class. diff --git a/common/lib/utils/cache_map.ts b/common/lib/utils/cache_map.ts index 19485421..69650202 100644 --- a/common/lib/utils/cache_map.ts +++ b/common/lib/utils/cache_map.ts @@ -17,7 +17,7 @@ import { getTimeInNanos } from "./utils"; export class CacheItem { - private readonly item: V; + readonly item: V; private _expirationTimeNs: bigint; constructor(item: V, expirationTime: bigint) { diff --git a/common/lib/utils/core_services_container.ts b/common/lib/utils/core_services_container.ts index 8ae9f27f..c3bed54d 100644 --- a/common/lib/utils/core_services_container.ts +++ b/common/lib/utils/core_services_container.ts @@ -1,20 +1,23 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { StorageService, StorageServiceImpl } from "./storage/storage_service"; +import { MonitorService, MonitorServiceImpl } from "./monitoring/monitor_service"; +import { EventPublisher } from "./events/event"; +import { BatchingEventPublisher } from "./events/batching_event_publisher"; /** * A singleton container object used to instantiate and access core universal services. This class should be used @@ -26,27 +29,25 @@ import { StorageService, StorageServiceImpl } from "./storage/storage_service"; export class CoreServicesContainer { private static readonly INSTANCE = new CoreServicesContainer(); - // private readonly monitorService: MonitorService; // TODO: implement monitor service - private readonly storageService: StorageService; + readonly monitorService: MonitorService; + readonly storageService: StorageService; + readonly eventPublisher: EventPublisher; private constructor() { - this.storageService = new StorageServiceImpl(); - // this.monitorService = new MonitorServiceImpl(); + this.eventPublisher = new BatchingEventPublisher(); + this.storageService = new StorageServiceImpl(this.eventPublisher); + this.monitorService = new MonitorServiceImpl(this.eventPublisher); } static getInstance(): CoreServicesContainer { return CoreServicesContainer.INSTANCE; } - getStorageService(): StorageService { - return this.storageService; - } - - // getMonitorService(): MonitorService { - // return this.monitorService; - // } - - static releaseResources(): void { - CoreServicesContainer.INSTANCE.storageService.releaseResources(); + static async releaseResources(): Promise { + await CoreServicesContainer.INSTANCE.storageService.releaseResources(); + await CoreServicesContainer.INSTANCE.monitorService.releaseResources(); + if (CoreServicesContainer.INSTANCE.eventPublisher instanceof BatchingEventPublisher) { + CoreServicesContainer.INSTANCE.eventPublisher.releaseResources(); + } } } diff --git a/common/lib/utils/errors.ts b/common/lib/utils/errors.ts index 4d854c0c..247bbba0 100644 --- a/common/lib/utils/errors.ts +++ b/common/lib/utils/errors.ts @@ -50,6 +50,8 @@ export class TransactionResolutionUnknownError extends FailoverError {} export class LoginError extends AwsWrapperError {} -export class InternalQueryTimeoutError extends AwsWrapperError {} +export class AwsTimeoutError extends AwsWrapperError {} + +export class InternalQueryTimeoutError extends AwsTimeoutError {} export class UnavailableHostError extends AwsWrapperError {} diff --git a/common/lib/utils/events/batching_event_publisher.ts b/common/lib/utils/events/batching_event_publisher.ts new file mode 100644 index 00000000..750afdee --- /dev/null +++ b/common/lib/utils/events/batching_event_publisher.ts @@ -0,0 +1,99 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event, EventClass, EventPublisher, EventSubscriber } from "./event"; +import { Messages } from "../messages"; +import { logger } from "../../../logutils"; + +const DEFAULT_MESSAGE_INTERVAL_MS = 30_000; // 30 seconds + +/** + * An event publisher that periodically publishes a batch of all unique events + * encountered during the latest time interval. + */ +export class BatchingEventPublisher implements EventPublisher { + protected readonly subscribersMap = new Map>(); + protected readonly pendingEvents = new Set(); + protected publishingInterval?: ReturnType; + + constructor(messageIntervalMs: number = DEFAULT_MESSAGE_INTERVAL_MS) { + this.initPublishingInterval(messageIntervalMs); + } + + protected initPublishingInterval(messageIntervalMs: number): void { + this.publishingInterval = setInterval(() => this.sendMessages(), messageIntervalMs); + // Unref the timer to prevent this background task from blocking the application from gracefully exiting. + this.publishingInterval.unref(); + } + + protected async sendMessages(): Promise { + for (const event of this.pendingEvents) { + this.pendingEvents.delete(event); + await this.deliverEvent(event); + } + } + + protected async deliverEvent(event: Event): Promise { + const subscribers = this.subscribersMap.get(event.constructor as EventClass); + if (!subscribers) { + return; + } + + for (const subscriber of subscribers) { + await subscriber.processEvent(event); + } + } + + subscribe(subscriber: EventSubscriber, eventClasses: Set): void { + for (const eventClass of eventClasses) { + let subscribers = this.subscribersMap.get(eventClass); + if (!subscribers) { + subscribers = new Set(); + this.subscribersMap.set(eventClass, subscribers); + } + subscribers.add(subscriber); + } + } + + unsubscribe(subscriber: EventSubscriber, eventClasses: Set): void { + for (const eventClass of eventClasses) { + const subscribers = this.subscribersMap.get(eventClass); + if (subscribers) { + subscribers.delete(subscriber); + if (subscribers.size === 0) { + this.subscribersMap.delete(eventClass); + } + } + } + } + + publish(event: Event): void { + if (event.isImmediateDelivery) { + this.deliverEvent(event).catch((err) => { + logger.debug(Messages.get("BatchingEventPublisher.errorDeliveringImmediateEvent", err?.message ?? String(err))); + }); + } else { + this.pendingEvents.add(event); + } + } + + releaseResources(): void { + if (this.publishingInterval) { + clearInterval(this.publishingInterval); + this.publishingInterval = undefined; + } + } +} diff --git a/common/lib/utils/events/data_access_event.ts b/common/lib/utils/events/data_access_event.ts new file mode 100644 index 00000000..bb58a168 --- /dev/null +++ b/common/lib/utils/events/data_access_event.ts @@ -0,0 +1,35 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; + +/** + * A class defining a data access event. The class specifies the class of the data + * that was accessed and the key for the data. + * + * Used by StorageService to notify MonitorService when data is accessed, + * allowing monitors to extend their expiration time. + */ +export class DataAccessEvent implements Event { + readonly isImmediateDelivery = false; + readonly dataClass: new (...args: any[]) => any; + readonly key: unknown; + + constructor(dataClass: new (...args: any[]) => any, key: unknown) { + this.dataClass = dataClass; + this.key = key; + } +} diff --git a/common/lib/utils/events/event.ts b/common/lib/utils/events/event.ts new file mode 100644 index 00000000..8a69c643 --- /dev/null +++ b/common/lib/utils/events/event.ts @@ -0,0 +1,64 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Constructor } from "../../types"; + +export type EventClass = Constructor; + +/** + * An interface for events that need to be communicated between different components. + */ +export interface Event { + readonly isImmediateDelivery: boolean; +} + +/** + * An event subscriber. Subscribers can subscribe to a publisher's events. + */ +export interface EventSubscriber { + /** + * Processes an event. This method will only be called on this subscriber + * if it has subscribed to the event class. + * @param event the event to process. + */ + processEvent(event: Event): Promise; +} + +/** + * An event publisher that publishes events to subscribers. + * Subscribers can specify which types of events they would like to receive. + */ +export interface EventPublisher { + /** + * Registers the given subscriber for the given event classes. + * @param subscriber the subscriber to be notified when the given event classes occur. + * @param eventClasses the classes of events that the subscriber should be notified of. + */ + subscribe(subscriber: EventSubscriber, eventClasses: Set): void; + + /** + * Unsubscribes the given subscriber from the given event classes. + * @param subscriber the subscriber to unsubscribe from the given event classes. + * @param eventClasses the classes of events that the subscriber wants to unsubscribe from. + */ + unsubscribe(subscriber: EventSubscriber, eventClasses: Set): void; + + /** + * Publishes an event. All subscribers to the given event class will be notified of the event. + * @param event the event to publish. + */ + publish(event: Event): void; +} diff --git a/common/lib/utils/events/monitor_reset_event.ts b/common/lib/utils/events/monitor_reset_event.ts new file mode 100644 index 00000000..2acf3bf9 --- /dev/null +++ b/common/lib/utils/events/monitor_reset_event.ts @@ -0,0 +1,32 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; + +/** + * Event indicating that a monitor should be reset with new endpoints. + * Used by ClusterTopologyMonitorImpl to reset monitoring when cluster topology changes. + */ +export class MonitorResetEvent implements Event { + readonly isImmediateDelivery = true; + readonly clusterId: string; + readonly endpoints: Set; + + constructor(clusterId: string, endpoints: Set) { + this.clusterId = clusterId; + this.endpoints = endpoints; + } +} diff --git a/common/lib/utils/events/monitor_stop_event.ts b/common/lib/utils/events/monitor_stop_event.ts new file mode 100644 index 00000000..029fa35e --- /dev/null +++ b/common/lib/utils/events/monitor_stop_event.ts @@ -0,0 +1,33 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Event } from "./event"; +import { Monitor } from "../monitoring/monitor"; + +/** + * Event indicating that a monitor should be stopped. + * Used by MonitorService to stop and remove monitors. + */ +export class MonitorStopEvent implements Event { + readonly isImmediateDelivery = true; + readonly monitorClass: new (...args: any[]) => Monitor; + readonly key: unknown; + + constructor(monitorClass: new (...args: any[]) => Monitor, key: unknown) { + this.monitorClass = monitorClass; + this.key = key; + } +} diff --git a/common/lib/utils/full_services_container.ts b/common/lib/utils/full_services_container.ts new file mode 100644 index 00000000..59902f88 --- /dev/null +++ b/common/lib/utils/full_services_container.ts @@ -0,0 +1,67 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { PluginService } from "../plugin_service"; +import { HostListProviderService } from "../host_list_provider_service"; +import { PluginManager } from "../index"; +import { ConnectionProvider } from "../connection_provider"; +import { TelemetryFactory } from "./telemetry/telemetry_factory"; +import { StorageService } from "./storage/storage_service"; +import { MonitorService } from "./monitoring/monitor_service"; +import { EventPublisher } from "./events/event"; +import { ImportantEventService } from "./important_event_service"; + +/** + * Container for services used throughout the wrapper. + */ +export interface FullServicesContainer { + storageService: StorageService; + monitorService: MonitorService; + eventPublisher: EventPublisher; + readonly defaultConnectionProvider: ConnectionProvider; + telemetryFactory: TelemetryFactory; + pluginManager: PluginManager; + hostListProviderService: HostListProviderService; + pluginService: PluginService; + importantEventService: ImportantEventService; +} + +export class FullServicesContainerImpl implements FullServicesContainer { + storageService: StorageService; + monitorService: MonitorService; + eventPublisher: EventPublisher; + readonly defaultConnectionProvider: ConnectionProvider; + telemetryFactory: TelemetryFactory; + pluginManager!: PluginManager; + hostListProviderService!: HostListProviderService; + pluginService!: PluginService; + importantEventService: ImportantEventService; + + constructor( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + defaultConnProvider: ConnectionProvider, + telemetryFactory: TelemetryFactory + ) { + this.storageService = storageService; + this.monitorService = monitorService; + this.eventPublisher = eventPublisher; + this.defaultConnectionProvider = defaultConnProvider; + this.telemetryFactory = telemetryFactory; + this.importantEventService = new ImportantEventService(); + } +} diff --git a/common/lib/utils/gdb_region_utils.ts b/common/lib/utils/gdb_region_utils.ts index 6b1fa5a4..815cc64e 100644 --- a/common/lib/utils/gdb_region_utils.ts +++ b/common/lib/utils/gdb_region_utils.ts @@ -66,10 +66,8 @@ export class GDBRegionUtils extends RegionUtils { const response = await rdsClient.send(command); return this.extractWriterClusterArn(response.GlobalClusters); } catch (error) { - if (error instanceof Error) { - logger.debug(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); - throw new AwsWrapperError(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); - } + logger.debug(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); + throw new AwsWrapperError(Messages.get("GDBRegionUtils.unableToRetrieveGlobalClusterARN")); } finally { rdsClient.destroy(); } diff --git a/common/lib/utils/important_event_service.ts b/common/lib/utils/important_event_service.ts new file mode 100644 index 00000000..5ae19fd1 --- /dev/null +++ b/common/lib/utils/important_event_service.ts @@ -0,0 +1,88 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export class ImportantEvent { + readonly timestamp: Date; + readonly description: string; + + constructor(timestamp: Date, description: string) { + this.timestamp = timestamp; + this.description = description; + } +} + +export class ImportantEventService { + private static readonly DEFAULT_EVENT_QUEUE_MS = 60000; + + private readonly events: ImportantEvent[] = []; + private readonly eventQueueMs: number; + private readonly isEnabled: boolean; + + constructor(isEnabled: boolean = true, eventQueueMs: number = ImportantEventService.DEFAULT_EVENT_QUEUE_MS) { + this.isEnabled = isEnabled; + this.eventQueueMs = eventQueueMs; + } + + clear(): void { + this.events.length = 0; + } + + registerEvent(descriptionSupplier: () => string): void { + if (!this.isEnabled) { + return; + } + + this.removeExpiredEvents(); + + this.events.push(new ImportantEvent(new Date(), descriptionSupplier())); + } + + getEvents(): ImportantEvent[] { + if (!this.isEnabled) { + return []; + } + + this.removeExpiredEvents(); + return [...this.events]; + } + + private removeExpiredEvents(): void { + if (!this.isEnabled || this.events.length === 0) { + return; + } + + const current = Date.now(); + const cutoffTime = current - this.eventQueueMs; + + while (this.events.length > 0 && this.events[0].timestamp.getTime() <= cutoffTime) { + this.events.shift(); + } + } +} + +export class DriverImportantEventService { + private static readonly INSTANCE = new ImportantEventService(true, 60000); + + private constructor() {} + + static getInstance(): ImportantEventService { + return DriverImportantEventService.INSTANCE; + } + + static clear(): void { + DriverImportantEventService.INSTANCE.clear(); + } +} diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 1f46e624..353c00de 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -89,7 +89,11 @@ const MESSAGES: Record = { "Failover.unableToConnectToWriter": "Unable to establish SQL connection to the writer instance.", "Failover.unableToConnectToWriterDueToError": "Unable to establish SQL connection to the writer instance: %s due to error: %s.", "Failover.unableToConnectToReader": "Unable to establish SQL connection to the reader instance.", + "Failover.unableToRefreshHostList": "The request to discover the new topology timed out or was unsuccessful.", "Failover.unableToDetermineWriter": "Unable to determine the current writer instance.", + "Failover.unexpectedReaderRole": "The new writer was identified to be '%s', but querying the instance for its role returned a role of %s.", + "Failover.strictReaderUnknownHostRole": + "Unable to determine host role for '%s'. Since failover mode is set to STRICT_READER and the host may be a writer, it will not be selected for reader failover.", "Failover.detectedError": "[Failover] Detected an error while executing a command: %s", "Failover.failoverDisabled": "Cluster-aware failover is disabled.", "Failover.establishedConnection": "[Failover] Connected to %s", @@ -99,6 +103,7 @@ const MESSAGES: Record = { "Failover.noOperationsAfterConnectionClosed": "No operations allowed after client ended.", "Failover.transactionResolutionUnknownError": "Unknown transaction resolution error occurred during failover.", "Failover.connectionExplicitlyClosed": "Unable to failover on an explicitly closed connection.", + "Failover.failoverReaderTimeout": "The reader failover process was not able to establish a connection before timing out.", "Failover.timeoutError": "Internal failover task has timed out.", "Failover.newWriterNotAllowed": "The failover process identified the new writer but the host is not in the list of allowed hosts. New writer host: '%s'. Allowed hosts: '%s'.", @@ -109,12 +114,9 @@ const MESSAGES: Record = { "StaleDnsHelper.staleDnsDetected": "Stale DNS data detected. Opening a connection to '%s'.", "StaleDnsHelper.reset": "Reset stored writer host.", "StaleDnsPlugin.requireDynamicProvider": "Dynamic host list provider is required.", + "StaleDnsHelper.currentWriterNotAllowed": "The current writer is not in the list of allowed hosts. Current host: '%s'. Allowed hosts: %s", "Client.methodNotSupported": "Method '%s' not supported.", "Client.invalidTransactionIsolationLevel": "An invalid transaction isolation level was provided: '%s'.", - "AuroraStaleDnsHelper.clusterEndpointDns": "Cluster endpoint resolves to '%s'.", - "AuroraStaleDnsHelper.writerHostSpec": "Writer host: '%s'.", - "AuroraStaleDnsHelper.writerInetAddress": "Writer host address: '%s'", - "AuroraStaleDnsHelper.staleDnsDetected": "Stale DNS data detected. Opening a connection to '%s'.", "ReadWriteSplittingPlugin.setReadOnlyOnClosedClient": "setReadOnly cannot be called on a closed client '%s'.", "ReadWriteSplittingPlugin.errorSwitchingToCachedReader": "An error occurred while trying to switch to a cached reader client: '%s'. Error message: '%s'. The driver will attempt to establish a new reader client.", @@ -138,7 +140,8 @@ const MESSAGES: Record = { "ReadWriteSplittingPlugin.failoverErrorWhileExecutingCommand": "Detected a failover error while executing a command: '%s'", "ReadWriteSplittingPlugin.noReadersAvailable": "The plugin was unable to establish a reader client to any reader instance.", "ReadWriteSplittingPlugin.successfullyConnectedToReader": "Successfully connected to a new reader host: '%s'", - "ReadWriteSplittingPlugin.previousReaderNotAllowed": "The previous reader connection cannot be used because it is no longer in the list of allowed hosts. Previous reader: %s. Allowed hosts: %s", + "ReadWriteSplittingPlugin.previousReaderNotAllowed": + "The previous reader connection cannot be used because it is no longer in the list of allowed hosts. Previous reader: %s. Allowed hosts: %s", "ReadWriteSplittingPlugin.failedToConnectToReader": "Failed to connect to reader host: '%s'", "ReadWriteSplittingPlugin.unsupportedHostSelectorStrategy": "Unsupported host selection strategy '%s' specified in plugin configuration parameter 'readerHostSelectorStrategy'. Please visit the Read/Write Splitting Plugin documentation for all supported strategies.", @@ -190,19 +193,31 @@ const MESSAGES: Record = { "MonitorImpl.stopMonitoringTaskNewContext": "Stop monitoring task for checking new contexts for '%s'", "MonitorService.startMonitoringNullMonitor": "Start monitoring called but could not find monitor for host: '%s'.", "MonitorService.emptyAliasSet": "Empty alias set passed for '%s'. Set should not be empty.", + "MonitorService.monitorClassMismatch": + "The monitor stored at '%s' did not have the expected type. The expected type was '%s', but the monitor '%s' had a type of '%s'.", + "MonitorService.monitorStuck": "Monitor '%s' has not been updated within the inactive timeout of %s milliseconds. The monitor will be stopped.", + "MonitorService.monitorTypeNotRegistered": + "The given monitor class '%s' is not registered. Please register the monitor class before running monitors of that class with the monitor service.", + "MonitorService.recreatingMonitor": "Recreating monitor: '%s'.", + "MonitorService.removedErrorMonitor": "Removed monitor in error state: '%s'.", + "MonitorService.removedExpiredMonitor": "Removed expired monitor: '%s'.", + "MonitorService.stopAndRemoveMissingMonitorType": + "The monitor service received a request to stop a monitor with type '%s' and key '%s', but the monitor service does not have any monitors registered under the given type. Please ensure monitors are registered under the correct type.", + "MonitorService.stopAndRemoveMonitorsMissingType": + "The monitor service received a request to stop all monitors with type '%s', but the monitor service does not have any monitors registered under the given type. Please ensure monitors are registered under the correct type.", + "MonitorService.cleanupTaskInterrupted": "Monitor service cleanup task interrupted.", "PluginService.hostListEmpty": "Current host list is empty.", "PluginService.releaseResources": "Releasing resources.", - "PluginService.hostsChangeListEmpty": "There are no changes in the hosts' availability.", "PluginService.failedToRetrieveHostPort": "Could not retrieve Host:Port for connection.", "PluginService.nonEmptyAliases": "fillAliases called when HostInfo already contains the following aliases: '%s'.", "PluginService.forceMonitoringRefreshTimeout": "A timeout error occurred after waiting '%s' ms for refreshed topology.", "PluginService.requiredBlockingHostListProvider": "The detected host list provider is not a BlockingHostListProvider. A BlockingHostListProvider is required to force refresh the host list. Detected host list provider: '%s'.", + "PluginService.requiredDynamicHostListProvider": + "The forceMonitoringRefresh method requires a DynamicHostListProvider. The current host list provider '%s' does not support this operation.", "PluginService.currentHostNotAllowed": "The current host is not in the list of allowed hosts. Current host: '%s'. Allowed hosts: '%s'.", "PluginService.currentHostNotDefined": "The current host is undefined.", - "MonitoringHostListProvider.requiresMonitor": - "The MonitoringRdsHostListProvider could not retrieve or initialize a ClusterTopologyMonitor for refreshing the topology.", - "MonitoringHostListProvider.errorForceRefresh": "The MonitoringRdsHostListProvider could not refresh the topology, caught error: '%s'", + "PartialPluginService.unexpectedMethodCall": "Unexpected method call: '%s'. This method is not supported by PartialPluginService.", "HostMonitoringConnectionPlugin.activatedMonitoring": "Executing method '%s', monitoring is activated.", "HostMonitoringConnectionPlugin.unableToIdentifyConnection": "Unable to identify the given connection: '%s', please ensure the correct host list provider is specified. The host list provider in use is: '%s'.", @@ -295,14 +310,20 @@ const MESSAGES: Record = { "An error occurred while attempting to obtain the writer id because the query was invalid. Please ensure you are connecting to an Aurora or RDS DB cluster. Error: '%s'", "ClusterTopologyMonitor.unableToConnect": "Could not connect to initial host: '%s'.", "ClusterTopologyMonitor.openedMonitoringConnection": "Opened monitoring connection to: '%s'.", - "ClusterTopologyMonitor.startMonitoring": "Start cluster monitoring task.", + "ClusterTopologyMonitor.startMonitoring": "[clusterId: '%s'] Start cluster topology monitoring for '%s'.", + "ClusterTopologyMonitor.startingHostMonitoringTasks": "Starting host monitoring tasks.", + "ClusterTopologyMonitor.stopHostMonitoringTask": "Stop cluster topology monitoring task for '%s'.", "ClusterTopologyMonitor.errorDuringMonitoring": "Error thrown during cluster topology monitoring: '%s'.", "ClusterTopologyMonitor.endMonitoring": "Stop cluster topology monitoring.", + "ClusterTopologyMonitor.matchingReaderTopologies": "Reader topologies have been consistent for '%s' ms. Updating topology cache.", + "ClusterTopologyMonitor.reset": "[clusterId: '%s'] Resetting cluster topology monitor for '%s'.", + "ClusterTopologyMonitor.resetEventReceived": "MonitorResetEvent received.", "HostMonitor.startMonitoring": "Host monitor '%s' started.", - "HostMonitor.detectedWriter": "Detected writer: '%s' - '%s'.", - "HostMonitor.endMonitoring": "Host monitor '%s' completed in '%s'.", + "HostMonitor.detectedWriter": "Detected writer: '%s'.", + "HostMonitor.endMonitoring": "Host monitor '%s' completed in '%s' ms.", "HostMonitor.writerHostChanged": "Writer host has changed from '%s' to '%s'.", "HostMonitor.writerIsStale": "Connected writer instance '%s' is stale.", + "HostMonitor.loginErrorDuringMonitoring": "Login error detected during monitoring.", "SlidingExpirationCacheWithCleanupTask.cleaningUp": "Cleanup interval of '%s' minutes has passed, cleaning up sliding expiration cache '%s'.", "SlidingExpirationCacheWithCleanupTask.cleanUpTaskInterrupted": "Sliding expiration cache '%s' cleanup task has been interrupted and is exiting.", "SlidingExpirationCacheWithCleanupTask.cleanUpTaskStopped": "Sliding expiration cache '%s' cleanup task has been stopped and is exiting.", @@ -384,7 +405,24 @@ const MESSAGES: Record = { "TopologyUtils.instanceIdRequired": "InstanceId must not be en empty string.", "TopologyUtils.errorGettingHostRole": "An error occurred while trying to get the host role.", "GlobalTopologyUtils.missingRegion": "Host '%s' is missing region information in the topology query result.", - "GlobalTopologyUtils.missingTemplateForRegion": "No cluster instance template found for region '%s' when processing host '%s'." + "GlobalTopologyUtils.missingTemplateForRegion": "No cluster instance template found for region '%s' when processing host '%s'.", + "Utils.globalClusterInstanceHostPatternsRequired": "The 'globalClusterInstanceHostPatterns' property is required for Global Aurora Databases.", + "Utils.invalidPatternFormat": + "Invalid pattern format '%s'. Expected format: 'region:host-pattern' (e.g., 'us-east-1:?.cluster-xyz.us-east-1.rds.amazonaws.com').", + "GlobalAuroraTopologyMonitor.cannotFindRegionTemplate": "Cannot find cluster instance template for region '%s'.", + "GlobalAuroraTopologyMonitor.invalidTopologyUtils": "TopologyUtils must implement GdbTopologyUtils for GlobalAuroraTopologyMonitor.", + "GlobalDbFailoverPlugin.missingHomeRegion": + "The 'failoverHomeRegion' property is required when connecting to a Global Aurora Database without a region in the URL.", + "GlobalDbFailoverPlugin.missingInitialHost": "Unable to determine the initial connection host.", + "GlobalDbFailoverPlugin.startFailover": "Starting Global DB failover procedure.", + "GlobalDbFailoverPlugin.isHomeRegion": "Is home region: %s", + "GlobalDbFailoverPlugin.currentFailoverMode": "Current Global DB failover mode: %s", + "GlobalDbFailoverPlugin.failoverElapsed": "Global DB failover elapsed time: %s ms", + "GlobalDbFailoverPlugin.unableToFindCandidateWithMatchingRole": + "Unable to find a candidate host with the expected role (%s) based on the given host selection strategy: %s", + "GlobalDbFailoverPlugin.unableToConnect": "Unable to establish a connection during Global DB failover.", + "BatchingEventPublisher.errorDeliveringImmediateEvent": "Error delivering immediate event: %s", + "WrapperProperty.invalidValue": "Invalid value '%s' for property '%s'. Allowed values: %s" }; export class Messages { diff --git a/common/lib/utils/monitoring/monitor.ts b/common/lib/utils/monitoring/monitor.ts new file mode 100644 index 00000000..81dadea4 --- /dev/null +++ b/common/lib/utils/monitoring/monitor.ts @@ -0,0 +1,128 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { FullServicesContainer } from "../full_services_container"; + +const DEFAULT_CLEANUP_INTERVAL_NANOS = BigInt(60_000_000_000); // 1 minute + +export enum MonitorState { + RUNNING, + STOPPED, + ERROR +} + +export enum MonitorErrorResponse { + NO_ACTION, + RECREATE +} + +export class MonitorSettings { + expirationTimeoutNanos: bigint; + inactiveTimeoutNanos: bigint; + errorResponses: Set; + + constructor(expirationTimeoutNanos: bigint, inactiveTimeoutNanos: bigint, errorResponses: Set) { + this.expirationTimeoutNanos = expirationTimeoutNanos; + this.inactiveTimeoutNanos = inactiveTimeoutNanos; + this.errorResponses = errorResponses; + } +} + +export interface Monitor { + start(): Promise; + + monitor(): Promise; + + stop(): Promise; + + close(): Promise; + + getLastActivityTimestampNanos(): bigint; + + getState(): MonitorState; + + canDispose(): boolean; +} + +export interface MonitorInitializer { + createMonitor(servicesContainer: FullServicesContainer): Monitor; +} + +export abstract class AbstractMonitor implements Monitor { + protected _stop = false; + protected terminationTimeoutMs: number; + protected lastActivityTimestampNanos: bigint; + protected state: MonitorState; + protected monitorPromise?: Promise; + + protected constructor(terminationTimeoutSec: number) { + this.terminationTimeoutMs = terminationTimeoutSec * 1000; + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); + this.state = MonitorState.STOPPED; + } + + async start(): Promise { + this.monitorPromise = this.run(); + } + + protected async run(): Promise { + try { + this.state = MonitorState.RUNNING; + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); + await this.monitor(); + } catch (error) { + this.state = MonitorState.ERROR; + } finally { + await this.close(); + } + } + + abstract monitor(): Promise; + + async stop(): Promise { + this._stop = true; + + if (this.monitorPromise) { + let timeoutId: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timeoutId = setTimeout(resolve, this.terminationTimeoutMs); + }); + await Promise.race([this.monitorPromise, timeout]); + if (timeoutId !== undefined) { + clearTimeout(timeoutId); + } + } + + await this.close(); + this.state = MonitorState.STOPPED; + } + + async close(): Promise { + // Do nothing + } + + getLastActivityTimestampNanos(): bigint { + return this.lastActivityTimestampNanos; + } + + getState(): MonitorState { + return this.state; + } + + canDispose(): boolean { + return true; + } +} diff --git a/common/lib/utils/monitoring/monitor_service.ts b/common/lib/utils/monitoring/monitor_service.ts new file mode 100644 index 00000000..138b429b --- /dev/null +++ b/common/lib/utils/monitoring/monitor_service.ts @@ -0,0 +1,450 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { Monitor, MonitorErrorResponse, MonitorInitializer, MonitorSettings, MonitorState } from "./monitor"; +import { Constructor } from "../../types"; +import { FullServicesContainer } from "../full_services_container"; +import { logger } from "../../../logutils"; +import { Messages } from "../messages"; +import { AwsWrapperError } from "../errors"; +import { ClusterTopologyMonitorImpl } from "../../host_list_provider/monitoring/cluster_topology_monitor"; +import { Topology } from "../../host_list_provider/topology"; +import { Event, EventPublisher, EventSubscriber } from "../events/event"; +import { DataAccessEvent } from "../events/data_access_event"; +import { MonitorStopEvent } from "../events/monitor_stop_event"; +import { convertNanosToMs, getTimeInNanos, sleepWithAbort } from "../utils"; +import { CacheItem } from "../cache_map"; + +const DEFAULT_CLEANUP_INTERVAL_NS = BigInt(60_000_000_000); // 1 minute +const FIFTEEN_MINUTES_NS = BigInt(15 * 60 * 1_000_000_000); +const THREE_MINUTES_NS = BigInt(3 * 60 * 1_000_000_000); + +export interface MonitorService { + registerMonitorTypeIfAbsent( + monitorClass: Constructor, + expirationTimeoutNanos: bigint, + inactiveTimeoutNanos: bigint, + errorResponses: Set, + producedDataClass?: Constructor + ): void; + + runIfAbsent( + monitorClass: Constructor, + key: unknown, + servicesContainer: FullServicesContainer, + originalProps: Map, + initializer: MonitorInitializer + ): Promise; + + get(monitorClass: Constructor, key: unknown): T | null; + + remove(monitorClass: Constructor, key: unknown): T | null; + + stopAndRemove(monitorClass: Constructor, key: unknown): Promise; + + stopAndRemoveMonitors(monitorClass: Constructor): Promise; + + stopAndRemoveAll(): Promise; + + releaseResources(): Promise; +} + +/** + * A container object that holds a monitor together with the supplier used to generate the monitor. + * The supplier can be used to recreate the monitor if it encounters an error or becomes stuck. + */ +class MonitorItem { + private readonly monitorSupplier: () => Monitor; + private readonly _monitor: Monitor; + + constructor(monitorSupplier: () => Monitor) { + this.monitorSupplier = monitorSupplier; + this._monitor = monitorSupplier(); + } + + getMonitorSupplier(): () => Monitor { + return this.monitorSupplier; + } + + getMonitor(): Monitor { + return this._monitor; + } +} + +/** + * A container that holds a cache of monitors of a given type with the related settings and info for that type. + */ +class CacheContainer { + private readonly settings: MonitorSettings; + private readonly cache: Map>; + private readonly producedDataClass: Constructor | null; + + constructor(settings: MonitorSettings, producedDataClass: Constructor | null) { + this.settings = settings; + this.producedDataClass = producedDataClass; + this.cache = new Map>(); + } + + getSettings(): MonitorSettings { + return this.settings; + } + + getCache(): Map> { + return this.cache; + } + + getProducedDataClass(): Constructor | null { + return this.producedDataClass; + } +} + +export class MonitorServiceImpl implements MonitorService, EventSubscriber { + private static defaultSuppliers: Map, () => CacheContainer> | null = null; + + // Lazy initialization for the default suppliers to avoid circular dependencies. + private static getDefaultSuppliers(): Map, () => CacheContainer> { + if (!MonitorServiceImpl.defaultSuppliers) { + const recreateOnError = new Set([MonitorErrorResponse.RECREATE]); + const defaultSettings = new MonitorSettings(FIFTEEN_MINUTES_NS, THREE_MINUTES_NS, recreateOnError); + + MonitorServiceImpl.defaultSuppliers = new Map([[ClusterTopologyMonitorImpl, () => new CacheContainer(defaultSettings, Topology)]]); + } + return MonitorServiceImpl.defaultSuppliers; + } + + protected readonly publisher: EventPublisher; + protected readonly monitorCaches = new Map, CacheContainer>(); + // Use a pending promise map to prevent race conditions when creating monitors. + private readonly pendingMonitors = new Map>(); + private cleanupTask: Promise | null = null; + private interruptCleanupTask: (() => void) | null = null; + private isInitialized: boolean = false; + + constructor(publisher: EventPublisher, cleanupIntervalNs: bigint = DEFAULT_CLEANUP_INTERVAL_NS) { + this.publisher = publisher; + this.publisher.subscribe(this, new Set([DataAccessEvent, MonitorStopEvent])); + this.initCleanupTask(cleanupIntervalNs); + } + + protected initCleanupTask(cleanupIntervalNs: bigint): void { + this.isInitialized = true; + this.cleanupTask = this.runCleanupLoop(cleanupIntervalNs); + } + + private async runCleanupLoop(cleanupIntervalNs: bigint): Promise { + while (this.isInitialized) { + const [sleepPromise, abortSleepFunc] = sleepWithAbort( + convertNanosToMs(cleanupIntervalNs), + Messages.get("MonitorService.cleanupTaskInterrupted") + ); + this.interruptCleanupTask = abortSleepFunc; + try { + await sleepPromise; + } catch { + // Sleep has been interrupted, exit cleanup task. + return; + } + + await this.checkMonitors(); + } + } + + protected async checkMonitors(): Promise { + for (const container of this.monitorCaches.values()) { + const cache = container.getCache(); + const keysToProcess = Array.from(cache.keys()); + + for (const key of keysToProcess) { + const cacheItem = cache.get(key); + if (!cacheItem) { + continue; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + continue; + } + + const monitor = monitorItem.getMonitor(); + const monitorSettings = container.getSettings(); + + // Check for stopped monitors + if (monitor.getState() === MonitorState.STOPPED) { + cache.delete(key); + await monitor.stop(); + continue; + } + + // Check for error state monitors + if (monitor.getState() === MonitorState.ERROR) { + cache.delete(key); + logger.debug(Messages.get("MonitorService.removedErrorMonitor", JSON.stringify(monitor))); + await this.handleMonitorError(container, key, monitorItem); + continue; + } + + // Check for inactive/stuck monitors + const inactiveTimeoutNs = monitorSettings.inactiveTimeoutNanos; + if (getTimeInNanos() - monitor.getLastActivityTimestampNanos() > inactiveTimeoutNs) { + cache.delete(key); + logger.info(Messages.get("MonitorService.monitorStuck", JSON.stringify(monitor), convertNanosToMs(inactiveTimeoutNs).toString())); + await this.handleMonitorError(container, key, monitorItem); + continue; + } + + // Check for expired monitors that can be disposed + if (cacheItem.isExpired() && monitor.canDispose()) { + cache.delete(key); + logger.info(Messages.get("MonitorService.removedExpiredMonitor", JSON.stringify(monitor))); + await monitor.stop(); + } + } + } + } + + protected async handleMonitorError(cacheContainer: CacheContainer, key: unknown, errorMonitorItem: MonitorItem): Promise { + const monitor = errorMonitorItem.getMonitor(); + await monitor.stop(); + + const errorResponses = cacheContainer.getSettings().errorResponses; + if (errorResponses && errorResponses.has(MonitorErrorResponse.RECREATE)) { + if (!cacheContainer.getCache().has(key)) { + logger.info(Messages.get("MonitorService.recreatingMonitor", JSON.stringify(monitor))); + const newMonitorItem = new MonitorItem(errorMonitorItem.getMonitorSupplier()); + const expirationNs = cacheContainer.getSettings().expirationTimeoutNanos; + cacheContainer.getCache().set(key, new CacheItem(newMonitorItem, getTimeInNanos() + expirationNs)); + await newMonitorItem.getMonitor().start(); + } + } + } + + registerMonitorTypeIfAbsent( + monitorClass: Constructor, + expirationTimeoutNanos: bigint, + inactiveTimeoutNanos: bigint, + errorResponses: Set, + producedDataClass?: Constructor + ): void { + if (this.monitorCaches.has(monitorClass)) { + return; + } + + const settings = new MonitorSettings(expirationTimeoutNanos, inactiveTimeoutNanos, errorResponses); + const cacheContainer = new CacheContainer(settings, producedDataClass ?? null); + this.monitorCaches.set(monitorClass, cacheContainer); + } + + async runIfAbsent( + monitorClass: Constructor, + key: unknown, + servicesContainer: FullServicesContainer, + _originalProps: Map, + initializer: MonitorInitializer + ): Promise { + let cacheContainer = this.monitorCaches.get(monitorClass); + + if (!cacheContainer) { + const supplier = MonitorServiceImpl.getDefaultSuppliers().get(monitorClass as Constructor); + if (!supplier) { + throw new AwsWrapperError(Messages.get("MonitorService.monitorTypeNotRegistered", monitorClass.name)); + } + + cacheContainer = supplier(); + this.monitorCaches.set(monitorClass, cacheContainer); + } + + const cache = cacheContainer.getCache(); + const existingCacheItem = cache.get(key); + if (existingCacheItem) { + const existingMonitorItem = existingCacheItem.get(true); + if (existingMonitorItem) { + existingCacheItem.updateExpiration(cacheContainer.getSettings().expirationTimeoutNanos); + return existingMonitorItem.getMonitor() as T; + } + } + + const pendingKey = `${monitorClass.name}:${JSON.stringify(key)}`; + + // Check if the monitor is already being created by another async task. + const pendingPromise = this.pendingMonitors.get(pendingKey); + if (pendingPromise) { + return (await pendingPromise) as T; + } + + // Use the pending promise pattern to create monitors. This prevents race condition. + const createPromise = (async (): Promise => { + try { + const recheckCacheItem = cache.get(key); + if (recheckCacheItem) { + const recheckMonitorItem = recheckCacheItem.get(true); + if (recheckMonitorItem) { + recheckCacheItem.updateExpiration(cacheContainer.getSettings().expirationTimeoutNanos); + return recheckMonitorItem.getMonitor(); + } + } + + const monitorItem = new MonitorItem(() => initializer.createMonitor(servicesContainer)); + const expirationNs = cacheContainer.getSettings().expirationTimeoutNanos; + cache.set(key, new CacheItem(monitorItem, getTimeInNanos() + expirationNs)); + await monitorItem.getMonitor().start(); + + return monitorItem.getMonitor(); + } finally { + // Delete the key once monitor has been successfully created. + this.pendingMonitors.delete(pendingKey); + } + })(); + + this.pendingMonitors.set(pendingKey, createPromise); + return (await createPromise) as T; + } + + get(monitorClass: Constructor, key: unknown): T | null { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + return null; + } + + const cacheItem = cacheContainer.getCache().get(key); + if (!cacheItem) { + return null; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + return null; + } + + const monitor = monitorItem.getMonitor(); + if (monitor instanceof monitorClass) { + return monitor as T; + } + + logger.info(Messages.get("MonitorService.monitorClassMismatch", JSON.stringify(key), monitorClass.name, JSON.stringify(monitor))); + return null; + } + + remove(monitorClass: Constructor, key: unknown): T | null { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + return null; + } + + const cache = cacheContainer.getCache(); + const cacheItem = cache.get(key); + if (!cacheItem) { + return null; + } + + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + return null; + } + + const monitor = monitorItem.getMonitor(); + if (monitor instanceof monitorClass) { + cache.delete(key); + return monitor as T; + } + + return null; + } + + async stopAndRemove(monitorClass: Constructor, key: unknown): Promise { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + logger.info(Messages.get("MonitorService.stopAndRemoveMissingMonitorType", monitorClass.name, String(key))); + return; + } + + const cache = cacheContainer.getCache(); + const cacheItem = cache.get(key); + if (cacheItem) { + cache.delete(key); + await cacheItem.get(true)?.getMonitor().stop(); + } + } + + async stopAndRemoveMonitors(monitorClass: Constructor): Promise { + const cacheContainer = this.monitorCaches.get(monitorClass); + if (!cacheContainer) { + logger.info(Messages.get("MonitorService.stopAndRemoveMonitorsMissingType", monitorClass.name)); + return; + } + + const cache = cacheContainer.getCache(); + for (const [key, cacheItem] of cache.entries()) { + cache.delete(key); + await cacheItem.get(true)?.getMonitor().stop(); + } + } + + async stopAndRemoveAll(): Promise { + for (const monitorClass of this.monitorCaches.keys()) { + await this.stopAndRemoveMonitors(monitorClass); + } + } + + async releaseResources(): Promise { + // Stop cleanup task + this.isInitialized = false; + this.interruptCleanupTask?.(); + if (this.cleanupTask) { + await this.cleanupTask; + } + + await this.stopAndRemoveAll(); + } + + async processEvent(event: Event): Promise { + if (event instanceof DataAccessEvent) { + for (const container of this.monitorCaches.values()) { + if (!container.getProducedDataClass() || event.dataClass !== container.getProducedDataClass()) { + continue; + } + + // The data produced by the monitor in this cache with this key has been accessed recently, + // so we extend the monitor's expiration. + container.getCache().get(event.key)?.updateExpiration(container.getSettings().expirationTimeoutNanos); + } + return; + } + + if (event instanceof MonitorStopEvent) { + await this.stopAndRemove(event.monitorClass, event.key); + return; + } + + // Other event types should be propagated to monitors + for (const container of this.monitorCaches.values()) { + for (const cacheItem of container.getCache().values()) { + const monitorItem = cacheItem.get(true); + if (!monitorItem) { + continue; + } + + const monitor = monitorItem.getMonitor(); + if (this.isEventSubscriber(monitor)) { + await (monitor as unknown as EventSubscriber).processEvent(event); + } + } + } + } + + private isEventSubscriber(obj: unknown): obj is EventSubscriber { + return typeof obj === "object" && obj !== null && "processEvent" in obj && typeof (obj as EventSubscriber).processEvent === "function"; + } +} diff --git a/common/lib/utils/rds_url_type.ts b/common/lib/utils/rds_url_type.ts index 46300354..64955089 100644 --- a/common/lib/utils/rds_url_type.ts +++ b/common/lib/utils/rds_url_type.ts @@ -20,6 +20,7 @@ export class RdsUrlType { public static readonly RDS_READER_CLUSTER = new RdsUrlType(true, true, true); public static readonly RDS_CUSTOM_CLUSTER = new RdsUrlType(true, false, true); public static readonly RDS_PROXY = new RdsUrlType(true, false, true); + public static readonly RDS_PROXY_ENDPOINT = new RdsUrlType(true, false, true); public static readonly RDS_INSTANCE = new RdsUrlType(true, false, true); public static readonly RDS_AURORA_LIMITLESS_DB_SHARD_GROUP = new RdsUrlType(true, false, true); public static readonly RDS_GLOBAL_WRITER_CLUSTER = new RdsUrlType(true, true, false); diff --git a/common/lib/utils/rds_utils.ts b/common/lib/utils/rds_utils.ts index 66543349..95c894d7 100644 --- a/common/lib/utils/rds_utils.ts +++ b/common/lib/utils/rds_utils.ts @@ -22,12 +22,13 @@ export class RdsUtils { // can be found at // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.Overview.Endpoints.html // - // Details how to use RDS Proxy endpoints can be found at + // Details how to use RDS Proxy endpoints can be found at // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/rds-proxy-endpoints.html // - // Values like "<...>" depend on particular Aurora cluster. + // Values like "<...>" depend on particular Aurora cluster. // For example: "" // + // // Cluster (Writer) Endpoint: .cluster-..rds.amazonaws.com // Example: test-postgres.cluster-123456789012.us-east-2.rds.amazonaws.com // @@ -41,7 +42,10 @@ export class RdsUtils { // Example: test-postgres-instance-1.123456789012.us-east-2.rds.amazonaws.com // // + // // Similar endpoints for China regions have different structure and are presented below. + // https://docs.amazonaws.cn/en_us/aws/latest/userguide/endpoints-Ningxia.html + // https://docs.amazonaws.cn/en_us/aws/latest/userguide/endpoints-Beijing.html // // Cluster (Writer) Endpoint: .cluster-.rds..amazonaws.com.cn // Example: test-postgres.cluster-123456789012.rds.cn-northwest-1.amazonaws.com.cn @@ -59,52 +63,51 @@ export class RdsUtils { // Governmental endpoints // https://aws.amazon.com/compliance/fips/#FIPS_Endpoints_by_Service // https://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/Region.html - + // + // + // Aurora Global Database // https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Concepts.Aurora_Fea_Regions_DB-eng.Feature.GlobalDatabase.html + // Global Database Endpoint: .global-.global.rds.amazonaws.com + // Example: test-global-db-name.global-123456789012.global.rds.amazonaws.com + // + // + // RDS Proxy + // RDS Proxy Endpoint: .proxy-..rds.amazonaws.com + // Example: test-rds-proxy-name.proxy-123456789012.us-east-2.rds.amazonaws.com + // + // RDS Proxy Custom Endpoint: .endpoint.proxy-..rds.amazonaws.com + // Example: test-custom-endpoint-name.endpoint.proxy-123456789012.us-east-2.rds.amazonaws.com + private static readonly AURORA_GLOBAL_WRITER_DNS_PATTERN = /^(?.+)\.(?global-)?(?[a-zA-Z0-9]+\.global\.rds\.amazonaws\.com\.?)$/i; private static readonly AURORA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; - private static readonly AURORA_INSTANCE_PATTERN = /^(?.+)\.(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.(com|au|eu|uk)\.?)$/i; private static readonly AURORA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; - private static readonly AURORA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.(com|au|eu|uk)\.?)$/i; private static readonly AURORA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.(amazonaws\.com(\.cn)?|sc2s\.sgov\.gov|c2s\.ic\.gov))$/i; - private static readonly AURORA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com)$/i; + /^(?.+)\.(?shardgrp-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.(amazonaws\.com\.?|amazonaws\.eu\.?|amazonaws\.au\.?|amazonaws\.uk\.?|amazonaws\.com\.cn\.?|sc2s\.sgov\.gov\.?|c2s\.ic\.gov\.?))$/i; private static readonly AURORA_CHINA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_OLD_CHINA_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_INSTANCE_PATTERN = - /^(?.+)\.(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_INSTANCE_PATTERN = - /^(?.+)\.(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_CHINA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_OLD_CHINA_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_LIMITLESS_CLUSTER_PATTERN = - /^(?.+)\.(?shardgrp-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_CUSTOM_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-custom-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_CHINA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-])+\.amazonaws\.com\.cn)$/i; - private static readonly AURORA_OLD_CHINA_PROXY_DNS_PATTERN = - /^(?.+)\.(?proxy-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-])+\.rds\.amazonaws\.com\.cn)$/i; - + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.(rds|rds-fips)\.amazonaws\.com\.cn\.?)$/i; private static readonly AURORA_GOV_DNS_PATTERN = - /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i; + /^(?.+)\.(?proxy-|cluster-|cluster-ro-|cluster-custom-|shardgrp-)?(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com\.?|c2s\.ic\.gov\.?|sc2s\.sgov\.gov\.?))$/i; private static readonly AURORA_GOV_CLUSTER_PATTERN = - /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com|c2s\.ic\.gov|sc2s\.sgov\.gov))$/i; + /^(?.+)\.(?cluster-|cluster-ro-)+(?[a-zA-Z0-9]+\.(rds|rds-fips)\.(?[a-zA-Z0-9-]+)\.(amazonaws\.com\.?|c2s\.ic\.gov\.?|sc2s\.sgov\.gov\.?))$/i; + + // RDS Proxy Custom Endpoint: .endpoint.proxy-..rds.amazonaws.com + private static readonly RDS_PROXY_ENDPOINT_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.?)$/i; + private static readonly RDS_PROXY_ENDPOINT_CHINA_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)+(?[a-zA-Z0-9]+\.rds\.(?[a-zA-Z0-9-]+)\.amazonaws\.com\.cn\.?)$/i; + private static readonly RDS_PROXY_ENDPOINT_OLD_CHINA_DNS_PATTERN = + /^(?.+)\.endpoint\.(?proxy-)?(?[a-zA-Z0-9]+\.(?[a-zA-Z0-9-]+)\.rds\.amazonaws\.com\.cn\.?)$/i; private static readonly ELB_PATTERN = /^(?.+)\.elb\.((?[a-zA-Z0-9-]+)\.amazonaws\.com)$/i; private static readonly IP_V4 = @@ -121,20 +124,24 @@ export class RdsUtils { private static readonly cachedPatterns = new Map(); private static readonly cachedDnsPatterns = new Map(); + private static prepareHostFunc?: (host: string) => string; public isRdsClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-") || equalsIgnoreCase(dnsGroup, "cluster-ro-"); } public isRdsCustomClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-custom-"); } public isRdsDns(host: string): boolean { + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -143,24 +150,46 @@ export class RdsUtils { const group = this.getRegexGroup(matcher, RdsUtils.DNS_GROUP); if (group) { - RdsUtils.cachedDnsPatterns.set(host, group); + RdsUtils.cachedDnsPatterns.set(preparedHost, group); } return matcher != null; } public isRdsInstance(host: string): boolean { - return !this.getDnsGroup(host) && this.isRdsDns(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !this.getDnsGroup(preparedHost) && this.isRdsDns(preparedHost); } isRdsProxyDns(host: string) { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return dnsGroup && dnsGroup.startsWith("proxy-"); } + isRdsProxyEndpointDns(host: string): boolean { + if (!host) { + return false; + } + + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = this.cacheMatcher( + preparedHost, + RdsUtils.RDS_PROXY_ENDPOINT_DNS_PATTERN, + RdsUtils.RDS_PROXY_ENDPOINT_CHINA_DNS_PATTERN, + RdsUtils.RDS_PROXY_ENDPOINT_OLD_CHINA_DNS_PATTERN + ); + if (this.getRegexGroup(matcher, RdsUtils.DNS_GROUP) !== null) { + return this.getRegexGroup(matcher, RdsUtils.INSTANCE_GROUP) !== null; + } + + return false; + } + getRdsClusterId(host: string): string | null { + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -179,8 +208,9 @@ export class RdsUtils { return null; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -198,8 +228,9 @@ export class RdsUtils { return "?"; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -214,8 +245,9 @@ export class RdsUtils { return null; } + const preparedHost = RdsUtils.getPreparedHost(host); const matcher = this.cacheMatcher( - host, + preparedHost, RdsUtils.AURORA_DNS_PATTERN, RdsUtils.AURORA_CHINA_DNS_PATTERN, RdsUtils.AURORA_OLD_CHINA_DNS_PATTERN, @@ -227,7 +259,7 @@ export class RdsUtils { return group; } - const elbMatcher = host.match(RdsUtils.ELB_PATTERN); + const elbMatcher = preparedHost.match(RdsUtils.ELB_PATTERN); if (elbMatcher && elbMatcher.length > 0) { return this.getRegexGroup(elbMatcher, RdsUtils.REGION_GROUP); } @@ -235,23 +267,36 @@ export class RdsUtils { return null; } + public isSameRegion(host1: string | null, host2: string | null): boolean { + if (!host1 || !host2) { + return false; + } + const host1Region = this.getRdsRegion(host1); + const host2Region = this.getRdsRegion(host2); + return host1Region !== null && equalsIgnoreCase(host1Region, host2Region); + } + public isGlobalDbWriterClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "global-"); } public isWriterClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-"); } public isReaderClusterDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); return equalsIgnoreCase(dnsGroup, "cluster-ro-"); } public isLimitlessDbShardGroupDns(host: string): boolean { - const dnsGroup = this.getDnsGroup(host); + const preparedHost = RdsUtils.getPreparedHost(host); + const dnsGroup = this.getDnsGroup(preparedHost); if (!dnsGroup) { return false; } @@ -263,25 +308,26 @@ export class RdsUtils { return null; } - const matcher = host.match(RdsUtils.AURORA_CLUSTER_PATTERN); + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = preparedHost.match(RdsUtils.AURORA_CLUSTER_PATTERN); if (matcher) { - return host.replace(RdsUtils.AURORA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_CLUSTER_PATTERN, "$.cluster-$"); } - const limitlessMatcher = host.match(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN); + const limitlessMatcher = preparedHost.match(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN); if (limitlessMatcher) { - return host.replace(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_LIMITLESS_CLUSTER_PATTERN, "$.cluster-$"); } - const chinaMatcher = host.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN); + const chinaMatcher = preparedHost.match(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN); if (chinaMatcher) { - return host.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_CHINA_CLUSTER_PATTERN, "$.cluster-$"); } - const oldChinaMatcher = host.match(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN); + const oldChinaMatcher = preparedHost.match(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN); if (oldChinaMatcher) { - return host.replace(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_OLD_CHINA_CLUSTER_PATTERN, "$.cluster-$"); } - const govMatcher = host.match(RdsUtils.AURORA_GOV_CLUSTER_PATTERN); + const govMatcher = preparedHost.match(RdsUtils.AURORA_GOV_CLUSTER_PATTERN); if (govMatcher) { - return host.replace(RdsUtils.AURORA_GOV_CLUSTER_PATTERN, "$.cluster-$"); + return preparedHost.replace(RdsUtils.AURORA_GOV_CLUSTER_PATTERN, "$.cluster-$"); } return null; } @@ -307,21 +353,24 @@ export class RdsUtils { return RdsUrlType.OTHER; } - if (this.isIPv4(host) || this.isIPv6(host)) { + const preparedHost = RdsUtils.getPreparedHost(host); + if (this.isIPv4(preparedHost) || this.isIPv6(preparedHost)) { return RdsUrlType.IP_ADDRESS; - } else if (this.isGlobalDbWriterClusterDns(host)) { + } else if (this.isGlobalDbWriterClusterDns(preparedHost)) { return RdsUrlType.RDS_GLOBAL_WRITER_CLUSTER; - } else if (this.isWriterClusterDns(host)) { + } else if (this.isWriterClusterDns(preparedHost)) { return RdsUrlType.RDS_WRITER_CLUSTER; - } else if (this.isReaderClusterDns(host)) { + } else if (this.isReaderClusterDns(preparedHost)) { return RdsUrlType.RDS_READER_CLUSTER; - } else if (this.isRdsCustomClusterDns(host)) { + } else if (this.isRdsCustomClusterDns(preparedHost)) { return RdsUrlType.RDS_CUSTOM_CLUSTER; - } else if (this.isLimitlessDbShardGroupDns(host)) { + } else if (this.isLimitlessDbShardGroupDns(preparedHost)) { return RdsUrlType.RDS_AURORA_LIMITLESS_DB_SHARD_GROUP; - } else if (this.isRdsProxyDns(host)) { + } else if (this.isRdsProxyDns(preparedHost)) { return RdsUrlType.RDS_PROXY; - } else if (this.isRdsDns(host)) { + } else if (this.isRdsProxyEndpointDns(preparedHost)) { + return RdsUrlType.RDS_PROXY_ENDPOINT; + } else if (this.isRdsDns(preparedHost)) { return RdsUrlType.RDS_INSTANCE; } else { // ELB URLs will also be classified as other @@ -330,23 +379,27 @@ export class RdsUtils { } public isGreenInstance(host: string) { - return host && RdsUtils.BG_GREEN_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return preparedHost && RdsUtils.BG_GREEN_HOST_PATTERN.test(preparedHost); } public isOldInstance(host: string): boolean { - return !!host && RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !!preparedHost && RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } public isNotOldInstance(host: string): boolean { if (!host) { return true; } - return !RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } // Verify that provided host is a blue host name and contains neither green prefix nor old prefix. public isNotGreenAndOldPrefixInstance(host: string): boolean { - return !!host && !RdsUtils.BG_GREEN_HOST_PATTERN.test(host) && !RdsUtils.BG_OLD_HOST_PATTERN.test(host); + const preparedHost = RdsUtils.getPreparedHost(host); + return !!preparedHost && !RdsUtils.BG_GREEN_HOST_PATTERN.test(preparedHost) && !RdsUtils.BG_OLD_HOST_PATTERN.test(preparedHost); } public removeGreenInstancePrefix(host: string): string { @@ -354,7 +407,8 @@ export class RdsUtils { return host; } - const matcher = host.match(RdsUtils.BG_GREEN_HOST_PATTERN); + const preparedHost = RdsUtils.getPreparedHost(host); + const matcher = preparedHost.match(RdsUtils.BG_GREEN_HOST_PATTERN); if (!matcher || matcher.length === 0) { return host; } @@ -427,4 +481,20 @@ export class RdsUtils { RdsUtils.cachedPatterns.clear(); RdsUtils.cachedDnsPatterns.clear(); } + + static setPrepareHostFunc(func?: (host: string) => string) { + RdsUtils.prepareHostFunc = func; + } + + static resetPrepareHostFunc() { + RdsUtils.prepareHostFunc = undefined; + } + + private static getPreparedHost(host: string): string { + const func = RdsUtils.prepareHostFunc; + if (!func) { + return host; + } + return func(host) ?? host; + } } diff --git a/common/lib/utils/service_utils.ts b/common/lib/utils/service_utils.ts new file mode 100644 index 00000000..b7bda542 --- /dev/null +++ b/common/lib/utils/service_utils.ts @@ -0,0 +1,124 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { FullServicesContainer, FullServicesContainerImpl } from "./full_services_container"; +import { StorageService } from "./storage/storage_service"; +import { PluginServiceImpl } from "../plugin_service"; +import { PluginManager } from "../plugin_manager"; +import { ConnectionProviderManager } from "../connection_provider_manager"; +import { DriverConnectionProvider } from "../driver_connection_provider"; +import { WrapperProperties } from "../wrapper_property"; +import { ConnectionProvider } from "../connection_provider"; +import { AwsClient } from "../aws_client"; +import { DatabaseDialect, DatabaseType } from "../database_dialect/database_dialect"; +import { DatabaseDialectCodes } from "../database_dialect/database_dialect_codes"; +import { DriverDialect } from "../driver_dialect/driver_dialect"; +import { MonitorService } from "./monitoring/monitor_service"; +import { TelemetryFactory } from "./telemetry/telemetry_factory"; +import { EventPublisher } from "./events/event"; +import { PartialPluginService } from "../partial_plugin_service"; +import { ConnectionUrlParser } from "./connection_url_parser"; + +export class ServiceUtils { + private static readonly _instance: ServiceUtils = new ServiceUtils(); + + static get instance(): ServiceUtils { + return this._instance; + } + + createStandardServiceContainer( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + client: AwsClient, + props: Map, + dbType: DatabaseType, + knownDialectsByCode: Map, + driverDialect: DriverDialect, + telemetryFactory: TelemetryFactory, + connectionProvider: ConnectionProvider | null + ): FullServicesContainer { + const servicesContainer: FullServicesContainer = new FullServicesContainerImpl( + storageService, + monitorService, + eventPublisher, + connectionProvider, + telemetryFactory + ); + + const pluginService = new PluginServiceImpl(servicesContainer, client, dbType, knownDialectsByCode, props, driverDialect); + const pluginManager = new PluginManager( + servicesContainer, + props, + new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(props)), + telemetryFactory + ); + + servicesContainer.pluginService = pluginService; + servicesContainer.pluginManager = pluginManager; + servicesContainer.hostListProviderService = pluginService; + + return servicesContainer; + } + + createMinimalServiceContainer( + storageService: StorageService, + monitorService: MonitorService, + eventPublisher: EventPublisher, + props: Map, + dialect: DatabaseDialect, + driverDialect: DriverDialect, + telemetryFactory: TelemetryFactory, + connectionProvider: ConnectionProvider | null, + connectionUrlParser: ConnectionUrlParser + ): FullServicesContainer { + const servicesContainer: FullServicesContainer = new FullServicesContainerImpl( + storageService, + monitorService, + eventPublisher, + connectionProvider, + telemetryFactory + ); + + const pluginService = new PartialPluginService(servicesContainer, props, dialect, driverDialect, connectionUrlParser); + const pluginManager = new PluginManager( + servicesContainer, + props, + new ConnectionProviderManager(connectionProvider ?? new DriverConnectionProvider(), WrapperProperties.CONNECTION_PROVIDER.get(props)), + telemetryFactory + ); + + servicesContainer.pluginService = pluginService; + servicesContainer.pluginManager = pluginManager; + servicesContainer.hostListProviderService = pluginService; + + return servicesContainer; + } + + createMinimalServiceContainerFrom(servicesContainer: FullServicesContainer, props: Map): FullServicesContainer { + return this.createMinimalServiceContainer( + servicesContainer.storageService, + servicesContainer.monitorService, + servicesContainer.eventPublisher, + props, + servicesContainer.pluginService.getDialect(), + servicesContainer.pluginService.getDriverDialect(), + servicesContainer.telemetryFactory, + servicesContainer.defaultConnectionProvider, + servicesContainer.pluginService.getConnectionUrlParser() + ); + } +} diff --git a/common/lib/utils/status_cache_item.ts b/common/lib/utils/status_cache_item.ts new file mode 100644 index 00000000..6e748965 --- /dev/null +++ b/common/lib/utils/status_cache_item.ts @@ -0,0 +1,23 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +export class StatusCacheItem { + readonly status: T; + + constructor(status: T) { + this.status = status; + } +} diff --git a/common/lib/utils/storage/expiration_cache.ts b/common/lib/utils/storage/expiration_cache.ts index 5afd57e9..72e3b7d1 100644 --- a/common/lib/utils/storage/expiration_cache.ts +++ b/common/lib/utils/storage/expiration_cache.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { getTimeInNanos } from "../utils"; import { ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; diff --git a/common/lib/utils/storage/storage_service.ts b/common/lib/utils/storage/storage_service.ts index 06f6ca00..dc2bd71d 100644 --- a/common/lib/utils/storage/storage_service.ts +++ b/common/lib/utils/storage/storage_service.ts @@ -1,26 +1,33 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { Constructor, ItemDisposalFunc, ShouldDisposeFunc } from "../../types"; import { ExpirationCache } from "./expiration_cache"; import { Topology } from "../../host_list_provider/topology"; import { AwsWrapperError } from "../errors"; import { Messages } from "../messages"; +import { EventPublisher } from "../events/event"; +import { DataAccessEvent } from "../events/data_access_event"; +import { AllowedAndBlockedHosts } from "../../allowed_and_blocked_hosts"; +import { BlueGreenStatus } from "../../plugins/bluegreen/blue_green_status"; +import { HostAvailabilityCacheItem } from "../../host_availability/host_availability_cache_item"; +import { StatusCacheItem } from "../status_cache_item"; const DEFAULT_CLEANUP_INTERVAL_NANOS = 5 * 60 * 1_000_000_000; // 5 minutes +const SIXTY_MINUTES_NANOS = BigInt(60 * 60 * 1_000_000_000); // 60 minutes /** * Interface for a storage service that manages items with expiration and disposal logic. @@ -62,9 +69,10 @@ export interface StorageService { * * @param itemClass The expected constructor/class of the item being retrieved * @param key The key for the item, e.g., "custom-endpoint.cluster-custom-XYZ.us-east-2.rds.amazonaws.com:5432" + * @param registerDataAccess Whether to register a data access event. Defaults to true. * @returns The item stored at the given key for the given item class, or null/undefined if not found */ - get(itemClass: Constructor, key?: unknown): V | null; + get(itemClass: Constructor, key?: unknown, registerDataAccess?: boolean): V | null; /** * Indicates whether an item exists under the given item class and key. @@ -108,31 +116,44 @@ export interface StorageService { * Cleanup method to stop the cleanup interval timer. * Should be called when the service is no longer needed. */ - releaseResources(): void; + releaseResources(): Promise; } type CacheSupplier = () => ExpirationCache; export class StorageServiceImpl implements StorageService { - private static readonly defaultCacheSuppliers: Map = new Map([[Topology, () => new ExpirationCache()]]); + private static readonly DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO = BigInt(5 * 60_000_000_000); // 5 minutes + + private static readonly defaultCacheSuppliers: Map = (() => { + const suppliers = new Map(); + suppliers.set(Topology, () => new ExpirationCache()); + suppliers.set(AllowedAndBlockedHosts, () => new ExpirationCache()); + suppliers.set(BlueGreenStatus, () => new ExpirationCache(false, SIXTY_MINUTES_NANOS, null, null)); + suppliers.set( + HostAvailabilityCacheItem, + () => new ExpirationCache(true, StorageServiceImpl.DEFAULT_HOST_AVAILABILITY_CACHE_EXPIRE_NANO, null, null) + ); + suppliers.set(StatusCacheItem, () => new ExpirationCache(false, SIXTY_MINUTES_NANOS, null, null)); + return suppliers; + })(); protected readonly caches: Map> = new Map(); protected cleanupIntervalHandle?: NodeJS.Timeout; + protected readonly publisher: EventPublisher; - constructor(cleanupIntervalNanos: number = DEFAULT_CLEANUP_INTERVAL_NANOS) { - this.initCleanupThread(cleanupIntervalNanos); + constructor(publisher: EventPublisher, cleanupIntervalNanos: number = DEFAULT_CLEANUP_INTERVAL_NANOS) { + this.publisher = publisher; + this.initCleanupTask(cleanupIntervalNanos); } - protected initCleanupThread(cleanupIntervalNanos: number): void { + protected initCleanupTask(cleanupIntervalNanos: number): void { const intervalMs = cleanupIntervalNanos / 1_000_000; this.cleanupIntervalHandle = setInterval(() => { this.removeExpiredItems(); }, intervalMs); - // Allow Node.js to exit even if this timer is active - if (this.cleanupIntervalHandle.unref) { - this.cleanupIntervalHandle.unref(); - } + // Unref the timer to prevent this background cleanup task from blocking the application from gracefully exiting. + this.cleanupIntervalHandle.unref(); } protected removeExpiredItems(): void { @@ -179,7 +200,7 @@ export class StorageServiceImpl implements StorageService { } } - get(itemClass: Constructor, key?: unknown): V | null { + get(itemClass: Constructor, key?: unknown, registerDataAccess: boolean = true): V | null { const cache = this.caches.get(itemClass); if (!cache) { return null; @@ -191,6 +212,10 @@ export class StorageServiceImpl implements StorageService { } if (value instanceof itemClass) { + if (registerDataAccess) { + const event = new DataAccessEvent(itemClass, key); + this.publisher.publish(event); + } return value as V; } @@ -223,7 +248,6 @@ export class StorageServiceImpl implements StorageService { for (const cache of this.caches.values()) { cache.clear(); } - this.caches.clear(); } @@ -235,15 +259,7 @@ export class StorageServiceImpl implements StorageService { return cache.size(); } - /** - * Registers a default cache supplier for a specific item class. - * This allows automatic cache creation when items of this class are stored. - */ - static registerDefaultCacheSupplier(itemClass: Constructor, supplier: CacheSupplier): void { - StorageServiceImpl.defaultCacheSuppliers.set(itemClass, supplier); - } - - releaseResources(): void { + async releaseResources(): Promise { if (this.cleanupIntervalHandle) { clearInterval(this.cleanupIntervalHandle); this.cleanupIntervalHandle = undefined; diff --git a/common/lib/utils/utils.ts b/common/lib/utils/utils.ts index 1e159510..4091b6a6 100644 --- a/common/lib/utils/utils.ts +++ b/common/lib/utils/utils.ts @@ -20,16 +20,26 @@ import { WrapperProperties } from "../wrapper_property"; import { HostRole } from "../host_role"; import { logger } from "../../logutils"; import { AwsWrapperError, InternalQueryTimeoutError } from "./errors"; -import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; export function sleep(ms: number) { return new Promise((resolve) => setTimeout(resolve, ms)); } +/** + * Creates a sleep promise that can be aborted before completion. + * + * @param ms - Duration to sleep in milliseconds + * @param message - Error message when aborted + * @returns A tuple of [sleepPromise, abortFunction] + * - sleepPromise: Resolves after ms milliseconds, or rejects if aborted + * - abortFunction: Call to cancel the sleep and reject the promise + */ export function sleepWithAbort(ms: number, message?: string) { let abortSleep; const promise = new Promise((resolve, reject) => { const timeout = setTimeout(resolve, ms); + // Unref the timer to prevent this background task from blocking the application from gracefully exiting. + timeout.unref(); abortSleep = () => { clearTimeout(timeout); reject(new AwsWrapperError(message)); @@ -65,7 +75,7 @@ export function logTopology(hosts: HostInfo[], msgPrefix: string) { return `${msgPrefix}${Messages.get("Utils.topology", msg)}`; } -export function getTimeInNanos() { +export function getTimeInNanos(): bigint { return process.hrtime.bigint(); } @@ -113,32 +123,47 @@ export function equalsIgnoreCase(value1: string | null, value2: string | null): return value1 != null && value2 != null && value1.localeCompare(value2, undefined, { sensitivity: "accent" }) === 0; } -export function isDialectTopologyAware(dialect: any): dialect is TopologyAwareDatabaseDialect { - return dialect; -} - export function containsHostAndPort(hosts: HostInfo[] | null | undefined, hostAndPort: string): boolean { - if (hosts?.length === 0) { + if (!hosts || hosts.length === 0) { return false; } return hosts.some((host) => host.hostAndPort === hostAndPort); } -export class Pair { - private readonly _left: K; - private readonly _right: V; - - constructor(value1: K, value2: V) { - this._left = value1; - this._right = value2; +export function parseInstanceTemplates( + instanceTemplatesString: string | null, + hostValidator: (hostPattern: string) => void, + hostInfoBuilderFunc: () => { withHost(host: string): { build(): HostInfo } } +): Map { + if (!instanceTemplatesString) { + throw new AwsWrapperError(Messages.get("Utils.globalClusterInstanceHostPatternsRequired")); } - get left(): K { - return this._left; - } + const instanceTemplates = new Map(); + const patterns = instanceTemplatesString.split(","); + + for (const pattern of patterns) { + const trimmedPattern = pattern.trim(); + const colonIndex = trimmedPattern.indexOf(":"); + if (colonIndex === -1) { + throw new AwsWrapperError(Messages.get("Utils.invalidPatternFormat", trimmedPattern)); + } - get right(): V { - return this._right; + const region = trimmedPattern.substring(0, colonIndex).trim(); + const hostPattern = trimmedPattern.substring(colonIndex + 1).trim(); + + if (!region || !hostPattern) { + throw new AwsWrapperError(Messages.get("Utils.invalidPatternFormat", trimmedPattern)); + } + + hostValidator(hostPattern); + + const hostInfo = hostInfoBuilderFunc().withHost(hostPattern).build(); + instanceTemplates.set(region, hostInfo); } + + logger.debug(`Detected Global Database patterns: ${JSON.stringify(Array.from(instanceTemplates.entries()))}`); + + return instanceTemplates; } diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index b39d6c3e..2390c782 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -16,18 +16,20 @@ import { ConnectionProvider } from "./connection_provider"; import { DatabaseDialect } from "./database_dialect/database_dialect"; -import { ClusterTopologyMonitorImpl } from "./host_list_provider/monitoring/cluster_topology_monitor"; -import { BlueGreenStatusProvider } from "./plugins/bluegreen/blue_green_status_provider"; +import { AwsWrapperError } from "./utils/errors"; +import { Messages } from "./utils/messages"; export class WrapperProperty { name: string; description: string; defaultValue: any; + allowedValues?: T[]; - constructor(name: string, description: string, defaultValue?: any) { + constructor(name: string, description: string, defaultValue?: any, allowedValues?: T[]) { this.name = name; this.description = description; this.defaultValue = defaultValue; + this.allowedValues = allowedValues; } get(props: Map): T { @@ -36,16 +38,29 @@ export class WrapperProperty { return this.defaultValue; } + if (val != null && this.allowedValues?.length > 0) { + if (!this.allowedValues.includes(val)) { + throw new AwsWrapperError(Messages.get("WrapperProperty.invalidValue", String(val), this.name, this.allowedValues.join(", "))); + } + } + return val; } set(props: Map, val: T) { + if (val != null && this.allowedValues?.length > 0) { + if (!this.allowedValues.includes(val)) { + throw new AwsWrapperError(Messages.get("WrapperProperty.invalidValue", String(val), this.name, this.allowedValues.join(", "))); + } + } props.set(this.name, val); } } export class WrapperProperties { static readonly MONITORING_PROPERTY_PREFIX: string = "monitoring_"; + static readonly TOPOLOGY_MONITORING_PROPERTY_PREFIX: string = "topology_monitoring_"; + static readonly BG_MONITORING_PROPERTY_PREFIX: string = "blue_green_monitoring_"; static readonly DEFAULT_PLUGINS = "initialConnection,auroraConnectionTracker,failover2,efm2"; static readonly DEFAULT_TOKEN_EXPIRATION_SEC = 15 * 60; @@ -210,6 +225,38 @@ export class WrapperProperties { ); static readonly FAILOVER_MODE = new WrapperProperty("failoverMode", "Set host role to follow during failover.", ""); + static readonly FAILOVER_HOME_REGION = new WrapperProperty("failoverHomeRegion", "Set home region for GDB failover.", null); + + static readonly ACTIVE_HOME_FAILOVER_MODE = new WrapperProperty( + "activeHomeFailoverMode", + "Set host role to follow during failover when GDB primary region is in home region.", + null, + [ + "strict-writer", + "strict-home-reader", + "strict-out-of-home-reader", + "strict-any-reader", + "home-reader-or-writer", + "out-of-home-reader-or-writer", + "any-reader-or-writer" + ] + ); + + static readonly INACTIVE_HOME_FAILOVER_MODE = new WrapperProperty( + "inactiveHomeFailoverMode", + "Set host role to follow during failover when GDB primary region is not in home region.", + null, + [ + "strict-writer", + "strict-home-reader", + "strict-out-of-home-reader", + "strict-any-reader", + "home-reader-or-writer", + "out-of-home-reader-or-writer", + "any-reader-or-writer" + ] + ); + static readonly FAILOVER_READER_HOST_SELECTOR_STRATEGY = new WrapperProperty( "failoverReaderHostSelectorStrategy", "The strategy that should be used to select a new reader host while opening a new connection.", @@ -244,6 +291,16 @@ export class WrapperProperties { "clusters. Otherwise, if unspecified, the pattern will be automatically created for AWS RDS clusters." ); + static readonly GLOBAL_CLUSTER_INSTANCE_HOST_PATTERNS = new WrapperProperty( + "globalClusterInstanceHostPatterns", + "Comma-separated list of the cluster instance DNS patterns that will be used to " + + "build complete instance endpoints. " + + 'A "?" character in these patterns should be used as a placeholder for cluster instance names. ' + + "This parameter is required for Global Aurora Databases. " + + "Each region in the Global Aurora Database should be specified in the list. " + + "Format: region1:pattern1,region2:pattern2" + ); + static readonly SINGLE_WRITER_CONNECTION_STRING = new WrapperProperty( "singleWriterConnectionString", "Set to true if you are providing a connection string with multiple comma-delimited hosts and your cluster has only one writer. The writer must be the first host in the connection string", @@ -477,11 +534,37 @@ export class WrapperProperties { "Default value 0 means the Wrapper will keep reusing the same cached reader connection.", 0 ); + static readonly SKIP_INACTIVE_WRITER_CLUSTER_CHECK = new WrapperProperty( + "skipInactiveWriterClusterEndpointCheck", + "Allows to avoid connection check for inactive cluster writer endpoint.", + false + ); + + static readonly INACTIVE_CLUSTER_WRITER_SUBSTITUTION_ROLE = new WrapperProperty( + "inactiveClusterWriterEndpointSubstitutionRole", + "Defines whether or not the inactive cluster writer endpoint in the initial connection URL should be replaced with a writer instance URL from the topology info when available.", + "writer", + ["writer", "none"] + ); + + static readonly VERIFY_OPENED_CONNECTION_ROLE = new WrapperProperty( + "verifyOpenedConnectionType", + "Defines whether an opened connection should be verified to be a writer or reader, or if no role verification should be performed.", + null, + ["writer", "reader", "none"] + ); + + static readonly VERIFY_INACTIVE_CLUSTER_WRITER_CONNECTION_ROLE = new WrapperProperty( + "verifyInactiveClusterWriterEndpointConnectionType", + "Defines whether inactive cluster writer connection should be verified to be a writer, or if no role verification should be performed.", + "writer", + ["writer", "none"] + ); private static readonly PREFIXES = [ WrapperProperties.MONITORING_PROPERTY_PREFIX, - ClusterTopologyMonitorImpl.MONITORING_PROPERTY_PREFIX, - BlueGreenStatusProvider.MONITORING_PROPERTY_PREFIX + WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX, + WrapperProperties.BG_MONITORING_PROPERTY_PREFIX ]; private static startsWithPrefix(key: string): boolean { diff --git a/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md b/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md index 339fb312..579ead16 100644 --- a/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md +++ b/docs/using-the-nodejs-wrapper/UsingTheConnectionPool.md @@ -407,7 +407,7 @@ const result = await pool.query("SELECT NOW()"); ### Resources Cleanup -Throughout the application lifetime, some plugins like the Aurora Connection Tracker Plugin or the Host Monitoring Connection Plugin may create background threads shared by all connections. +Throughout the application lifetime, some plugins like the Aurora Connection Tracker Plugin or the Host Monitoring Connection Plugin may create background tasks shared by all connections. At the end of your application, call `PluginManager.releaseResources()` to clean up these shared resources. diff --git a/eslint.config.js b/eslint.config.js index b02eb491..85a4dfb9 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -35,7 +35,7 @@ const compat = new FlatCompat({ }); export default defineConfig([ - globalIgnores(["**/node_modules", "**/gradle", "**/dist", "**/coverage"]), + globalIgnores(["**/node_modules", "**/gradle", "**/dist", "**/coverage", "**/*.js"]), { extends: compat.extends( "eslint:recommended", diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index 185a6875..45b9bed7 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -39,15 +39,17 @@ import { ClientUtils } from "../../common/lib/utils/client_utils"; import { RdsMultiAZClusterMySQLDatabaseDialect } from "./dialect/rds_multi_az_mysql_database_dialect"; import { TelemetryTraceLevel } from "../../common/lib/utils/telemetry/telemetry_trace_level"; import { MySQL2DriverDialect } from "./dialect/mysql2_driver_dialect"; -import { isDialectTopologyAware } from "../../common/lib/utils/utils"; +import { isDialectTopologyAware } from "../../common/lib/database_dialect/topology_aware_database_dialect"; import { MySQLClient, MySQLPoolClient } from "./mysql_client"; import { DriverConnectionProvider } from "../../common/lib/driver_connection_provider"; +import { GlobalAuroraMySQLDatabaseDialect } from "./dialect/global_aurora_mysql_database_dialect"; class BaseAwsMySQLClient extends AwsClient implements MySQLClient { private static readonly knownDialectsByCode: Map = new Map([ [DatabaseDialectCodes.MYSQL, new MySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MYSQL, new RdsMySQLDatabaseDialect()], [DatabaseDialectCodes.AURORA_MYSQL, new AuroraMySQLDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, new GlobalAuroraMySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()] ]); @@ -113,7 +115,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { return result; } - isReadOnly(): boolean { + isReadOnly(): boolean | undefined { return this.pluginService.getSessionStateService().getReadOnly(); } @@ -129,7 +131,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { return result; } - getAutoCommit(): boolean { + getAutoCommit(): boolean | undefined { return this.pluginService.getSessionStateService().getAutoCommit(); } @@ -142,7 +144,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.pluginService.getSessionStateService().setCatalog(catalog); } - getCatalog(): string { + getCatalog(): string | undefined { return this.pluginService.getSessionStateService().getCatalog(); } @@ -150,7 +152,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { throw new UnsupportedMethodError(Messages.get("Client.methodNotSupported", "setSchema")); } - getSchema(): string { + getSchema(): string | undefined { throw new UnsupportedMethodError(Messages.get("Client.methodNotSupported", "getSchema")); } @@ -181,7 +183,7 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.pluginService.getSessionStateService().setTransactionIsolation(level); } - getTransactionIsolation(): TransactionIsolationLevel { + getTransactionIsolation(): TransactionIsolationLevel | undefined { return this.pluginService.getSessionStateService().getTransactionIsolation(); } @@ -197,6 +199,10 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { this.properties, "end", () => { + if (!this.targetClient) { + return Promise.resolve(undefined); + } + this.pluginService.removeErrorListener(this.targetClient); const res = ClientUtils.queryWithTimeout(this.targetClient.end(), this.properties); this.targetClient = undefined; diff --git a/mysql/lib/dialect/aurora_mysql_database_dialect.ts b/mysql/lib/dialect/aurora_mysql_database_dialect.ts index 3ebfa224..a4f3c69a 100644 --- a/mysql/lib/dialect/aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/aurora_mysql_database_dialect.ts @@ -15,20 +15,16 @@ */ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { HostRole } from "../../../common/lib/host_role"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { - MonitoringRdsHostListProvider -} from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; -import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect, BlueGreenDialect { private static readonly TOPOLOGY_QUERY: string = @@ -50,18 +46,9 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements private static readonly TOPOLOGY_TABLE_EXIST_QUERY: string = "SELECT 1 AS tmp FROM information_schema.tables WHERE table_schema = 'mysql' AND table_name = 'rds_topology'"; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider( - props, - originalUrl, - topologyUtils, - hostListProviderService, - (hostListProviderService) - ); - } - return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } async queryForTopology(targetClient: ClientWrapper): Promise { @@ -142,7 +129,7 @@ export class AuroraMySQLDatabaseDialect extends MySQLDatabaseDialect implements } getDialectUpdateCandidates(): string[] { - return [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL]; + return [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL]; } async isBlueGreenStatusAvailable(clientWrapper: ClientWrapper): Promise { diff --git a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts index 1b51b4e1..faf89a3c 100644 --- a/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts +++ b/mysql/lib/dialect/global_aurora_mysql_database_dialect.ts @@ -1,23 +1,27 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { AuroraMySQLDatabaseDialect } from "./aurora_mysql_database_dialect"; import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { GlobalAuroraHostListProvider } from "../../../common/lib/host_list_provider/global_aurora_host_list_provider"; +import { GlobalTopologyUtils } from "../../../common/lib/host_list_provider/global_topology_utils"; export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect implements GlobalAuroraTopologyDialect { private static readonly GLOBAL_STATUS_TABLE_EXISTS_QUERY = @@ -29,8 +33,8 @@ export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect " upper(table_schema) = 'INFORMATION_SCHEMA' AND upper(table_name) = 'AURORA_GLOBAL_DB_INSTANCE_STATUS'"; private static readonly GLOBAL_TOPOLOGY_QUERY = - "SELECT SERVER_ID, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS IS_WRITER, " + - "VISIBILITY_LAG_IN_MSEC, AWS_REGION " + + "SELECT server_id, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS is_writer, " + + "visibility_lag_in_msec, aws_region " + "FROM information_schema.aurora_global_db_instance_status"; private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM information_schema.aurora_global_db_status"; @@ -68,7 +72,14 @@ export class GlobalAuroraMySQLDatabaseDialect extends AuroraMySQLDatabaseDialect return []; } - // TODO: implement GetHostListProvider once GDBHostListProvider is implemented + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new GlobalAuroraHostListProvider( + props, + originalUrl, + new GlobalTopologyUtils(this, servicesContainer.pluginService.getHostInfoBuilder()), + servicesContainer + ); + } async queryForTopology(targetClient: ClientWrapper): Promise { const res = await targetClient.query(GlobalAuroraMySQLDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); diff --git a/mysql/lib/dialect/mysql2_driver_dialect.ts b/mysql/lib/dialect/mysql2_driver_dialect.ts index c9e3e584..3e7b9f3a 100644 --- a/mysql/lib/dialect/mysql2_driver_dialect.ts +++ b/mysql/lib/dialect/mysql2_driver_dialect.ts @@ -47,6 +47,8 @@ export class MySQL2DriverDialect implements DriverDialect { preparePoolClientProperties(props: Map, poolConfig: AwsPoolConfig | undefined): any { const finalPoolConfig: PoolOptions = {}; const finalClientProps = WrapperProperties.removeWrapperProperties(props); + this.setKeepAliveProperties(finalClientProps, props.get(WrapperProperties.KEEPALIVE_PROPERTIES.name)); + this.setConnectTimeout(finalClientProps, props.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name)); Object.assign(finalPoolConfig, Object.fromEntries(finalClientProps.entries())); finalPoolConfig.connectionLimit = poolConfig?.maxConnections; @@ -69,6 +71,9 @@ export class MySQL2DriverDialect implements DriverDialect { } setQueryTimeout(props: Map, sql?: any, wrapperQueryTimeout?: any) { + if (!sql) { + return; + } const timeout = wrapperQueryTimeout ?? props.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name); if (timeout && !sql[MySQL2DriverDialect.QUERY_TIMEOUT_PROPERTY_NAME]) { sql[MySQL2DriverDialect.QUERY_TIMEOUT_PROPERTY_NAME] = Number(timeout); diff --git a/mysql/lib/dialect/mysql_database_dialect.ts b/mysql/lib/dialect/mysql_database_dialect.ts index 084ea6b2..f0fe20ad 100644 --- a/mysql/lib/dialect/mysql_database_dialect.ts +++ b/mysql/lib/dialect/mysql_database_dialect.ts @@ -28,6 +28,7 @@ import { ErrorHandler } from "../../../common/lib/error_handler"; import { MySQLErrorHandler } from "../mysql_error_handler"; import { Messages } from "../../../common/lib/utils/messages"; import { HostRole } from "../../../common/lib/host_role"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class MySQLDatabaseDialect implements DatabaseDialect { protected dialectName: string = this.constructor.name; @@ -108,8 +109,8 @@ export class MySQLDatabaseDialect implements DatabaseDialect { }); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), servicesContainer.hostListProviderService); } getErrorHandler(): ErrorHandler { diff --git a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts index f070e8a1..33ac2169 100644 --- a/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts +++ b/mysql/lib/dialect/rds_multi_az_mysql_database_dialect.ts @@ -15,7 +15,6 @@ */ import { MySQLDatabaseDialect } from "./mysql_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { HostRole } from "../../../common/lib/host_role"; @@ -24,10 +23,9 @@ import { AwsWrapperError } from "../../../common/lib/utils/errors"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { FailoverRestriction } from "../../../common/lib/plugins/failover/failover_restriction"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { PluginService } from "../../../common/lib/plugin_service"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect implements TopologyAwareDatabaseDialect { private static readonly TOPOLOGY_QUERY: string = "SELECT id, endpoint, port FROM mysql.rds_topology"; @@ -71,18 +69,9 @@ export class RdsMultiAZClusterMySQLDatabaseDialect extends MySQLDatabaseDialect .catch(() => false); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider( - props, - originalUrl, - topologyUtils, - hostListProviderService, - (hostListProviderService) - ); - } - return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } async queryForTopology(targetClient: ClientWrapper): Promise { diff --git a/package.json b/package.json index 7977242a..ad0de7f9 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "format": "prettier . --write --config .prettierrc", "integration": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" npx jest --config=jest.integration.config.json --runInBand --verbose", "debug-integration": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules --inspect-brk=0.0.0.0:5005\" npx jest --config=jest.integration.config.json --runInBand", - "lint": "eslint --fix --ext .ts .", + "lint": "eslint --ext .ts .", "test": "cross-env NODE_OPTIONS=\"$NODE_OPTIONS --experimental-vm-modules\" npx jest --config=jest.unit.config.json", "fix-imports": "babel --extensions \".js\" dist -d dist", "build": "tsc", diff --git a/pg/lib/client.ts b/pg/lib/client.ts index 13e0bd71..b8c55464 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -38,15 +38,17 @@ import { ClientWrapper } from "../../common/lib/client_wrapper"; import { RdsMultiAZClusterPgDatabaseDialect } from "./dialect/rds_multi_az_pg_database_dialect"; import { TelemetryTraceLevel } from "../../common/lib/utils/telemetry/telemetry_trace_level"; import { NodePostgresDriverDialect } from "./dialect/node_postgres_driver_dialect"; -import { isDialectTopologyAware } from "../../common/lib/utils/utils"; +import { isDialectTopologyAware } from "../../common/lib/database_dialect/topology_aware_database_dialect"; import { PGClient, PGPoolClient } from "./pg_client"; import { DriverConnectionProvider } from "../../common/lib/driver_connection_provider"; +import { GlobalAuroraPgDatabaseDialect } from "./dialect/global_aurora_pg_database_dialect"; class BaseAwsPgClient extends AwsClient implements PGClient { private static readonly knownDialectsByCode: Map = new Map([ [DatabaseDialectCodes.PG, new PgDatabaseDialect()], [DatabaseDialectCodes.RDS_PG, new RdsPgDatabaseDialect()], [DatabaseDialectCodes.AURORA_PG, new AuroraPgDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_PG, new GlobalAuroraPgDatabaseDialect()], [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()] ]); @@ -82,7 +84,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { return result; } - isReadOnly(): boolean { + isReadOnly(): boolean | undefined { return this.pluginService.getSessionStateService().getReadOnly(); } @@ -120,7 +122,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { this.pluginService.getSessionStateService().setTransactionIsolation(level); } - getTransactionIsolation(): TransactionIsolationLevel { + getTransactionIsolation(): TransactionIsolationLevel | undefined { return this.pluginService.getSessionStateService().getTransactionIsolation(); } @@ -147,7 +149,7 @@ class BaseAwsPgClient extends AwsClient implements PGClient { return result; } - getSchema(): string { + getSchema(): string | undefined { return this.pluginService.getSessionStateService().getSchema(); } @@ -399,7 +401,7 @@ export class AwsPgPoolClient implements PGPoolClient { await awsPGPooledConnection.connect(); const res = await awsPGPooledConnection.query(queryTextOrConfig as any, values); await awsPGPooledConnection.end(); - return res; + return res as any; } catch (error: any) { if (!(error instanceof FailoverSuccessError)) { // Release pooled connection. diff --git a/pg/lib/dialect/aurora_pg_database_dialect.ts b/pg/lib/dialect/aurora_pg_database_dialect.ts index c717c2d6..baf8e14c 100644 --- a/pg/lib/dialect/aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/aurora_pg_database_dialect.ts @@ -15,7 +15,6 @@ */ import { PgDatabaseDialect } from "./pg_database_dialect"; -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_host_list_provider"; import { TopologyAwareDatabaseDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; @@ -23,11 +22,10 @@ import { HostRole } from "../../../common/lib"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { DatabaseDialectCodes } from "../../../common/lib/database_dialect/database_dialect_codes"; import { LimitlessDatabaseDialect } from "../../../common/lib/database_dialect/limitless_database_dialect"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { PluginService } from "../../../common/lib/plugin_service"; import { BlueGreenDialect, BlueGreenResult } from "../../../common/lib/database_dialect/blue_green_dialect"; -import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect, LimitlessDatabaseDialect, BlueGreenDialect { private static readonly VERSION = process.env.npm_package_version; @@ -53,22 +51,13 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo private static readonly TOPOLOGY_TABLE_EXIST_QUERY: string = "SELECT pg_catalog.'get_blue_green_fast_switchover_metadata'::regproc"; - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider( - props, - originalUrl, - topologyUtils, - hostListProviderService, - (hostListProviderService) - ); - } - return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } async queryForTopology(targetClient: ClientWrapper): Promise { - const res = await targetClient.query(AuroraPgDatabaseDialect.TOPOLOGY_QUERY); + const res = await targetClient.queryWithTimeout(AuroraPgDatabaseDialect.TOPOLOGY_QUERY); const results: TopologyQueryResult[] = []; const rows: any[] = res.rows; rows.forEach((row) => { @@ -149,7 +138,7 @@ export class AuroraPgDatabaseDialect extends PgDatabaseDialect implements Topolo } getDialectUpdateCandidates(): string[] { - return [DatabaseDialectCodes.RDS_MULTI_AZ_PG]; + return [DatabaseDialectCodes.GLOBAL_AURORA_PG, DatabaseDialectCodes.RDS_MULTI_AZ_PG]; } getLimitlessRoutersQuery(): string { diff --git a/pg/lib/dialect/global_aurora_pg_database_dialect.ts b/pg/lib/dialect/global_aurora_pg_database_dialect.ts index c2a86454..d452428e 100644 --- a/pg/lib/dialect/global_aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/global_aurora_pg_database_dialect.ts @@ -1,23 +1,27 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { AuroraPgDatabaseDialect } from "./aurora_pg_database_dialect"; import { GlobalAuroraTopologyDialect } from "../../../common/lib/database_dialect/topology_aware_database_dialect"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; +import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; +import { GlobalAuroraHostListProvider } from "../../../common/lib/host_list_provider/global_aurora_host_list_provider"; +import { GlobalTopologyUtils } from "../../../common/lib/host_list_provider/global_topology_utils"; export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect implements GlobalAuroraTopologyDialect { private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_status'::regproc"; @@ -77,10 +81,17 @@ export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect imple return []; } - // TODO: implement GetHostListProvider once GDBHostListProvider is implemented + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new GlobalAuroraHostListProvider( + props, + originalUrl, + new GlobalTopologyUtils(this, servicesContainer.pluginService.getHostInfoBuilder()), + servicesContainer + ); + } async queryForTopology(targetClient: ClientWrapper): Promise { - const res = await targetClient.query(GlobalAuroraPgDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); + const res = await targetClient.queryWithTimeout(GlobalAuroraPgDatabaseDialect.GLOBAL_TOPOLOGY_QUERY); const hosts: TopologyQueryResult[] = []; const rows: any[] = res.rows; rows.forEach((row) => { diff --git a/pg/lib/dialect/node_postgres_driver_dialect.ts b/pg/lib/dialect/node_postgres_driver_dialect.ts index 7cf65939..40d1ba38 100644 --- a/pg/lib/dialect/node_postgres_driver_dialect.ts +++ b/pg/lib/dialect/node_postgres_driver_dialect.ts @@ -49,8 +49,12 @@ export class NodePostgresDriverDialect implements DriverDialect { preparePoolClientProperties(props: Map, poolConfig: AwsPoolConfig | undefined): any { const finalPoolConfig: pkgPg.PoolConfig = {}; const finalClientProps = WrapperProperties.removeWrapperProperties(props); + this.setKeepAliveProperties(finalClientProps, props.get(WrapperProperties.KEEPALIVE_PROPERTIES.name)); + this.setConnectTimeout(finalClientProps, props.get(WrapperProperties.WRAPPER_CONNECT_TIMEOUT.name)); + this.setQueryTimeout(finalClientProps, undefined, props.get(WrapperProperties.WRAPPER_QUERY_TIMEOUT.name)); Object.assign(finalPoolConfig, Object.fromEntries(finalClientProps.entries())); + finalPoolConfig.max = poolConfig?.maxConnections; finalPoolConfig.idleTimeoutMillis = poolConfig?.idleTimeoutMillis; finalPoolConfig.allowExitOnIdle = poolConfig?.allowExitOnIdle; diff --git a/pg/lib/dialect/pg_database_dialect.ts b/pg/lib/dialect/pg_database_dialect.ts index 3b03ef24..0afa54f2 100644 --- a/pg/lib/dialect/pg_database_dialect.ts +++ b/pg/lib/dialect/pg_database_dialect.ts @@ -25,6 +25,7 @@ import { FailoverRestriction } from "../../../common/lib/plugins/failover/failov import { ErrorHandler } from "../../../common/lib/error_handler"; import { PgErrorHandler } from "../pg_error_handler"; import { Messages } from "../../../common/lib/utils/messages"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class PgDatabaseDialect implements DatabaseDialect { protected dialectName: string = this.constructor.name; @@ -105,8 +106,8 @@ export class PgDatabaseDialect implements DatabaseDialect { }); } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + return new ConnectionStringHostListProvider(props, originalUrl, this.getDefaultPort(), servicesContainer.hostListProviderService); } getErrorHandler(): ErrorHandler { diff --git a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts index d3e47f94..5b4b79b2 100644 --- a/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts +++ b/pg/lib/dialect/rds_multi_az_pg_database_dialect.ts @@ -14,7 +14,6 @@ limitations under the License. */ -import { HostListProviderService } from "../../../common/lib/host_list_provider_service"; import { HostListProvider } from "../../../common/lib/host_list_provider/host_list_provider"; import { ClientWrapper } from "../../../common/lib/client_wrapper"; import { AwsWrapperError, HostRole } from "../../../common/lib"; @@ -24,10 +23,9 @@ import { RdsHostListProvider } from "../../../common/lib/host_list_provider/rds_ import { PgDatabaseDialect } from "./pg_database_dialect"; import { ErrorHandler } from "../../../common/lib/error_handler"; import { MultiAzPgErrorHandler } from "../multi_az_pg_error_handler"; -import { WrapperProperties } from "../../../common/lib/wrapper_property"; -import { PluginService } from "../../../common/lib/plugin_service"; -import { MonitoringRdsHostListProvider } from "../../../common/lib/host_list_provider/monitoring/monitoring_host_list_provider"; -import { TopologyQueryResult, TopologyUtils } from "../../../common/lib/host_list_provider/topology_utils"; +import { TopologyQueryResult } from "../../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../../common/lib/host_list_provider/aurora_topology_utils"; +import { FullServicesContainer } from "../../../common/lib/utils/full_services_container"; export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implements TopologyAwareDatabaseDialect { constructor() { @@ -64,18 +62,9 @@ export class RdsMultiAZClusterPgDatabaseDialect extends PgDatabaseDialect implem } } - getHostListProvider(props: Map, originalUrl: string, hostListProviderService: HostListProviderService): HostListProvider { - const topologyUtils: TopologyUtils = new TopologyUtils(this, hostListProviderService.getHostInfoBuilder()); - if (WrapperProperties.PLUGINS.get(props).includes("failover2")) { - return new MonitoringRdsHostListProvider( - props, - originalUrl, - topologyUtils, - hostListProviderService, - (hostListProviderService) - ); - } - return new RdsHostListProvider(props, originalUrl, topologyUtils, hostListProviderService); + getHostListProvider(props: Map, originalUrl: string, servicesContainer: FullServicesContainer): HostListProvider { + const topologyUtils = new AuroraTopologyUtils(this, servicesContainer.hostListProviderService.getHostInfoBuilder()); + return new RdsHostListProvider(props, originalUrl, topologyUtils, servicesContainer); } async queryForTopology(targetClient: ClientWrapper): Promise { diff --git a/tests/integration/container/tests/aurora_failover.test.ts b/tests/integration/container/tests/aurora_failover.test.ts deleted file mode 100644 index f580bc1f..00000000 --- a/tests/integration/container/tests/aurora_failover.test.ts +++ /dev/null @@ -1,315 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { TestEnvironment } from "./utils/test_environment"; -import { DriverHelper } from "./utils/driver_helper"; -import { AuroraTestUtility } from "./utils/aurora_test_utility"; -import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../index"; -import { DatabaseEngine } from "./utils/database_engine"; -import { QueryResult } from "pg"; -import { ProxyHelper } from "./utils/proxy_helper"; -import { logger } from "../../../../common/logutils"; -import { features, instanceCount } from "./config"; -import { TestEnvironmentFeatures } from "./utils/test_environment_features"; -import { RdsUtils } from "../../../../common/lib/utils/rds_utils"; - -const itIf = - features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && - !features.includes(TestEnvironmentFeatures.PERFORMANCE) && - !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && - instanceCount >= 2 - ? it - : it.skip; -const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; -const itIfThreeInstanceAuroraCluster = instanceCount == 3 && !features.includes(TestEnvironmentFeatures.RDS_MULTI_AZ_SUPPORTED) ? it : it.skip; - -let env: TestEnvironment; -let driver; -let client: any; -let secondaryClient: any; -let initClientFunc: (props: any) => any; - -let auroraTestUtility: AuroraTestUtility; - -async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { - let config: any = { - user: env.databaseInfo.username, - host: host, - database: env.databaseInfo.defaultDbName, - password: env.databaseInfo.password, - port: port, - plugins: "failover", - failoverTimeoutMs: 250000, - enableTelemetry: true, - telemetryTracesBackend: "OTLP", - telemetryMetricsBackend: "OTLP" - }; - if (connectToProxy) { - config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; - } - config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); - return config; -} - -async function initConfigWithEFM2(host: string, port: number, connectToProxy: boolean): Promise { - const config: any = await initDefaultConfig(host, port, connectToProxy); - config["plugins"] = "failover,efm2"; - config["failoverTimeoutMs"] = 20000; - config["failureDetectionCount"] = 2; - config["failureDetectionInterval"] = 1000; - config["failureDetectionTime"] = 2000; - config["connectTimeout"] = 10000; - config["wrapperQueryTimeout"] = 20000; - config["monitoring_wrapperQueryTimeout"] = 3000; - config["monitoring_wrapperConnectTimeout"] = 3000; - return config; -} - -describe("aurora failover", () => { - beforeEach(async () => { - logger.info(`Test started: ${expect.getState().currentTestName}`); - env = await TestEnvironment.getCurrent(); - - auroraTestUtility = new AuroraTestUtility(env.region); - driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - initClientFunc = DriverHelper.getClient(driver); - await ProxyHelper.enableAllConnectivity(); - await TestEnvironment.verifyClusterStatus(); - - client = null; - secondaryClient = null; - }, 1320000); - - afterEach(async () => { - if (client !== null) { - try { - await client.end(); - } catch (error) { - // pass - } - } - - if (secondaryClient !== null) { - try { - await secondaryClient.end(); - } catch (error) { - // pass - } - } - await PluginManager.releaseResources(); - logger.info(`Test finished: ${expect.getState().currentTestName}`); - }, 1320000); - - itIfThreeInstanceAuroraCluster( - "writer failover efm", - async () => { - // Connect to writer instance. - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - writerConfig["failoverMode"] = "reader-or-writer"; - - client = initClientFunc(writerConfig); - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - const instances = env.databaseInfo.instances; - const readerInstance = instances[1].instanceId; - await ProxyHelper.disableAllConnectivity(env.engine); - - try { - await ProxyHelper.enableConnectivity(initialWriterId); - - // Sleep query activates monitoring connection after monitoring_wrapperQueryTimeout time is reached. - await auroraTestUtility.queryInstanceIdWithSleep(client); - - await ProxyHelper.enableConnectivity(readerInstance); - await ProxyHelper.disableConnectivity(env.engine, initialWriterId); - } catch (error) { - fail("The disable connectivity task was unexpectedly interrupted."); - } - // Failure occurs on connection invocation. - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "fails from writer to new writer on connection invocation", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "writer fails within transaction", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - - // Crash instance 1 and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); - }).rejects.toThrow(TransactionResolutionUnknownError); - - // Attempt to query the instance id. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - - // Assert that we are connected to the new writer after failover happens. - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - - const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); - expect(currentConnectionId).toBe(nextClusterWriterId); - expect(initialWriterId).not.toBe(nextClusterWriterId); - - // Assert that NO row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("0"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); - } - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, - 2000000 - ); - - itIf( - "fails from writer and transfers session state", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); - - await client.setReadOnly(true); - await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - - if (driver === DatabaseEngine.PG) { - await client.setSchema(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - await client.setAutoCommit(false); - await client.setCatalog(env.databaseInfo.defaultDbName); - } - - // Failover cluster and nominate a new writer - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - expect(client.isReadOnly()).toBe(true); - expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - if (driver === DatabaseEngine.PG) { - expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - expect(client.getAutoCommit()).toBe(false); - expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); - } - }, - 1320000 - ); - - itIfTwoInstance( - "fails from reader to writer", - async () => { - // Connect to writer instance - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Get a reader instance - let readerInstanceHost; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - readerInstanceHost = host.host; - } - } - if (!readerInstanceHost) { - throw new Error("Could not find a reader instance"); - } - const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); - - secondaryClient = initClientFunc(readerConfig); - await secondaryClient.connect(); - - // Crash the reader instance - const rdsUtils = new RdsUtils(); - const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); - if (readerInstanceId) { - await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(secondaryClient); - }).rejects.toThrow(FailoverSuccessError); - - await ProxyHelper.enableConnectivity(readerInstanceId); - - // Assert that we are currently connected to the writer instance - const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).toBe(initialWriterId); - } - }, - 1320000 - ); -}); diff --git a/tests/integration/container/tests/aurora_failover2.test.ts b/tests/integration/container/tests/aurora_failover2.test.ts deleted file mode 100644 index 61ebf362..00000000 --- a/tests/integration/container/tests/aurora_failover2.test.ts +++ /dev/null @@ -1,259 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { TestEnvironment } from "./utils/test_environment"; -import { DriverHelper } from "./utils/driver_helper"; -import { AuroraTestUtility } from "./utils/aurora_test_utility"; -import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../index"; -import { DatabaseEngine } from "./utils/database_engine"; -import { QueryResult } from "pg"; -import { ProxyHelper } from "./utils/proxy_helper"; -import { logger } from "../../../../common/logutils"; -import { features, instanceCount } from "./config"; -import { TestEnvironmentFeatures } from "./utils/test_environment_features"; -import { RdsUtils } from "../../../../common/lib/utils/rds_utils"; - -const itIf = - features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && - !features.includes(TestEnvironmentFeatures.PERFORMANCE) && - !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && - instanceCount >= 2 - ? it - : it.skip; -const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; - -let env: TestEnvironment; -let driver; -let client: any; -let secondaryClient: any; -let initClientFunc: (props: any) => any; - -let auroraTestUtility: AuroraTestUtility; - -async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { - let config: any = { - user: env.databaseInfo.username, - host: host, - database: env.databaseInfo.defaultDbName, - password: env.databaseInfo.password, - port: port, - plugins: "failover2", - failoverTimeoutMs: 250000, - enableTelemetry: true, - telemetryTracesBackend: "OTLP", - telemetryMetricsBackend: "OTLP" - }; - if (connectToProxy) { - config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; - } - config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); - return config; -} - -describe("aurora failover2", () => { - beforeEach(async () => { - logger.info(`Test started: ${expect.getState().currentTestName}`); - env = await TestEnvironment.getCurrent(); - - auroraTestUtility = new AuroraTestUtility(env.region); - driver = DriverHelper.getDriverForDatabaseEngine(env.engine); - initClientFunc = DriverHelper.getClient(driver); - await ProxyHelper.enableAllConnectivity(); - await TestEnvironment.verifyClusterStatus(); - - client = null; - secondaryClient = null; - }, 1320000); - - afterEach(async () => { - if (client !== null) { - try { - await client.end(); - } catch (error) { - // pass - } - } - - if (secondaryClient !== null) { - try { - await secondaryClient.end(); - } catch (error) { - // pass - } - } - await PluginManager.releaseResources(); - logger.info(`Test finished: ${expect.getState().currentTestName}`); - }, 1320000); - - itIf( - "fails from writer to new writer on connection invocation", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Crash instance 1 and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - }, - 1320000 - ); - - itIf( - "writer fails within transaction", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - await DriverHelper.executeQuery(env.engine, client, "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)"); - - await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); // start transaction - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); - - // Crash instance 1 and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); - }).rejects.toThrow(TransactionResolutionUnknownError); - - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - // Assert that we are connected to the new writer after failover happens. - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - - const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); - expect(currentConnectionId).toBe(nextClusterWriterId); - expect(initialWriterId).not.toBe(nextClusterWriterId); - - // Assert that NO row has been inserted to the table. - const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); - if (env.engine === DatabaseEngine.PG) { - expect((result as QueryResult).rows[0]["count"]).toBe("0"); - } else if (env.engine === DatabaseEngine.MYSQL) { - expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); - } - - await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); - }, - 2000000 - ); - - itIf( - "fails from writer and transfers session state", - async () => { - const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); - client = initClientFunc(config); - - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); - - await client.setReadOnly(true); - await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - - if (driver === DatabaseEngine.PG) { - await client.setSchema(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - await client.setAutoCommit(false); - await client.setCatalog(env.databaseInfo.defaultDbName); - } - - // Failover cluster and nominate a new writer. - await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(client); - }).rejects.toThrow(FailoverSuccessError); - - // Assert that we are connected to the new writer after failover happens. - const currentConnectionId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).not.toBe(initialWriterId); - expect(client.isReadOnly()).toBe(true); - expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); - if (driver === DatabaseEngine.PG) { - expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); - } else if (driver === DatabaseEngine.MYSQL) { - expect(client.getAutoCommit()).toBe(false); - expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); - } - }, - 1320000 - ); - - itIfTwoInstance( - "fails from reader to writer", - async () => { - // Connect to writer instance. - const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); - client = initClientFunc(writerConfig); - await client.connect(); - const initialWriterId = await auroraTestUtility.queryInstanceId(client); - expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); - - // Get a reader instance. - let readerInstanceHost; - for (const host of env.proxyDatabaseInfo.instances) { - if (host.instanceId && host.instanceId !== initialWriterId) { - readerInstanceHost = host.host; - } - } - if (!readerInstanceHost) { - throw new Error("Could not find a reader instance"); - } - const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); - - secondaryClient = initClientFunc(readerConfig); - await secondaryClient.connect(); - - // Crash the reader instance. - const rdsUtils = new RdsUtils(); - const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); - if (readerInstanceId) { - await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); - - await expect(async () => { - await auroraTestUtility.queryInstanceId(secondaryClient); - }).rejects.toThrow(FailoverSuccessError); - - await ProxyHelper.enableConnectivity(readerInstanceId); - - // Assert that we are currently connected to the writer instance. - const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); - expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); - expect(currentConnectionId).toBe(initialWriterId); - } - }, - 1320000 - ); -}); diff --git a/tests/integration/container/tests/connect_execute_time_plugin.test.ts b/tests/integration/container/tests/connect_execute_time_plugin.test.ts index 3ff336ad..b8091d26 100644 --- a/tests/integration/container/tests/connect_execute_time_plugin.test.ts +++ b/tests/integration/container/tests/connect_execute_time_plugin.test.ts @@ -70,8 +70,7 @@ describe("aurora connect and execute time plugin", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { diff --git a/tests/integration/container/tests/failover/aurora_failover.test.ts b/tests/integration/container/tests/failover/aurora_failover.test.ts new file mode 100644 index 00000000..7d91ee45 --- /dev/null +++ b/tests/integration/container/tests/failover/aurora_failover.test.ts @@ -0,0 +1,125 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager } from "../../../../../index"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { createFailoverTests } from "./failover_tests"; + +const itIfThreeInstanceAuroraCluster = instanceCount == 3 && !features.includes(TestEnvironmentFeatures.RDS_MULTI_AZ_SUPPORTED) ? it : it.skip; + +describe("aurora failover", createFailoverTests({ plugins: "failover" })); + +describe("aurora failover - efm specific", () => { + let env: TestEnvironment; + let client: any; + let initClientFunc: (props: any) => any; + let auroraTestUtility: AuroraTestUtility; + + async function initConfigWithEFM2(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: "failover,efm2", + failoverTimeoutMs: 20000, + failureDetectionCount: 2, + failureDetectionInterval: 1000, + failureDetectionTime: 2000, + connectTimeout: 10000, + wrapperQueryTimeout: 20000, + monitoring_wrapperQueryTimeout: 3000, + monitoring_wrapperConnectTimeout: 3000, + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP" + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; + } + + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + auroraTestUtility = new AuroraTestUtility(env.region); + const driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + client = null; + }, 1320000); + + afterEach(async () => { + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIfThreeInstanceAuroraCluster( + "writer failover efm", + async () => { + // Connect to writer instance + const writerConfig = await initConfigWithEFM2(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + writerConfig["failoverMode"] = "reader-or-writer"; + + client = initClientFunc(writerConfig); + await client.connect(); + + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + const instances = env.databaseInfo.instances; + const readerInstance = instances[1].instanceId; + await ProxyHelper.disableAllConnectivity(env.engine); + + try { + await ProxyHelper.enableConnectivity(initialWriterId); + + // Sleep query activates monitoring connection after monitoring_wrapperQueryTimeout time is reached + await auroraTestUtility.queryInstanceIdWithSleep(client); + + await ProxyHelper.enableConnectivity(readerInstance); + await ProxyHelper.disableConnectivity(env.engine, initialWriterId); + } catch (error) { + fail("The disable connectivity task was unexpectedly interrupted."); + } + // Failure occurs on connection invocation + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1320000 + ); +}); diff --git a/tests/integration/container/tests/failover/aurora_failover2.test.ts b/tests/integration/container/tests/failover/aurora_failover2.test.ts new file mode 100644 index 00000000..aab1564e --- /dev/null +++ b/tests/integration/container/tests/failover/aurora_failover2.test.ts @@ -0,0 +1,19 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { createFailoverTests } from "./failover_tests"; + +describe("aurora failover2", createFailoverTests({ plugins: "failover2" })); diff --git a/tests/integration/container/tests/failover/failover_tests.ts b/tests/integration/container/tests/failover/failover_tests.ts new file mode 100644 index 00000000..ce430464 --- /dev/null +++ b/tests/integration/container/tests/failover/failover_tests.ts @@ -0,0 +1,271 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager, TransactionIsolationLevel, TransactionResolutionUnknownError } from "../../../../../index"; +import { DatabaseEngine } from "../utils/database_engine"; +import { QueryResult } from "pg"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { RdsUtils } from "../../../../../common/lib/utils/rds_utils"; + +export interface FailoverTestOptions { + plugins: string; + getExtraConfig?: () => Record; +} + +export function createFailoverTests(options: FailoverTestOptions) { + const itIf = + features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && + !features.includes(TestEnvironmentFeatures.PERFORMANCE) && + !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && + instanceCount >= 2 + ? it + : it.skip; + const itIfTwoInstance = instanceCount == 2 ? itIf : it.skip; + + return () => { + let env: TestEnvironment; + let driver: any; + let client: any; + let secondaryClient: any; + let initClientFunc: (props: any) => any; + let auroraTestUtility: AuroraTestUtility; + + async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: options.plugins, + failoverTimeoutMs: 250000, + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP", + ...options.getExtraConfig?.() + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; + } + + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + auroraTestUtility = new AuroraTestUtility(env.region); + driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + client = null; + secondaryClient = null; + }, 1320000); + + afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); + + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + if (secondaryClient !== null) { + try { + await secondaryClient.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIf( + "fails from writer to new writer on connection invocation", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + }, + 1320000 + ); + + itIf( + "writer fails within transaction", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + await DriverHelper.executeQuery( + env.engine, + client, + "CREATE TABLE test3_3 (id int not null primary key, test3_3_field varchar(255) not null)" + ); + + await DriverHelper.executeQuery(env.engine, client, "START TRANSACTION"); + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (1, 'test field string 1')"); + + // Crash instance 1 and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await DriverHelper.executeQuery(env.engine, client, "INSERT INTO test3_3 VALUES (2, 'test field string 2')"); + }).rejects.toThrow(TransactionResolutionUnknownError); + + // Attempt to query the instance id + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + + // Assert that we are connected to the new writer after failover happens + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + + const nextClusterWriterId = await auroraTestUtility.getClusterWriterInstanceId(); + expect(currentConnectionId).toBe(nextClusterWriterId); + expect(initialWriterId).not.toBe(nextClusterWriterId); + + // Assert that NO row has been inserted to the table + const result = await DriverHelper.executeQuery(env.engine, client, "SELECT count(*) from test3_3"); + if (env.engine === DatabaseEngine.PG) { + expect((result as QueryResult).rows[0]["count"]).toBe("0"); + } else if (env.engine === DatabaseEngine.MYSQL) { + expect(JSON.parse(JSON.stringify(result))[0][0]["count(*)"]).toBe(0); + } + + await DriverHelper.executeQuery(env.engine, client, "DROP TABLE IF EXISTS test3_3"); + }, + 2000000 + ); + + itIf( + "fails from writer and transfers session state", + async () => { + const config = await initDefaultConfig(env.databaseInfo.writerInstanceEndpoint, env.databaseInfo.instanceEndpointPort, false); + client = initClientFunc(config); + + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toBe(true); + + await client.setReadOnly(true); + await client.setTransactionIsolation(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); + + if (driver === DatabaseEngine.PG) { + await client.setSchema(env.databaseInfo.defaultDbName); + } else if (driver === DatabaseEngine.MYSQL) { + await client.setAutoCommit(false); + await client.setCatalog(env.databaseInfo.defaultDbName); + } + + // Failover cluster and nominate a new writer + await auroraTestUtility.failoverClusterAndWaitUntilWriterChanged(); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + // Assert that we are connected to the new writer after failover happens + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).not.toBe(initialWriterId); + expect(client.isReadOnly()).toBe(true); + expect(client.getTransactionIsolation()).toBe(TransactionIsolationLevel.TRANSACTION_SERIALIZABLE); + if (driver === DatabaseEngine.PG) { + expect(client.getSchema()).toBe(env.databaseInfo.defaultDbName); + } else if (driver === DatabaseEngine.MYSQL) { + expect(client.getAutoCommit()).toBe(false); + expect(client.getCatalog()).toBe(env.databaseInfo.defaultDbName); + } + }, + 1320000 + ); + + itIfTwoInstance( + "fails from reader to writer", + async () => { + // Connect to writer instance + const writerConfig = await initDefaultConfig(env.proxyDatabaseInfo.writerInstanceEndpoint, env.proxyDatabaseInfo.instanceEndpointPort, true); + client = initClientFunc(writerConfig); + await client.connect(); + const initialWriterId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); + + // Get a reader instance + let readerInstanceHost; + for (const host of env.proxyDatabaseInfo.instances) { + if (host.instanceId && host.instanceId !== initialWriterId) { + readerInstanceHost = host.host; + } + } + if (!readerInstanceHost) { + throw new Error("Could not find a reader instance"); + } + const readerConfig = await initDefaultConfig(readerInstanceHost, env.proxyDatabaseInfo.instanceEndpointPort, true); + + secondaryClient = initClientFunc(readerConfig); + await secondaryClient.connect(); + + // Crash the reader instance + const rdsUtils = new RdsUtils(); + const readerInstanceId = rdsUtils.getRdsInstanceId(readerInstanceHost); + if (readerInstanceId) { + await ProxyHelper.disableConnectivity(env.engine, readerInstanceId); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(secondaryClient); + }).rejects.toThrow(FailoverSuccessError); + + await ProxyHelper.enableConnectivity(readerInstanceId); + + // Assert that we are currently connected to the writer instance + const currentConnectionId = await auroraTestUtility.queryInstanceId(secondaryClient); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(true); + expect(currentConnectionId).toBe(initialWriterId); + } + }, + 1320000 + ); + }; +} diff --git a/tests/integration/container/tests/failover/gdb_failover.test.ts b/tests/integration/container/tests/failover/gdb_failover.test.ts new file mode 100644 index 00000000..261d98c0 --- /dev/null +++ b/tests/integration/container/tests/failover/gdb_failover.test.ts @@ -0,0 +1,184 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { TestEnvironment } from "../utils/test_environment"; +import { DriverHelper } from "../utils/driver_helper"; +import { AuroraTestUtility } from "../utils/aurora_test_utility"; +import { FailoverSuccessError, PluginManager } from "../../../../../index"; +import { ProxyHelper } from "../utils/proxy_helper"; +import { logger } from "../../../../../common/logutils"; +import { features, instanceCount } from "../config"; +import { TestEnvironmentFeatures } from "../utils/test_environment_features"; +import { createFailoverTests } from "./failover_tests"; + +const itIf = + features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && + !features.includes(TestEnvironmentFeatures.PERFORMANCE) && + !features.includes(TestEnvironmentFeatures.RUN_AUTOSCALING_TESTS_ONLY) && + instanceCount >= 2 + ? it + : it.skip; +const itIfNetworkOutages = features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED) && instanceCount >= 2 ? itIf : it.skip; + +let env: TestEnvironment; +let driver: any; +let client: any; +let initClientFunc: (props: any) => any; + +let auroraTestUtility: AuroraTestUtility; + +async function initDefaultConfig(host: string, port: number, connectToProxy: boolean): Promise { + let config: any = { + user: env.databaseInfo.username, + host: host, + database: env.databaseInfo.defaultDbName, + password: env.databaseInfo.password, + port: port, + plugins: "gdbFailover", + failoverTimeoutMs: 250000, + activeHomeFailoverMode: "strict-writer", + inactiveHomeFailoverMode: "strict-writer", + enableTelemetry: true, + telemetryTracesBackend: "OTLP", + telemetryMetricsBackend: "OTLP" + }; + if (connectToProxy) { + config["clusterInstanceHostPattern"] = "?." + env.proxyDatabaseInfo.instanceEndpointSuffix; + } + config = DriverHelper.addDriverSpecificConfiguration(config, env.engine); + return config; +} + +describe("gdb failover", () => { + // Inherit shared failover tests with GDB-specific configuration + // This mirrors the Java pattern where GdbFailoverTest extends FailoverTest + describe( + "failover tests", + createFailoverTests({ + plugins: "gdbFailover", + getExtraConfig: () => ({ + // These settings mimic failover/failover2 plugin logic when connecting to non-GDB Aurora or RDS DB clusters. + activeHomeFailoverMode: "strict-writer", + inactiveHomeFailoverMode: "strict-writer" + }) + }) + ); + + // GDB-specific tests (overrides from Java GdbFailoverTest) + describe("gdb-specific tests", () => { + beforeEach(async () => { + logger.info(`Test started: ${expect.getState().currentTestName}`); + env = await TestEnvironment.getCurrent(); + + auroraTestUtility = new AuroraTestUtility(env.region); + driver = DriverHelper.getDriverForDatabaseEngine(env.engine); + initClientFunc = DriverHelper.getClient(driver); + await ProxyHelper.enableAllConnectivity(); + await TestEnvironment.verifyClusterStatus(); + + client = null; + await PluginManager.releaseResources(); + }, 1320000); + + afterEach(async () => { + await ProxyHelper.enableAllConnectivity(); + + if (client !== null) { + try { + await client.end(); + } catch (error) { + // pass + } + } + await PluginManager.releaseResources(); + logger.info(`Test finished: ${expect.getState().currentTestName}`); + }, 1320000); + + itIfNetworkOutages( + "reader failover with home-reader-or-writer mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "home-reader-or-writer"; + config["inactiveHomeFailoverMode"] = "home-reader-or-writer"; + + client = initClientFunc(config); + await client.connect(); + + await ProxyHelper.disableConnectivity(env.engine, initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + }, + 1320000 + ); + + itIfNetworkOutages( + "reader failover with strict-home-reader mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "strict-home-reader"; + config["inactiveHomeFailoverMode"] = "strict-home-reader"; + + client = initClientFunc(config); + await client.connect(); + + await ProxyHelper.disableConnectivity(env.engine, initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + + const currentConnectionId = await auroraTestUtility.queryInstanceId(client); + expect(await auroraTestUtility.isDbInstanceWriter(currentConnectionId)).toBe(false); + }, + 1320000 + ); + + itIfNetworkOutages( + "writer reelected with home-reader-or-writer mode", + async () => { + const initialWriterId = env.proxyDatabaseInfo.writerInstanceId; + const initialWriterHost = env.proxyDatabaseInfo.writerInstanceEndpoint; + const initialWriterPort = env.proxyDatabaseInfo.instanceEndpointPort; + + const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); + config["activeHomeFailoverMode"] = "home-reader-or-writer"; + config["inactiveHomeFailoverMode"] = "home-reader-or-writer"; + + client = initClientFunc(config); + await client.connect(); + + // Failover usually changes the writer instance, but we want to test re-election of the same writer, so we will + // simulate this by temporarily disabling connectivity to the writer. + await auroraTestUtility.simulateTemporaryFailure(initialWriterId!); + + await expect(async () => { + await auroraTestUtility.queryInstanceId(client); + }).rejects.toThrow(FailoverSuccessError); + }, + 1320000 + ); + }); +}); diff --git a/tests/integration/container/tests/fastest_response_strategy.test.ts b/tests/integration/container/tests/fastest_response_strategy.test.ts index ff44b922..9e7cdb98 100644 --- a/tests/integration/container/tests/fastest_response_strategy.test.ts +++ b/tests/integration/container/tests/fastest_response_strategy.test.ts @@ -89,8 +89,7 @@ describe("aurora fastest response strategy", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { diff --git a/tests/integration/container/tests/initial_connection_strategy.test.ts b/tests/integration/container/tests/initial_connection_strategy.test.ts index c540c72a..98ef8551 100644 --- a/tests/integration/container/tests/initial_connection_strategy.test.ts +++ b/tests/integration/container/tests/initial_connection_strategy.test.ts @@ -73,8 +73,7 @@ describe("aurora initial connection strategy", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); numReaders = env.databaseInfo.instances.length - 1; }, 1320000); diff --git a/tests/integration/container/tests/read_write_splitting.test.ts b/tests/integration/container/tests/read_write_splitting.test.ts index bb838105..f3af2378 100644 --- a/tests/integration/container/tests/read_write_splitting.test.ts +++ b/tests/integration/container/tests/read_write_splitting.test.ts @@ -34,8 +34,6 @@ import { ProxyHelper } from "./utils/proxy_helper"; import { logger } from "../../../../common/logutils"; import { TestEnvironmentFeatures } from "./utils/test_environment_features"; import { features, instanceCount } from "./config"; -import { RdsHostListProvider } from "../../../../common/lib/host_list_provider/rds_host_list_provider"; -import { PluginServiceImpl } from "../../../../common/lib/plugin_service"; const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && @@ -64,6 +62,8 @@ async function initConfig(host: string, port: number, connectToProxy: boolean, p port: port, plugins: plugins, enableTelemetry: true, + wrapperQueryTimeout: 10000, + wrapperConnectTimeout: 3000, telemetryTracesBackend: "OTLP", telemetryMetricsBackend: "OTLP" }; @@ -107,8 +107,7 @@ describe("aurora read write splitting", () => { await TestEnvironment.verifyAllInstancesHasRightState("available"); await TestEnvironment.verifyAllInstancesUp(); - RdsHostListProvider.clearAll(); - PluginServiceImpl.clearHostAvailabilityCache(); + await PluginManager.releaseResources(); }, 1320000); afterEach(async () => { @@ -609,9 +608,9 @@ describe("aurora read write splitting", () => { connectionsSet.add(client); } } finally { - for (const connection of connectionsSet) { + connectionsSet.forEach(async (connection) => { await connection.end(); - } + }); } }, 1000000 @@ -663,9 +662,9 @@ describe("aurora read write splitting", () => { connectionsSet.add(client); } } finally { - for (const connection of connectionsSet) { + connectionsSet.forEach(async (connection) => { await connection.end(); - } + }); } }, 1000000 diff --git a/tests/integration/container/tests/session_state.test.ts b/tests/integration/container/tests/session_state.test.ts index 5f06d420..015e8de9 100644 --- a/tests/integration/container/tests/session_state.test.ts +++ b/tests/integration/container/tests/session_state.test.ts @@ -119,7 +119,7 @@ describe("session state", () => { expect(autoCommit[0][0].autocommit).toEqual(1); expect(transactionIsolation[0][0].level).toEqual("REPEATABLE-READ"); - await client.getPluginService().setCurrentClient(newClient.targetClient); + await client.pluginService.setCurrentClient(newClient.targetClient); expect(client.targetClient).not.toEqual(targetClient); expect(client.targetClient).toEqual(newTargetClient); @@ -154,7 +154,7 @@ describe("session state", () => { expect(schema.rows[0]["search_path"]).not.toEqual("testSessionState"); expect(transactionIsolation.rows[0]["transaction_isolation"]).toEqual("read committed"); - await client.getPluginService().setCurrentClient(newClient.targetClient); + await client.pluginService.setCurrentClient(newClient.targetClient); expect(client.targetClient).not.toEqual(targetClient); expect(client.targetClient).toEqual(newTargetClient); diff --git a/tests/integration/container/tests/utils/aurora_test_utility.ts b/tests/integration/container/tests/utils/aurora_test_utility.ts index b0e0bad8..bdfb9c71 100644 --- a/tests/integration/container/tests/utils/aurora_test_utility.ts +++ b/tests/integration/container/tests/utils/aurora_test_utility.ts @@ -42,6 +42,7 @@ import { TestInstanceInfo } from "./test_instance_info"; import { TestEnvironmentInfo } from "./test_environment_info"; import { DatabaseEngine } from "./database_engine"; import { DatabaseEngineDeployment } from "./database_engine_deployment"; +import { ProxyHelper } from "./proxy_helper"; const instanceClass: string = "db.r5.large"; @@ -492,4 +493,38 @@ export class AuroraTestUtility { logger.debug("switchoverBlueGreenDeployment request is sent."); } } + + async simulateTemporaryFailure(instanceName: string, delayMs: number = 0, failureDurationMs: number = 5000): Promise { + const env = await TestEnvironment.getCurrent(); + const deployment = env.deployment; + const clusterEndpoint = env.proxyDatabaseInfo.clusterEndpoint; + const clusterReadOnlyEndpoint = env.proxyDatabaseInfo.clusterReadOnlyEndpoint; + + (async () => { + try { + if (delayMs > 0) { + await sleep(delayMs); + } + + await ProxyHelper.disableConnectivity(env.engine, instanceName); + + if (deployment === DatabaseEngineDeployment.RDS_MULTI_AZ_CLUSTER) { + await ProxyHelper.disableConnectivity(env.engine, clusterEndpoint); + await ProxyHelper.disableConnectivity(env.engine, clusterReadOnlyEndpoint); + } + + await sleep(failureDurationMs); + + await ProxyHelper.enableConnectivity(instanceName); + if (deployment === DatabaseEngineDeployment.RDS_MULTI_AZ_CLUSTER) { + await ProxyHelper.enableConnectivity(clusterEndpoint); + await ProxyHelper.enableConnectivity(clusterReadOnlyEndpoint); + } + } catch (e: any) { + logger.error(`Error during simulateTemporaryFailure: ${e.message}`); + } + })(); + + await sleep(500); + } } diff --git a/tests/integration/container/tests/utils/test_environment.ts b/tests/integration/container/tests/utils/test_environment.ts index cc2d52e9..459229fa 100644 --- a/tests/integration/container/tests/utils/test_environment.ts +++ b/tests/integration/container/tests/utils/test_environment.ts @@ -37,6 +37,7 @@ import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions"; import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics"; import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc"; import { logger } from "../../../../../common/logutils"; +import { RdsUtils } from "../../../../../common/lib/utils/rds_utils"; import pkgPg from "pg"; import { ConnectionOptions, createConnection } from "mysql2/promise"; import { readFileSync } from "fs"; @@ -238,6 +239,14 @@ export class TestEnvironment { await TestEnvironment.initProxies(env); } + // Helps to eliminate problem with proxied endpoints. + RdsUtils.setPrepareHostFunc((host: string) => { + if (host.endsWith(".proxied")) { + return host.substring(0, host.length - ".proxied".length); + } + return host; + }); + const contextManager = new AsyncHooksContextManager(); contextManager.enable(); context.setGlobalContextManager(contextManager); diff --git a/tests/plugin_benchmarks.ts b/tests/plugin_benchmarks.ts index d28f031a..edcd56e7 100644 --- a/tests/plugin_benchmarks.ts +++ b/tests/plugin_benchmarks.ts @@ -25,8 +25,8 @@ import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/ import { HostInfoBuilder } from "../common/lib"; import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; import { NullTelemetryFactory } from "../common/lib/utils/telemetry/null_telemetry_factory"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { ConnectionProviderManager } from "../common/lib/connection_provider_manager"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockPluginService = mock(PluginServiceImpl); @@ -47,8 +47,8 @@ when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); when(mockPluginService.getDriverDialect()).thenReturn(mockDialect); const connectionString = "my.domain.com"; -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); function getProps(plugins: string) { const props = new Map(); @@ -59,7 +59,7 @@ function getProps(plugins: string) { function getPluginManager(props: Map) { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, props, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/plugin_manager_benchmarks.ts b/tests/plugin_manager_benchmarks.ts index 1b84b5bc..801a4f88 100644 --- a/tests/plugin_manager_benchmarks.ts +++ b/tests/plugin_manager_benchmarks.ts @@ -16,7 +16,6 @@ import { add, complete, configure, cycle, save, suite } from "benny"; import { ConnectionPlugin, ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { instance, mock, when } from "ts-mockito"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; import { PluginServiceImpl } from "../common/lib/plugin_service"; @@ -32,6 +31,7 @@ import { ConnectionPluginFactory } from "../common/lib/plugin_factory"; import { DefaultPlugin } from "../common/lib/plugins/default_plugin"; import { AwsPGClient } from "../pg/lib"; import { ConfigurationProfileBuilder } from "../common/lib/profile/configuration_profile_builder"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockHostListProviderService = mock(); @@ -43,8 +43,8 @@ when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); when(mockPluginService.getCurrentClient()).thenReturn(mockClient); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsWithNoPlugins = new Map(); const propsWithPlugins = new Map(); @@ -53,7 +53,7 @@ WrapperProperties.PLUGINS.set(propsWithNoPlugins, ""); function getPluginManagerWithPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() @@ -62,7 +62,7 @@ function getPluginManagerWithPlugins() { function getPluginManagerWithNoPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithNoPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/plugin_manager_telemetry_benchmarks.ts b/tests/plugin_manager_telemetry_benchmarks.ts index 8420c8ea..3a0f19bb 100644 --- a/tests/plugin_manager_telemetry_benchmarks.ts +++ b/tests/plugin_manager_telemetry_benchmarks.ts @@ -16,7 +16,6 @@ import { add, complete, configure, cycle, save, suite } from "benny"; import { ConnectionPlugin, ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { instance, mock, when } from "ts-mockito"; import { SimpleHostAvailabilityStrategy } from "../common/lib/host_availability/simple_host_availability_strategy"; import { PluginService, PluginServiceImpl } from "../common/lib/plugin_service"; @@ -44,6 +43,7 @@ import { ConnectionPluginFactory } from "../common/lib/plugin_factory"; import { ConfigurationProfileBuilder } from "../common/lib/profile/configuration_profile_builder"; import { AwsPGClient } from "../pg/lib"; import { resourceFromAttributes } from "@opentelemetry/resources"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockHostListProviderService = mock(); @@ -55,8 +55,8 @@ when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); when(mockPluginService.getDriverDialect()).thenReturn(new NodePostgresDriverDialect()); when(mockPluginService.getCurrentClient()).thenReturn(mockClient); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsWithNoPlugins = new Map(); const propsWithPlugins = new Map(); @@ -80,7 +80,7 @@ async function createPlugins(numPlugins: number, pluginService: PluginService, c function getPluginManagerWithPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory @@ -89,7 +89,7 @@ function getPluginManagerWithPlugins() { function getPluginManagerWithNoPlugins() { return new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsWithNoPlugins, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory diff --git a/tests/plugin_telemetry_benchmarks.ts b/tests/plugin_telemetry_benchmarks.ts index 04d791e5..b6c391a5 100644 --- a/tests/plugin_telemetry_benchmarks.ts +++ b/tests/plugin_telemetry_benchmarks.ts @@ -17,7 +17,6 @@ import { anything, instance, mock, when } from "ts-mockito"; import { ConnectionProvider, HostInfoBuilder, PluginManager } from "../common/lib"; import { PluginServiceImpl } from "../common/lib/plugin_service"; -import { PluginServiceManagerContainer } from "../common/lib/plugin_service_manager_container"; import { WrapperProperties } from "../common/lib/wrapper_property"; import { add, complete, configure, cycle, save, suite } from "benny"; import { TestConnectionWrapper } from "./testplugin/test_connection_wrapper"; @@ -41,6 +40,7 @@ import { PgClientWrapper } from "../common/lib/pg_client_wrapper"; import { DriverDialect } from "../common/lib/driver_dialect/driver_dialect"; import { NodePostgresDriverDialect } from "../pg/lib/dialect/node_postgres_driver_dialect"; import { resourceFromAttributes } from "@opentelemetry/resources"; +import { FullServicesContainerImpl } from "../common/lib/utils/full_services_container"; const mockConnectionProvider = mock(); const mockPluginService = mock(PluginServiceImpl); @@ -60,8 +60,8 @@ when(mockPluginService.getCurrentClient()).thenReturn(mockClientWrapper.client); when(mockPluginService.getDriverDialect()).thenReturn(mockDialect); const connectionString = "my.domain.com"; -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); -pluginServiceManagerContainer.pluginService = instance(mockPluginService); +const servicesContainer = mock(FullServicesContainerImpl); +when(servicesContainer.pluginService).thenReturn(instance(mockPluginService)); const propsExecute = new Map(); const propsReadWrite = new Map(); @@ -84,19 +84,19 @@ WrapperProperties.TELEMETRY_TRACES_BACKEND.set(propsReadWrite, "OTLP"); WrapperProperties.TELEMETRY_TRACES_BACKEND.set(props, "OTLP"); const pluginManagerExecute = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsExecute, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory ); const pluginManagerReadWrite = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, propsReadWrite, new ConnectionProviderManager(instance(mockConnectionProvider), null), telemetryFactory ); const pluginManager = new PluginManager( - pluginServiceManagerContainer, + servicesContainer, props, new ConnectionProviderManager(instance(mockConnectionProvider), null), new NullTelemetryFactory() diff --git a/tests/testplugin/benchmark_plugin_factory.ts b/tests/testplugin/benchmark_plugin_factory.ts index f1cd65e6..d4101cba 100644 --- a/tests/testplugin/benchmark_plugin_factory.ts +++ b/tests/testplugin/benchmark_plugin_factory.ts @@ -14,13 +14,14 @@ limitations under the License. */ -import { ConnectionPlugin, PluginService, AwsWrapperError } from "../../index"; +import { ConnectionPlugin, AwsWrapperError } from "../../index"; import { Messages } from "../../common/lib/utils/messages"; import { BenchmarkPlugin } from "./benchmark_plugin"; import { ConnectionPluginFactory } from "../../common/lib/plugin_factory"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; export class BenchmarkPluginFactory extends ConnectionPluginFactory { - async getInstance(pluginService: PluginService, properties: object): Promise { + async getInstance(servicesContainer: FullServicesContainer, properties: object): Promise { try { return new BenchmarkPlugin(); } catch (error: any) { diff --git a/tests/unit/aurora_initial_connection_strategy_plugin.test.ts b/tests/unit/aurora_initial_connection_strategy_plugin.test.ts index ea07276e..920057ec 100644 --- a/tests/unit/aurora_initial_connection_strategy_plugin.test.ts +++ b/tests/unit/aurora_initial_connection_strategy_plugin.test.ts @@ -109,7 +109,7 @@ describe("Aurora initial connection strategy plugin", () => { when(mockPluginService.connect(anything(), anything(), anything())).thenResolve(writerClient); expect(await plugin.connect(hostInfo, props, true, mockFunc)).toBe(writerClient); - verify(mockPluginService.forceRefreshHostList(writerClient)).never(); + verify(mockPluginService.forceRefreshHostList()).never(); }); it("test reader - not found", async () => { @@ -128,7 +128,7 @@ describe("Aurora initial connection strategy plugin", () => { when(mockPluginService.getHostInfoByStrategy(anything(), anything())).thenReturn(instance(mockReaderHostInfo)); expect(await plugin.connect(hostInfo, props, true, mockFunc)).toBe(readerClient); - verify(mockPluginService.forceRefreshHostList(readerClient)).never(); + verify(mockPluginService.forceRefreshHostList()).never(); }); it("test reader - resolves to writer", async () => { diff --git a/tests/unit/batching_event_publisher.test.ts b/tests/unit/batching_event_publisher.test.ts new file mode 100644 index 00000000..bb079c5c --- /dev/null +++ b/tests/unit/batching_event_publisher.test.ts @@ -0,0 +1,141 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { BatchingEventPublisher } from "../../common/lib/utils/events/batching_event_publisher"; +import { DataAccessEvent } from "../../common/lib/utils/events/data_access_event"; +import { Event, EventSubscriber } from "../../common/lib/utils/events/event"; + +class TestableEventPublisher extends BatchingEventPublisher { + constructor() { + super(0); // Pass 0 to avoid starting the interval + } + + protected initPublishingInterval(_messageIntervalMs: number): void { + // Do nothing. + } + + get subscriberCount(): number { + return this.subscribersMap.size; + } + + get pendingEventCount(): number { + return this.pendingEvents.size; + } + + async triggerSendMessages(): Promise { + await this.sendMessages(); + } +} + +// A simple class to use as the dataClass in DataAccessEvent +class TestDataClass {} + +describe("BatchingEventPublisher", () => { + let publisher: TestableEventPublisher; + let mockSubscriber: EventSubscriber; + let processEventCalls: Event[]; + + beforeEach(() => { + publisher = new TestableEventPublisher(); + processEventCalls = []; + mockSubscriber = { + processEvent: async (event: Event) => { + processEventCalls.push(event); + } + }; + }); + + afterEach(() => { + publisher.releaseResources(); + }); + + it("should publish events to subscribers and deduplicate", async () => { + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.subscribe(mockSubscriber, eventSubscriptions); + expect(publisher.subscriberCount).toBe(1); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + publisher.publish(event); + + await publisher.triggerSendMessages(); + + expect(publisher.pendingEventCount).toBe(0); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls[0]).toBe(event); + + publisher.unsubscribe(mockSubscriber, eventSubscriptions); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(publisher.pendingEventCount).toBe(0); + + expect(processEventCalls.length).toBe(1); + }); + + it("should deliver immediate events synchronously", () => { + const immediateEvent: Event = { + isImmediateDelivery: true + }; + + const eventSubscriptions = new Set([immediateEvent.constructor as new (...args: any[]) => Event]); + publisher.subscribe(mockSubscriber, eventSubscriptions); + + publisher.publish(immediateEvent); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls[0]).toBe(immediateEvent); + + expect(publisher.pendingEventCount).toBe(0); + }); + + it("should not deliver events to unsubscribed subscribers", async () => { + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.unsubscribe(mockSubscriber, eventSubscriptions); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(processEventCalls.length).toBe(0); + }); + + it("should handle multiple subscribers", async () => { + const processEventCalls2: Event[] = []; + const mockSubscriber2: EventSubscriber = { + processEvent: async (event: Event) => { + processEventCalls2.push(event); + } + }; + + const eventSubscriptions = new Set([DataAccessEvent]); + + publisher.subscribe(mockSubscriber, eventSubscriptions); + publisher.subscribe(mockSubscriber2, eventSubscriptions); + + const event = new DataAccessEvent(TestDataClass, "key"); + publisher.publish(event); + await publisher.triggerSendMessages(); + + expect(processEventCalls.length).toBe(1); + expect(processEventCalls2.length).toBe(1); + }); +}); diff --git a/tests/unit/connection_plugin_chain_builder.test.ts b/tests/unit/connection_plugin_chain_builder.test.ts index ee152a19..b9cb9c78 100644 --- a/tests/unit/connection_plugin_chain_builder.test.ts +++ b/tests/unit/connection_plugin_chain_builder.test.ts @@ -30,23 +30,33 @@ import { ConnectionProviderManager } from "../../common/lib/connection_provider_ import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { AbstractConnectionPlugin } from "../../common/lib/abstract_connection_plugin"; import { ConnectionPluginFactory } from "../../common/lib/plugin_factory"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockPluginServiceInstance: PluginService = instance(mockPluginService); const mockDefaultConnProvider: ConnectionProvider = mock(DriverConnectionProvider); const mockEffectiveConnProvider: ConnectionProvider = mock(DriverConnectionProvider); +const mockServicesContainer: FullServicesContainer = { + pluginService: mockPluginServiceInstance, + telemetryFactory: new NullTelemetryFactory() +} as unknown as FullServicesContainer; + describe("testConnectionPluginChainBuilder", () => { beforeAll(() => { when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); }); + afterEach(async () => { + await PluginManager.releaseResources(); + }); + it.each([["iam,staleDns,failover"], ["iam, staleDns, failover"]])("sort plugins", async (plugins) => { const props = new Map(); props.set(WrapperProperties.PLUGINS.name, plugins); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -65,7 +75,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.AUTO_SORT_PLUGIN_ORDER.name, false); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -84,7 +94,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "executeTime,connectTime,iam"); let result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -99,7 +109,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "iam,executeTime,connectTime,failover"); result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -120,7 +130,7 @@ describe("testConnectionPluginChainBuilder", () => { props.set(WrapperProperties.PLUGINS.name, "test"); const result = await ConnectionPluginChainBuilder.getPlugins( - mockPluginServiceInstance, + mockServicesContainer, props, new ConnectionProviderManager(mockDefaultConnProvider, mockEffectiveConnProvider), null @@ -133,8 +143,8 @@ describe("testConnectionPluginChainBuilder", () => { }); class TestPluginFactory extends ConnectionPluginFactory { - async getInstance(pluginService: PluginService, properties: Map): Promise { - return new TestPlugin(pluginService, properties); + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + return new TestPlugin(servicesContainer.pluginService, properties); } } diff --git a/tests/unit/database_dialect.test.ts b/tests/unit/database_dialect.test.ts index 36b55a71..b8ff3472 100644 --- a/tests/unit/database_dialect.test.ts +++ b/tests/unit/database_dialect.test.ts @@ -23,7 +23,6 @@ import { RdsPgDatabaseDialect } from "../../pg/lib/dialect/rds_pg_database_diale import { DatabaseDialect, DatabaseType } from "../../common/lib/database_dialect/database_dialect"; import { DatabaseDialectCodes } from "../../common/lib/database_dialect/database_dialect_codes"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; import { AwsPGClient } from "../../pg/lib"; import { WrapperProperties } from "../../common/lib/wrapper_property"; import { HostInfoBuilder } from "../../common/lib/host_info_builder"; @@ -31,10 +30,18 @@ import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availabili import { ClientWrapper } from "../../common/lib/client_wrapper"; import { RdsMultiAZClusterMySQLDatabaseDialect } from "../../mysql/lib/dialect/rds_multi_az_mysql_database_dialect"; import { RdsMultiAZClusterPgDatabaseDialect } from "../../pg/lib/dialect/rds_multi_az_pg_database_dialect"; +import { GlobalAuroraMySQLDatabaseDialect } from "../../mysql/lib/dialect/global_aurora_mysql_database_dialect"; +import { GlobalAuroraPgDatabaseDialect } from "../../pg/lib/dialect/global_aurora_pg_database_dialect"; import { DatabaseDialectManager } from "../../common/lib/database_dialect/database_dialect_manager"; import { NodePostgresDriverDialect } from "../../pg/lib/dialect/node_postgres_driver_dialect"; import { mock } from "ts-mockito"; import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; +import { StorageService } from "../../common/lib/utils/storage/storage_service"; +import { ConnectionProvider } from "../../common/lib"; +import { TelemetryFactory } from "../../common/lib/utils/telemetry/telemetry_factory"; +import { MonitorService } from "../../common/lib/utils/monitoring/monitor_service"; +import { FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; +import { EventPublisher } from "../../common/lib/utils/events/event"; const LOCALHOST = "localhost"; const RDS_DATABASE = "database-1.xyz.us-east-2.rds.amazonaws.com"; @@ -44,14 +51,16 @@ const mysqlDialects: Map = new Map([ [DatabaseDialectCodes.MYSQL, new MySQLDatabaseDialect()], [DatabaseDialectCodes.RDS_MYSQL, new RdsMySQLDatabaseDialect()], [DatabaseDialectCodes.AURORA_MYSQL, new AuroraMySQLDatabaseDialect()], - [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()] + [DatabaseDialectCodes.RDS_MULTI_AZ_MYSQL, new RdsMultiAZClusterMySQLDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_MYSQL, new GlobalAuroraMySQLDatabaseDialect()] ]); const pgDialects: Map = new Map([ [DatabaseDialectCodes.PG, new PgDatabaseDialect()], [DatabaseDialectCodes.RDS_PG, new RdsPgDatabaseDialect()], [DatabaseDialectCodes.AURORA_PG, new AuroraPgDatabaseDialect()], - [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()] + [DatabaseDialectCodes.RDS_MULTI_AZ_PG, new RdsMultiAZClusterPgDatabaseDialect()], + [DatabaseDialectCodes.GLOBAL_AURORA_PG, new GlobalAuroraPgDatabaseDialect()] ]); const MYSQL_QUERY = "SHOW VARIABLES LIKE 'version_comment'"; @@ -180,7 +189,13 @@ const expectedDialectMapping: Map = ne ] ]); -const pluginServiceManagerContainer = new PluginServiceManagerContainer(); +const fullServicesContainer = new FullServicesContainerImpl( + mock(), + mock(), + mock(), + mock(), + mock() +); const mockClient = new AwsPGClient({}); const mockDriverDialect = mock(NodePostgresDriverDialect); @@ -275,14 +290,9 @@ describe("test database dialects", () => { }).build(); const mockClientWrapper: ClientWrapper = new PgClientWrapper(mockTargetClient, currentHostInfo, new Map()); - const pluginService = new PluginServiceImpl( - pluginServiceManagerContainer, - mockClient, - databaseType, - expectedDialect!.dialects, - props, - mockDriverDialect - ); + const pluginService = new PluginServiceImpl(fullServicesContainer, mockClient, databaseType, expectedDialect!.dialects, props, mockDriverDialect); + fullServicesContainer.hostListProviderService = pluginService; + fullServicesContainer.pluginService = pluginService; await pluginService.updateDialect(mockClientWrapper); expect(pluginService.getDialect()).toBe(expectedDialectClass); }); diff --git a/tests/unit/failover2_plugin.test.ts b/tests/unit/failover2_plugin.test.ts index b36a2baf..8db14ad2 100644 --- a/tests/unit/failover2_plugin.test.ts +++ b/tests/unit/failover2_plugin.test.ts @@ -40,6 +40,7 @@ import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; import { DriverDialect } from "../../common/lib/driver_dialect/driver_dialect"; import { Failover2Plugin } from "../../common/lib/plugins/failover2/failover2_plugin"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -57,7 +58,10 @@ const properties: Map = new Map(); let plugin: Failover2Plugin; function initializePlugin(mockPluginServiceInstance: PluginService): void { - plugin = new Failover2Plugin(mockPluginServiceInstance, properties, new RdsUtils()); + const mockContainer = { + pluginService: mockPluginServiceInstance + } as unknown as FullServicesContainer; + plugin = new Failover2Plugin(mockContainer, properties, new RdsUtils()); } describe("reader failover handler", () => { diff --git a/tests/unit/failover_plugin.test.ts b/tests/unit/failover_plugin.test.ts index 0cbbe34d..62750090 100644 --- a/tests/unit/failover_plugin.test.ts +++ b/tests/unit/failover_plugin.test.ts @@ -17,7 +17,6 @@ import { AwsClient } from "../../common/lib/aws_client"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { HostInfoBuilder } from "../../common/lib/host_info_builder"; -import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; import { FailoverMode } from "../../common/lib/plugins/failover/failover_mode"; import { FailoverPlugin } from "../../common/lib/plugins/failover/failover_plugin"; @@ -44,6 +43,7 @@ import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { HostChangeOptions } from "../../common/lib/host_change_options"; import { Messages } from "../../common/lib/utils/messages"; +import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); diff --git a/tests/unit/iam_authentication_plugin.test.ts b/tests/unit/iam_authentication_plugin.test.ts index f2ee620f..58d7ad7b 100644 --- a/tests/unit/iam_authentication_plugin.test.ts +++ b/tests/unit/iam_authentication_plugin.test.ts @@ -110,6 +110,7 @@ describe("testIamAuth", () => { afterEach(() => { reset(spyIamAuthUtils); + PluginManager.releaseResources(); }); it("testPostgresConnectValidTokenInCache", async () => { diff --git a/tests/unit/notification_pipeline.test.ts b/tests/unit/notification_pipeline.test.ts index cb1d315e..0d4d95c5 100644 --- a/tests/unit/notification_pipeline.test.ts +++ b/tests/unit/notification_pipeline.test.ts @@ -16,7 +16,6 @@ import { HostChangeOptions } from "../../common/lib/host_change_options"; import { OldConnectionSuggestionAction } from "../../common/lib/old_connection_suggestion_action"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; import { DefaultPlugin } from "../../common/lib/plugins/default_plugin"; import { instance, mock } from "ts-mockito"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; @@ -24,6 +23,7 @@ import { DriverConnectionProvider } from "../../common/lib/driver_connection_pro import { ConnectionProviderManager } from "../../common/lib/connection_provider_manager"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { PluginManager } from "../../common/lib"; +import { FullServicesContainer, FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; class TestPlugin extends DefaultPlugin { counter: number = 0; @@ -43,7 +43,7 @@ class TestPlugin extends DefaultPlugin { } } -const container: PluginServiceManagerContainer = new PluginServiceManagerContainer(); +const container: FullServicesContainer = mock(FullServicesContainerImpl); const props: Map = mock(Map); const hostListChanges: Map> = mock(Map>); const connectionChanges: Set = mock(Set); @@ -64,6 +64,10 @@ describe("notificationPipelineTest", () => { pluginManager["_plugins"] = [plugin]; }); + afterEach(async () => { + await PluginManager.releaseResources(); + }); + it("test_notifyConnectionChanged", async () => { const result: Set = await pluginManager.notifyConnectionChanged(connectionChanges, null); expect(plugin.counter).toBe(1); diff --git a/tests/unit/plugin_service.test.ts b/tests/unit/plugin_service.test.ts index 2d6b145b..2b14f31c 100644 --- a/tests/unit/plugin_service.test.ts +++ b/tests/unit/plugin_service.test.ts @@ -26,7 +26,7 @@ import { HostInfoBuilder } from "../../common/lib"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { DatabaseDialectCodes } from "../../common/lib/database_dialect/database_dialect_codes"; import { AwsClient } from "../../common/lib/aws_client"; -import { PluginServiceManagerContainer } from "../../common/lib/plugin_service_manager_container"; +import { FullServicesContainer, FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; import { AllowedAndBlockedHosts } from "../../common/lib/allowed_and_blocked_hosts"; import { DatabaseType } from "../../common/lib/database_dialect/database_dialect"; @@ -64,7 +64,7 @@ let pluginService: TestPluginService; describe("testCustomEndpoint", () => { beforeEach(() => { pluginService = new TestPluginService( - new PluginServiceManagerContainer(), + mock(FullServicesContainerImpl), mockAwsClient, DatabaseType.MYSQL, knownDialectsByCode, diff --git a/tests/unit/rds_host_list_provider.test.ts b/tests/unit/rds_host_list_provider.test.ts index 99a91d3d..6f675a8c 100644 --- a/tests/unit/rds_host_list_provider.test.ts +++ b/tests/unit/rds_host_list_provider.test.ts @@ -14,11 +14,10 @@ limitations under the License. */ -import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { AwsClient } from "../../common/lib/aws_client"; -import { AwsWrapperError, HostInfo, HostInfoBuilder } from "../../common/lib"; +import { AwsWrapperError, HostInfo, HostInfoBuilder, PluginManager } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { ConnectionUrlParser } from "../../common/lib/utils/connection_url_parser"; import { AwsPGClient } from "../../pg/lib"; @@ -30,13 +29,17 @@ import { PgClientWrapper } from "../../common/lib/pg_client_wrapper"; import { CoreServicesContainer } from "../../common/lib/utils/core_services_container"; import { StorageService } from "../../common/lib/utils/storage/storage_service"; import { Topology } from "../../common/lib/host_list_provider/topology"; +import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; import { TopologyQueryResult, TopologyUtils } from "../../common/lib/host_list_provider/topology_utils"; +import { FullServicesContainerImpl } from "../../common/lib/utils/full_services_container"; const mockClient: AwsClient = mock(AwsPGClient); const mockDialect: AuroraPgDatabaseDialect = mock(AuroraPgDatabaseDialect); +const mockServiceContainer: FullServicesContainerImpl = mock(FullServicesContainerImpl); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const connectionUrlParser: ConnectionUrlParser = new PgConnectionUrlParser(); const mockTopologyUtils: TopologyUtils = mock(TopologyUtils); +const storageService: StorageService = CoreServicesContainer.getInstance().storageService; const hosts: HostInfo[] = [ createHost({ @@ -56,20 +59,15 @@ const currentHostInfo = createHost({ }); const clientWrapper: ClientWrapper = new PgClientWrapper(undefined, currentHostInfo, new Map()); - const mockClientWrapper: ClientWrapper = mock(clientWrapper); -const storageService: StorageService = CoreServicesContainer.getInstance().getStorageService(); - -const defaultRefreshRateNano: number = 5 * 1_000_000_000; - function createHost(config: any): HostInfo { const info = new HostInfoBuilder(config); return info.build(); } function getRdsHostListProvider(originalHost: string): RdsHostListProvider { - const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockTopologyUtils), instance(mockPluginService)); + const provider = new RdsHostListProvider(new Map(), originalHost, instance(mockTopologyUtils), instance(mockServiceContainer)); provider.init(); return provider; } @@ -84,11 +82,13 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockClient.targetClient).thenReturn(mockClientWrapper); when(mockPluginService.getHostInfoBuilder()).thenReturn(new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() })); + when(mockServiceContainer.hostListProviderService).thenReturn(instance(mockPluginService)); + when(mockServiceContainer.pluginService).thenReturn(instance(mockPluginService)); + when(mockServiceContainer.storageService).thenReturn(storageService); }); - afterEach(() => { - RdsHostListProvider.clearAll(); - CoreServicesContainer.getInstance().getStorageService().clearAll(); + afterEach(async () => { + await PluginManager.releaseResources(); reset(mockDialect); reset(mockClientWrapper); @@ -103,7 +103,7 @@ describe("testRdsHostListProvider", () => { const expected: HostInfo[] = hosts; storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); @@ -117,7 +117,6 @@ describe("testRdsHostListProvider", () => { when(mockPluginService.isClientValid(anything())).thenResolve(true); - storageService.set(rdsHostListProvider.clusterId, new Topology(hosts)); const newHosts: HostInfo[] = [ createHost({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy(), @@ -126,13 +125,12 @@ describe("testRdsHostListProvider", () => { ]; when(mockClient.isValid()).thenResolve(true); - when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve(newHosts)); + when(mockPluginService.isDialectConfirmed()).thenReturn(true); + when((spiedProvider as any).forceRefreshMonitor(anything(), anything())).thenReturn(Promise.resolve(newHosts)); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(1); expect(result.hosts).toEqual(newHosts); - - verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); }); it("testGetTopology_noForceUpdate_queryReturnsEmptyHostList", async () => { @@ -145,7 +143,7 @@ describe("testRdsHostListProvider", () => { storageService.set(rdsHostListProvider.clusterId, new Topology(expected)); when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, false); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts.length).toEqual(2); expect(result.hosts).toEqual(expected); verify(spiedProvider.getCurrentTopology(anything(), anything())).atMost(1); @@ -165,7 +163,7 @@ describe("testRdsHostListProvider", () => { when(spiedProvider.getCurrentTopology(mockClientWrapper, anything())).thenReturn(Promise.resolve([])); - const result = await rdsHostListProvider.getTopology(mockClientWrapper, true); + const result = await rdsHostListProvider.getTopology(); expect(result.hosts).toBeTruthy(); for (let i = 0; i < result.hosts.length; i++) { expect(result.hosts[i].equals(initialHosts[i])).toBeTruthy(); diff --git a/tests/unit/read_write_splitting.test.ts b/tests/unit/read_write_splitting.test.ts index bb291e50..612435e1 100644 --- a/tests/unit/read_write_splitting.test.ts +++ b/tests/unit/read_write_splitting.test.ts @@ -252,7 +252,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(readerHost1); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); + when(mockHostListProviderService.isDynamicHostListProvider()).thenReturn(true); when(mockHostListProviderService.getHostListProvider()).thenReturn(mockHostListProviderInstance); const target = new TestReadWriteSplitting( @@ -405,7 +405,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHostUnknownRole); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); - when(mockHostListProviderService.isStaticHostListProvider()).thenReturn(false); + when(mockHostListProviderService.isDynamicHostListProvider()).thenReturn(true); const target = new TestReadWriteSplitting( mockPluginServiceInstance, diff --git a/tests/unit/sliding_expiration_cache.test.ts b/tests/unit/sliding_expiration_cache.test.ts index 181924f9..d9f2808b 100644 --- a/tests/unit/sliding_expiration_cache.test.ts +++ b/tests/unit/sliding_expiration_cache.test.ts @@ -15,8 +15,8 @@ */ import { SlidingExpirationCache } from "../../common/lib/utils/sliding_expiration_cache"; -import { convertMsToNanos, convertNanosToMs, sleep } from "../../common/lib/utils/utils"; import { SlidingExpirationCacheWithCleanupTask } from "../../common/lib/utils/sliding_expiration_cache_with_cleanup_task"; +import { convertMsToNanos, convertNanosToMs, sleep } from "../../common/lib/utils/utils"; class DisposableItem { shouldDispose: boolean; @@ -127,7 +127,7 @@ describe("test_sliding_expiration_cache", () => { expect(item2.disposed).toEqual(true); }); - it("test async cleanup thread", async () => { + it("test async cleanup task", async () => { const cleanupIntervalNanos = BigInt(300_000_000); // .3 seconds const disposeMs = 1000; const target = new SlidingExpirationCacheWithCleanupTask( diff --git a/tests/unit/stale_dns_helper.test.ts b/tests/unit/stale_dns_helper.test.ts index 086b75b8..017b4c65 100644 --- a/tests/unit/stale_dns_helper.test.ts +++ b/tests/unit/stale_dns_helper.test.ts @@ -34,14 +34,6 @@ const mockHostListProviderService = mock(); const props: Map = new Map(); const writerInstance = new HostInfo("writer-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const writerCluster = new HostInfo("my-cluster.cluster-XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const writerClusterInvalidClusterInetAddress = new HostInfo("my-cluster.cluster-invalid.us-west-2.rds.amazonaws.com", 1234, HostRole.WRITER); -const readerA = new HostInfo("reader-a-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.READER, HostAvailability.AVAILABLE); -const readerB = new HostInfo("reader-b-host.XYZ.us-west-2.rds.amazonaws.com", 1234, HostRole.READER, HostAvailability.AVAILABLE); - -const clusterHostList = [writerCluster, readerA, readerB]; -const readerHostList = [readerA, readerB]; -const instanceHostList = [writerInstance, readerA, readerB]; const mockInitialConn = mock(AwsClient); const mockHostInfo = mock(HostInfo); @@ -87,208 +79,6 @@ describe("test_stale_dns_helper", () => { expect(returnConn).toBe(mockInitialClientWrapper); }); - it("test_get_verified_connection_cluster_inet_address_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - when(target.lookupResult(anything())).thenReturn(); - - const returnConn = await targetInstance.getVerifiedConnection( - writerClusterInvalidClusterInetAddress.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockInitialClientWrapper).toBe(returnConn); - expect(mockConnectFunc).toHaveBeenCalled(); - }); - - it("test_get_verified_connection__no_writer_hostinfo", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - when(mockPluginService.getHosts()).thenReturn(readerHostList); - when(mockPluginService.getAllHosts()).thenReturn(readerHostList); - - when(mockPluginService.getCurrentHostInfo()).thenReturn(readerA); - - const lookupAddress = { address: "2.2.2.2", family: 0 }; - when(target.lookupResult(anything())).thenResolve(lookupAddress); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(readerA.role).toBe(HostRole.READER); - verify(mockPluginService.forceRefreshHostList(anything())).once(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_rds_cluster_dns_true", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - - const lookupAddress = { address: "5.5.5.5", family: 0 }; - when(target.lookupResult(anything())).thenResolve(lookupAddress); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - verify(mockPluginService.refreshHostList(anything())).once(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_host_address_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(instanceHostList); - when(mockPluginService.getAllHosts()).thenReturn(instanceHostList); - - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - }); - - it("test_get_verified_connection__writer_host_info_none", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(readerHostList); - when(mockPluginService.getAllHosts()).thenReturn(readerHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - verify(mockPluginService.connect(anything(), anything())).never(); - }); - - it("test_get_verified_connection__writer_host_address_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - when(mockPluginService.getHosts()).thenReturn(instanceHostList); - when(mockPluginService.getAllHosts()).thenReturn(instanceHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "5.5.5.5", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockConnectFunc).toHaveBeenCalled(); - expect(mockInitialClientWrapper).toBe(returnConn); - verify(mockPluginService.connect(anything(), anything())).never(); - }); - - it("test_get_verified_connection__writer_host_address_not_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - targetInstance["writerHostInfo"] = writerCluster; - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "8.8.8.8", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - false, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - expect(mockInitialConn.targetClient).not.toBe(returnConn); - expect(mockConnectFunc).toHaveBeenCalled(); - verify(mockPluginService.connect(anything(), anything())).once(); - }); - - it("test_get_verified_connection__initial_connection_writer_host_address_not_equals_cluster_inet_address", async () => { - const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); - const targetInstance = instance(target); - - when(mockPluginService.getHosts()).thenReturn(clusterHostList); - when(mockPluginService.getAllHosts()).thenReturn(clusterHostList); - const mockHostListProviderServiceInstance = instance(mockHostListProviderService); - targetInstance["writerHostInfo"] = writerCluster; - when(mockHostListProviderService.getInitialConnectionHostInfo()).thenReturn(writerCluster); - - const firstCall = { address: "5.5.5.5", family: 0 }; - const secondCall = { address: "8.8.8.8", family: 0 }; - - when(target.lookupResult(anything())).thenResolve(firstCall, secondCall); - - const returnConn = await targetInstance.getVerifiedConnection( - writerCluster.host, - true, - mockHostListProviderServiceInstance, - props, - mockConnectFunc - ); - - verify(mockPluginService.connect(anything(), anything())).once(); - expect(targetInstance["writerHostInfo"]).toBe(mockHostListProviderServiceInstance.getInitialConnectionHostInfo()); - expect(mockInitialConn.targetClient).not.toBe(returnConn); - }); - it("test_notify_host_list_changed", () => { const target: StaleDnsHelper = spy(new StaleDnsHelper(instance(mockPluginService))); const targetInstance = instance(target); @@ -301,6 +91,5 @@ describe("test_stale_dns_helper", () => { targetInstance.notifyHostListChanged(changes); expect(targetInstance["writerHostInfo"]).toBeNull(); - expect(targetInstance["writerHostAddress"]).toBe(""); }); }); diff --git a/tests/unit/storage_service.test.ts b/tests/unit/storage_service.test.ts index 25d32e48..a52d739c 100644 --- a/tests/unit/storage_service.test.ts +++ b/tests/unit/storage_service.test.ts @@ -1,18 +1,18 @@ /* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ import { StorageService } from "../../common/lib/utils/storage/storage_service"; import { Topology } from "../../common/lib/host_list_provider/topology"; @@ -24,7 +24,7 @@ describe("test_storage_service", () => { let storageService: StorageService; beforeEach(() => { - storageService = CoreServicesContainer.getInstance().getStorageService(); + storageService = CoreServicesContainer.getInstance().storageService; }); afterEach(() => { diff --git a/tests/unit/topology_utils.test.ts b/tests/unit/topology_utils.test.ts index a757c8a6..b3e927c0 100644 --- a/tests/unit/topology_utils.test.ts +++ b/tests/unit/topology_utils.test.ts @@ -14,9 +14,10 @@ limitations under the License. */ -import { TopologyQueryResult, TopologyUtils } from "../../common/lib/host_list_provider/topology_utils"; +import { TopologyQueryResult } from "../../common/lib/host_list_provider/topology_utils"; +import { AuroraTopologyUtils } from "../../common/lib/host_list_provider/aurora_topology_utils"; import { anything, instance, mock, reset, when } from "ts-mockito"; -import { HostInfo, HostInfoBuilder } from "../../common/lib"; +import { HostInfo, HostInfoBuilder, PluginManager } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { AuroraPgDatabaseDialect } from "../../pg/lib/dialect/aurora_pg_database_dialect"; import { ClientWrapper } from "../../common/lib/client_wrapper"; @@ -43,8 +44,8 @@ function createHost(config: any): HostInfo { return info.build(); } -function getTopologyUtils(): TopologyUtils { - return new TopologyUtils(instance(mockDialect), hostInfoBuilder); +function getTopologyUtils(): AuroraTopologyUtils { + return new AuroraTopologyUtils(instance(mockDialect), hostInfoBuilder); } describe("testTopologyUtils", () => { @@ -54,9 +55,13 @@ describe("testTopologyUtils", () => { reset(mockNonTopologyDialect); }); + afterEach(async () => { + await PluginManager.releaseResources(); + }); + it("testQueryForTopology_withNonTopologyAwareDialect_throwsError", async () => { const hostInfoBuilder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); - const topologyUtils = new TopologyUtils(instance(mockNonTopologyDialect) as any, hostInfoBuilder); + const topologyUtils = new AuroraTopologyUtils(instance(mockNonTopologyDialect) as any, hostInfoBuilder); const initialHost = createHost({ host: "initial-host", diff --git a/tests/unit/writer_failover_handler.test.ts b/tests/unit/writer_failover_handler.test.ts index 2d4dcf46..bb07d9d9 100644 --- a/tests/unit/writer_failover_handler.test.ts +++ b/tests/unit/writer_failover_handler.test.ts @@ -211,7 +211,7 @@ describe("writer failover handler", () => { expect(result.topology.length).toBe(4); expect(result.topology[0].host).toBe("new-writer-host"); - verify(mockPluginService.forceRefreshHostList(anything())).atLeast(1); + verify(mockPluginService.forceRefreshHostList()).atLeast(1); verify(mockPluginService.setAvailability(newWriterHost.allAliases, HostAvailability.AVAILABLE)).once(); clearTimeout(timeoutId); }, 10000); From b9c34971f3950cc893d488daaff59f4368798da6 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 20 Apr 2026 00:31:45 -0700 Subject: [PATCH 14/20] feat: GDB RW Splitting (#626) --- .github/workflows/integration_tests.yml | 1 + common/lib/connection_plugin_chain_builder.ts | 3 + .../monitoring/cluster_topology_monitor.ts | 6 + .../rds_host_list_provider.ts | 15 ++- common/lib/partial_plugin_service.ts | 2 +- common/lib/pg_client_wrapper.ts | 2 +- common/lib/plugin_service.ts | 23 ++++ .../plugins/bluegreen/blue_green_plugin.ts | 15 ++- .../bluegreen/blue_green_plugin_factory.ts | 2 +- .../lib/plugins/failover/failover_plugin.ts | 16 ++- .../failover/failover_plugin_factory.ts | 2 +- .../failover/writer_failover_handler.ts | 25 +++- .../gdb_read_write_splitting_plugin.ts | 96 ++++++++++++++++ ...gdb_read_write_splitting_plugin_factory.ts | 39 +++++++ .../read_write_splitting_plugin.ts | 4 +- .../fastest_response_strategy_plugin.ts | 14 +-- ...fastest_respose_strategy_plugin_factory.ts | 5 +- .../host_response_time_monitor.ts | 108 +++++++++++------- .../host_response_time_service.ts | 97 +++++++--------- common/lib/utils/errors.ts | 2 + common/lib/utils/messages.ts | 5 + common/lib/wrapper_property.ts | 14 +++ index.ts | 2 +- tests/integration/container/tests/config.ts | 5 + .../tests/failover/gdb_failover.test.ts | 1 + .../tests/iam_authentication.test.ts | 9 -- .../tests/parameterized_queries.test.ts | 2 +- .../container/tests/pg_pool.test.ts | 6 + .../container/tests/utils/test_environment.ts | 13 +++ .../host/TestEnvironmentConfig.java | 4 + .../host/util/ContainerHelper.java | 1 - tests/unit/failover_plugin.test.ts | 9 +- tests/unit/read_write_splitting.test.ts | 1 + tests/unit/writer_failover_handler.test.ts | 101 +++++++++++++++- 34 files changed, 495 insertions(+), 155 deletions(-) create mode 100644 common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts create mode 100644 common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 8b59c9ef..4b0e28e9 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -152,6 +152,7 @@ jobs: with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_int_latest_tests + role-duration-seconds: 21600 aws-region: ${{ secrets.AWS_DEFAULT_REGION }} output-credentials: true diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 1cad14c7..38ec0a4f 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -45,6 +45,7 @@ import { HostMonitoring2PluginFactory } from "./plugins/efm2/host_monitoring2_pl import { BlueGreenPluginFactory } from "./plugins/bluegreen/blue_green_plugin_factory"; import { GlobalDbFailoverPluginFactory } from "./plugins/gdb_failover/global_db_failover_plugin_factory"; import { FullServicesContainer } from "./utils/full_services_container"; +import { GdbReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory"; /* Type alias used for plugin factory sorting. It holds a reference to a plugin @@ -65,6 +66,7 @@ export class ConnectionPluginChainBuilder { ["staleDns", { factory: StaleDnsPluginFactory, weight: 500 }], ["bg", { factory: BlueGreenPluginFactory, weight: 550 }], ["readWriteSplitting", { factory: ReadWriteSplittingPluginFactory, weight: 600 }], + ["gdbReadWriteSplitting", { factory: GdbReadWriteSplittingPluginFactory, weight: 610 }], ["failover", { factory: FailoverPluginFactory, weight: 700 }], ["failover2", { factory: Failover2PluginFactory, weight: 710 }], ["gdbFailover", { factory: GlobalDbFailoverPluginFactory, weight: 720 }], @@ -87,6 +89,7 @@ export class ConnectionPluginChainBuilder { [StaleDnsPluginFactory, 500], [BlueGreenPluginFactory, 550], [ReadWriteSplittingPluginFactory, 600], + [GdbReadWriteSplittingPluginFactory, 610], [FailoverPluginFactory, 700], [Failover2PluginFactory, 710], [GlobalDbFailoverPluginFactory, 720], diff --git a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts index e9b4dcfd..46241678 100644 --- a/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts +++ b/common/lib/host_list_provider/monitoring/cluster_topology_monitor.ts @@ -413,6 +413,7 @@ export class ClusterTopologyMonitorImpl extends AbstractMonitor implements Clust if (writerClient && writerClientHostInfo) { logger.debug(Messages.get("ClusterTopologyMonitor.writerPickedUpFromHostMonitors", writerClientHostInfo.toString())); + const oldMonitoringClient = this.monitoringClient; this.monitoringClient = writerClient; this.writerHostInfo = writerClientHostInfo; this.isVerifiedWriterConnection = true; @@ -425,6 +426,11 @@ export class ClusterTopologyMonitorImpl extends AbstractMonitor implements Clust this.readerTopologiesById.clear(); this.completedOneCycle.clear(); + // Close the old monitoring client that was replaced by the new writer client. + if (oldMonitoringClient && oldMonitoringClient !== this.monitoringClient) { + await this.closeConnection(oldMonitoringClient); + } + await this.delay(true); continue; } else { diff --git a/common/lib/host_list_provider/rds_host_list_provider.ts b/common/lib/host_list_provider/rds_host_list_provider.ts index 9558e1aa..4185d7db 100644 --- a/common/lib/host_list_provider/rds_host_list_provider.ts +++ b/common/lib/host_list_provider/rds_host_list_provider.ts @@ -144,11 +144,21 @@ export class RdsHostListProvider implements DynamicHostListProvider { if (!this.pluginService.isDialectConfirmed()) { // We need to confirm the dialect before creating a topology monitor so that it uses the correct SQL queries. - // We will return the original hosts parsed from the connections string until the dialect has been confirmed. + // Return the original hosts parsed from the connection string. return this.initialHostList; } - return await this.forceRefreshMonitor(verifyTopology, timeoutMs); + const hosts = await this.forceRefreshMonitor(verifyTopology, timeoutMs); + if (hosts && hosts.length > 0) { + return hosts; + } + + // Check for cached topology as a fallback. + const storedTopology = this.getStoredTopology(); + if (storedTopology && storedTopology.length > 0) { + return storedTopology; + } + return this.initialHostList; } async getHostRole(client: ClientWrapper, _dialect: DatabaseDialect): Promise { @@ -236,6 +246,7 @@ export class RdsHostListProvider implements DynamicHostListProvider { } async getCurrentTopology(targetClient: ClientWrapper, dialect: DatabaseDialect): Promise { + this.init(); return await this.topologyUtils.queryForTopology(targetClient, dialect, this.initialHost, this.clusterInstanceTemplate); } diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts index 2097d093..b3b8f729 100644 --- a/common/lib/partial_plugin_service.ts +++ b/common/lib/partial_plugin_service.ts @@ -264,7 +264,7 @@ export class PartialPluginService implements PluginService, HostListProviderServ } isDialectConfirmed(): boolean { - throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "isDialectConfirmed")); + return true; } setInTransaction(inTransaction: boolean): void { diff --git a/common/lib/pg_client_wrapper.ts b/common/lib/pg_client_wrapper.ts index 71af129c..a9c2e320 100644 --- a/common/lib/pg_client_wrapper.ts +++ b/common/lib/pg_client_wrapper.ts @@ -60,7 +60,7 @@ export class PgClientWrapper implements ClientWrapper { async abort(): Promise { try { - return await ClientUtils.queryWithTimeout(this.end(), this.properties); + this.client?.connection?.stream?.destroy(); } catch (error: any) { // Ignore } diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 68f69c97..8e8a5a1a 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -281,7 +281,28 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return this.dialect; } + private static readonly DIALECT_CONFIRMED_STATUS_KEY = "DialectConfirmed"; + + private getDialectConfirmedCacheKey(): string { + let clusterId = WrapperProperties.CLUSTER_ID.defaultValue; + try { + clusterId = this._hostListProvider?.getClusterId() ?? WrapperProperties.CLUSTER_ID.defaultValue; + } catch (e) { + // May fail if the host list provider does not support getClusterId. In this case use the default value. + } + return `${clusterId}::${PluginServiceImpl.DIALECT_CONFIRMED_STATUS_KEY}`; + } + isDialectConfirmed(): boolean { + if (this._isDialectConfirmed) { + return true; + } + + const cacheItem = this.storageService.get(StatusCacheItem, this.getDialectConfirmedCacheKey()); + if (cacheItem && cacheItem.status === true) { + this._isDialectConfirmed = true; + } + return this._isDialectConfirmed; } @@ -634,6 +655,8 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this.dialect = await this.dbDialectProvider.getDialectForUpdate(targetClient, this.initialHost, this.props.get(WrapperProperties.HOST.name)); this._isDialectConfirmed = true; + this.storageService.set(this.getDialectConfirmedCacheKey(), new StatusCacheItem(true)); + if (originalDialect === this.dialect) { return; } diff --git a/common/lib/plugins/bluegreen/blue_green_plugin.ts b/common/lib/plugins/bluegreen/blue_green_plugin.ts index 17cacb8d..ddcd34e8 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin.ts @@ -27,9 +27,10 @@ import { IamAuthenticationPlugin } from "../../authentication/iam_authentication import { BlueGreenRole } from "./blue_green_role"; import { ExecuteRouting, RoutingResultHolder } from "./routing/execute_routing"; import { CanReleaseResources } from "../../can_release_resources"; +import { FullServicesContainer } from "../../utils/full_services_container"; export interface BlueGreenProviderSupplier { - create(pluginService: PluginService, props: Map, bgdId: string): BlueGreenStatusProvider; + create(servicesContainer: FullServicesContainer, props: Map, bgdId: string): BlueGreenStatusProvider; } export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanReleaseResources { @@ -42,6 +43,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele private static readonly CLOSED_METHOD_NAMES: Set = new Set(["end", "abort"]); protected readonly pluginService: PluginService; + protected readonly servicesContainer: FullServicesContainer; protected readonly properties: Map; protected bgProviderSupplier: BlueGreenProviderSupplier; protected bgStatus: BlueGreenStatus = null; @@ -53,18 +55,19 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele protected endTimeNano: bigint = BigInt(0); private static provider: Map = new Map(); - constructor(pluginService: PluginService, properties: Map, bgProviderSupplier: BlueGreenProviderSupplier = null) { + constructor(servicesContainer: FullServicesContainer, properties: Map, bgProviderSupplier: BlueGreenProviderSupplier = null) { super(); if (!bgProviderSupplier) { bgProviderSupplier = { - create: (pluginService: PluginService, props: Map, bgdId: string): BlueGreenStatusProvider => { - return new BlueGreenStatusProvider(pluginService, props, bgdId); + create: (servicesContainer: FullServicesContainer, props: Map, bgdId: string): BlueGreenStatusProvider => { + return new BlueGreenStatusProvider(servicesContainer, props, bgdId); } }; } this.properties = properties; - this.pluginService = pluginService; + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.bgProviderSupplier = bgProviderSupplier; this.bgdId = WrapperProperties.BGD_ID.get(this.properties).trim().toLowerCase(); } @@ -215,7 +218,7 @@ export class BlueGreenPlugin extends AbstractConnectionPlugin implements CanRele private initProvider() { const provider = BlueGreenPlugin.provider.get(this.bgdId); if (!provider) { - const provider = this.bgProviderSupplier.create(this.pluginService, this.properties, this.bgdId); + const provider = this.bgProviderSupplier.create(this.servicesContainer, this.properties, this.bgdId); BlueGreenPlugin.provider.set(this.bgdId, provider); } } diff --git a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts index d20e00b6..c7f58106 100644 --- a/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts +++ b/common/lib/plugins/bluegreen/blue_green_plugin_factory.ts @@ -28,7 +28,7 @@ export class BlueGreenPluginFactory extends ConnectionPluginFactory { if (!BlueGreenPluginFactory.blueGreenPlugin) { BlueGreenPluginFactory.blueGreenPlugin = await import("./blue_green_plugin"); } - return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(servicesContainer.pluginService, props); + return new BlueGreenPluginFactory.blueGreenPlugin.BlueGreenPlugin(servicesContainer, props); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "BlueGreenPluginFactory")); } diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index bbb81e73..ceefae36 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -43,6 +43,7 @@ import { ClientWrapper } from "../../client_wrapper"; import { getWriter, logTopology } from "../../utils/utils"; import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; +import { FullServicesContainer } from "../../utils/full_services_container"; export class FailoverPlugin extends AbstractConnectionPlugin { private static readonly TELEMETRY_WRITER_FAILOVER = "failover to writer instance"; @@ -79,18 +80,19 @@ export class FailoverPlugin extends AbstractConnectionPlugin { private hostListProviderService?: HostListProviderService; private readonly pluginService: PluginService; + private readonly servicesContainer: FullServicesContainer; protected enableFailoverSetting: boolean = WrapperProperties.ENABLE_CLUSTER_AWARE_FAILOVER.defaultValue; - constructor(pluginService: PluginService, properties: Map, rdsHelper: RdsUtils); + constructor(servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils); constructor( - pluginService: PluginService, + servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils, readerFailoverHandler: ClusterAwareReaderFailoverHandler, writerFailoverHandler: ClusterAwareWriterFailoverHandler ); constructor( - pluginService: PluginService, + servicesContainer: FullServicesContainer, properties: Map, rdsHelper: RdsUtils, readerFailoverHandler?: ClusterAwareReaderFailoverHandler, @@ -98,7 +100,8 @@ export class FailoverPlugin extends AbstractConnectionPlugin { ) { super(); this._properties = properties; - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; + this.servicesContainer = servicesContainer; this._rdsHelper = rdsHelper; this.initSettings(); @@ -106,7 +109,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._readerFailoverHandler = readerFailoverHandler ? readerFailoverHandler : new ClusterAwareReaderFailoverHandler( - pluginService, + this.pluginService, properties, this.failoverTimeoutMsSetting, this.failoverReaderConnectTimeoutMsSetting, @@ -115,7 +118,8 @@ export class FailoverPlugin extends AbstractConnectionPlugin { this._writerFailoverHandler = writerFailoverHandler ? writerFailoverHandler : new ClusterAwareWriterFailoverHandler( - pluginService, + this.pluginService, + this.servicesContainer, this._readerFailoverHandler, properties, this.failoverTimeoutMsSetting, diff --git a/common/lib/plugins/failover/failover_plugin_factory.ts b/common/lib/plugins/failover/failover_plugin_factory.ts index cab67b40..82ac4483 100644 --- a/common/lib/plugins/failover/failover_plugin_factory.ts +++ b/common/lib/plugins/failover/failover_plugin_factory.ts @@ -29,7 +29,7 @@ export class FailoverPluginFactory extends ConnectionPluginFactory { if (!FailoverPluginFactory.failoverPlugin) { FailoverPluginFactory.failoverPlugin = await import("./failover_plugin"); } - return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(servicesContainer.pluginService, properties, new RdsUtils()); + return new FailoverPluginFactory.failoverPlugin.FailoverPlugin(servicesContainer, properties, new RdsUtils()); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FailoverPlugin")); } diff --git a/common/lib/plugins/failover/writer_failover_handler.ts b/common/lib/plugins/failover/writer_failover_handler.ts index b8780b88..e14a73fd 100644 --- a/common/lib/plugins/failover/writer_failover_handler.ts +++ b/common/lib/plugins/failover/writer_failover_handler.ts @@ -27,6 +27,9 @@ import { logger } from "../../../logutils"; import { WrapperProperties } from "../../wrapper_property"; import { ClientWrapper } from "../../client_wrapper"; import { FailoverRestriction } from "./failover_restriction"; +import { FullServicesContainer } from "../../utils/full_services_container"; +import { ServiceUtils } from "../../utils/service_utils"; +import { DatabaseDialect } from "../../database_dialect/database_dialect"; export interface WriterFailoverHandler { failover(currentTopology: HostInfo[]): Promise; @@ -47,6 +50,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler static readonly RECONNECT_WRITER_TASK = "TaskA"; static readonly WAIT_NEW_WRITER_TASK = "TaskB"; private readonly pluginService: PluginService; + private readonly servicesContainer: FullServicesContainer; private readonly readerFailoverHandler: ClusterAwareReaderFailoverHandler; private readonly initialConnectionProps: Map; maxFailoverTimeoutMs = 60000; // 60 sec @@ -55,6 +59,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler constructor( pluginService: PluginService, + servicesContainer: FullServicesContainer, readerFailoverHandler: ClusterAwareReaderFailoverHandler, initialConnectionProps: Map, failoverTimeoutMs?: number, @@ -62,6 +67,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler reconnectWriterIntervalMs?: number ) { this.pluginService = pluginService; + this.servicesContainer = servicesContainer; this.readerFailoverHandler = readerFailoverHandler; this.initialConnectionProps = initialConnectionProps; this.maxFailoverTimeoutMs = failoverTimeoutMs ?? this.maxFailoverTimeoutMs; @@ -69,16 +75,29 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler this.reconnectionWriterIntervalMs = reconnectWriterIntervalMs ?? this.reconnectionWriterIntervalMs; } + protected async newServicesContainer(): Promise { + const container = ServiceUtils.instance.createMinimalServiceContainerFrom(this.servicesContainer, this.initialConnectionProps); + await container.pluginManager.init(); + const initialHostInfo = this.pluginService.getInitialConnectionHostInfo(); + if (initialHostInfo) { + container.hostListProviderService.setInitialConnectionHostInfo(initialHostInfo); + } + return container; + } + async failover(currentTopology: HostInfo[]): Promise { if (!currentTopology || currentTopology.length == 0) { logger.error(Messages.get("ClusterAwareWriterFailoverHandler.failoverCalledWithInvalidTopology")); return ClusterAwareWriterFailoverHandler.DEFAULT_RESULT; } + const taskAContainer = await this.newServicesContainer(); + const taskBContainer = await this.newServicesContainer(); + const reconnectToWriterHandlerTask = new ReconnectToWriterHandlerTask( currentTopology, getWriter(currentTopology), - this.pluginService, + taskAContainer.pluginService, this.initialConnectionProps, this.reconnectionWriterIntervalMs, Date.now() + this.maxFailoverTimeoutMs @@ -88,7 +107,7 @@ export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler currentTopology, getWriter(currentTopology), this.readerFailoverHandler, - this.pluginService, + taskBContainer.pluginService, this.initialConnectionProps, this.readTopologyIntervalMs, Date.now() + this.maxFailoverTimeoutMs @@ -379,7 +398,7 @@ class WaitForNewWriterHandlerTask { async refreshTopologyAndConnectToNewWriter(): Promise { const allowOldWriter: boolean = this.pluginService.getDialect().getFailoverRestrictions().includes(FailoverRestriction.ENABLE_WRITER_IN_TASK_B); - while (this.pluginService.getCurrentClient() && Date.now() < this.endTime && !this.failoverCompleted) { + while (Date.now() < this.endTime && !this.failoverCompleted) { try { if (this.currentReaderTargetClient) { await this.pluginService.forceRefreshHostList(); diff --git a/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts new file mode 100644 index 00000000..b9f4d94f --- /dev/null +++ b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin.ts @@ -0,0 +1,96 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ReadWriteSplittingPlugin } from "./read_write_splitting_plugin"; +import { WrapperProperties } from "../../wrapper_property"; +import { HostInfo } from "../../host_info"; +import { RdsUtils } from "../../utils/rds_utils"; +import { ReadWriteSplittingError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { logger } from "../../../logutils"; +import { ClientWrapper } from "../../client_wrapper"; +import { equalsIgnoreCase } from "../../utils/utils"; + +export class GdbReadWriteSplittingPlugin extends ReadWriteSplittingPlugin { + protected readonly rdsUtils: RdsUtils = new RdsUtils(); + + protected restrictWriterToHomeRegion: boolean; + protected restrictReaderToHomeRegion: boolean; + + protected isInitialized: boolean = false; + protected homeRegion: string; + + protected initSettings(initHostInfo: HostInfo, properties: Map): void { + if (this.isInitialized) { + return; + } + this.restrictWriterToHomeRegion = WrapperProperties.GDB_RW_RESTRICT_WRITER_TO_HOME_REGION.get(properties); + this.restrictReaderToHomeRegion = WrapperProperties.GDB_RW_RESTRICT_READER_TO_HOME_REGION.get(properties); + + this.homeRegion = WrapperProperties.GDB_RW_HOME_REGION.get(properties); + if (!this.homeRegion) { + const rdsUrlType = this.rdsUtils.identifyRdsType(initHostInfo.host); + if (rdsUrlType.hasRegion) { + this.homeRegion = this.rdsUtils.getRdsRegion(initHostInfo.host); + } + } + + if (!this.homeRegion) { + throw new ReadWriteSplittingError(Messages.get("GdbReadWriteSplittingPlugin.missingHomeRegion", initHostInfo.host)); + } + + logger.debug(Messages.get("GdbReadWriteSplittingPlugin.parameterValue", "gdbRwHomeRegion", this.homeRegion)); + + this.isInitialized = true; + } + + override async connect( + hostInfo: HostInfo, + props: Map, + isInitialConnection: boolean, + connectFunc: () => Promise + ): Promise { + this.initSettings(hostInfo, props); + return super.connect(hostInfo, props, isInitialConnection, connectFunc); + } + + override setWriterClient(writerTargetClient: ClientWrapper | undefined, writerHostInfo: HostInfo) { + if ( + this.restrictWriterToHomeRegion && + writerHostInfo != null && + !equalsIgnoreCase(this.rdsUtils.getRdsRegion(writerHostInfo.host), this.homeRegion) + ) { + throw new ReadWriteSplittingError( + Messages.get("GdbReadWriteSplittingPlugin.cantConnectWriterOutOfHomeRegion", writerHostInfo.host, this.homeRegion) + ); + } + super.setWriterClient(writerTargetClient, writerHostInfo); + } + + protected getReaderHostCandidates(): HostInfo[] { + if (this.restrictReaderToHomeRegion) { + const hostsInRegion: HostInfo[] = this.pluginService + .getHosts() + .filter((x) => equalsIgnoreCase(this.rdsUtils.getRdsRegion(x.host), this.homeRegion)); + + if (hostsInRegion.length === 0) { + throw new ReadWriteSplittingError(Messages.get("GdbReadWriteSplittingPlugin.noAvailableReadersInHomeRegion", this.homeRegion)); + } + return hostsInRegion; + } + return super.getReaderHostCandidates(); + } +} diff --git a/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts new file mode 100644 index 00000000..7aa6d1ad --- /dev/null +++ b/common/lib/plugins/read_write_splitting/gdb_read_write_splitting_plugin_factory.ts @@ -0,0 +1,39 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionPluginFactory } from "../../plugin_factory"; +import { ConnectionPlugin } from "../../connection_plugin"; +import { AwsWrapperError } from "../../utils/errors"; +import { Messages } from "../../utils/messages"; +import { FullServicesContainer } from "../../utils/full_services_container"; + +export class GdbReadWriteSplittingPluginFactory extends ConnectionPluginFactory { + private static gdbReadWriteSplittingPlugin: any; + + async getInstance(servicesContainer: FullServicesContainer, properties: Map): Promise { + try { + if (!GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin) { + GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin = await import("./gdb_read_write_splitting_plugin"); + } + return new GdbReadWriteSplittingPluginFactory.gdbReadWriteSplittingPlugin.GdbReadWriteSplittingPlugin( + servicesContainer.pluginService, + properties + ); + } catch (error: any) { + throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "gdbReadWriteSplittingPlugin")); + } + } +} diff --git a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts index 6ade242f..2d70b893 100644 --- a/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts +++ b/common/lib/plugins/read_write_splitting/read_write_splitting_plugin.ts @@ -64,7 +64,7 @@ export class ReadWriteSplittingPlugin extends AbstractReadWriteSplittingPlugin { if (!isInitialConnection || !this._hostListProviderService?.isDynamicHostListProvider()) { return result; } - const currentRole = this.pluginService.getCurrentHostInfo()?.role; + const currentRole = await this.pluginService.getHostRole(result); if (currentRole == HostRole.UNKNOWN) { logAndThrowError(Messages.get("ReadWriteSplittingPlugin.errorVerifyingInitialHostRole")); @@ -181,7 +181,7 @@ export class ReadWriteSplittingPlugin extends AbstractReadWriteSplittingPlugin { } } - protected getReaderHostCandidates(): HostInfo[] | undefined { + protected getReaderHostCandidates(): HostInfo[] { return this.pluginService.getHosts(); } } diff --git a/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts b/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts index e0d3a174..54e10a7a 100644 --- a/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts +++ b/common/lib/plugins/strategy/fastest_response/fastest_response_strategy_plugin.ts @@ -26,6 +26,7 @@ import { HostChangeOptions } from "../../../host_change_options"; import { RandomHostSelector } from "../../../random_host_selector"; import { Messages } from "../../../utils/messages"; import { equalsIgnoreCase, logAndThrowError } from "../../../utils/utils"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { static readonly FASTEST_RESPONSE_STRATEGY_NAME: string = "fastestResponse"; @@ -43,13 +44,13 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { private pluginService: PluginService; private randomHostSelector: RandomHostSelector = new RandomHostSelector(); - constructor(pluginService: PluginService, properties: Map, hostResponseTimeService?: HostResponseTimeService) { + constructor(servicesContainer: FullServicesContainer, properties: Map, hostResponseTimeService?: HostResponseTimeService) { super(); - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; this.properties = properties; this.hostResponseTimeService = hostResponseTimeService ?? - new HostResponseTimeServiceImpl(pluginService, properties, WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties)); + new HostResponseTimeServiceImpl(servicesContainer, properties, WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties)); this.cacheExpirationNanos = BigInt(WrapperProperties.RESPONSE_MEASUREMENT_INTERVAL_MILLIS.get(this.properties) * 1_000_000); } @@ -91,18 +92,13 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { if (!this.acceptsStrategy(role, strategy)) { logAndThrowError(Messages.get("FastestResponseStrategyPlugin.unsupportedHostSelectorStrategy", strategy)); } - // The cache holds a host with the fastest response time. - // If the cache doesn't have a host for a role, it's necessary to find the fastest host in the topology. const fastestResponseHost: HostInfo = FastestResponseStrategyPlugin.cachedFastestResponseHostByRole.get(role); if (fastestResponseHost) { - // Found the fastest host. Find the host in the latest topology. const foundHost = this.pluginService.getHosts().find((host) => host === fastestResponseHost); if (foundHost) { - // Found a host in the topology. return foundHost; } } - // Cached result isn't available. Need to find the fastest response time host. const calculatedFastestResponseHost: ResponseTimeTuple[] = this.pluginService .getHosts() .filter((host) => role === host.role) @@ -113,8 +109,6 @@ export class FastestResponseStrategyPlugin extends AbstractConnectionPlugin { const calculatedHost = calculatedFastestResponseHost.length === 0 ? null : calculatedFastestResponseHost[0]; if (!calculatedHost) { - // Unable to identify the fastest response host. - // As a last resort, let's use a random host selector. return this.randomHostSelector.getHost(hosts, role, this.properties); } FastestResponseStrategyPlugin.cachedFastestResponseHostByRole.put(role, calculatedHost.hostInfo, Number(this.cacheExpirationNanos)); diff --git a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts index 9518fdfd..340e6f72 100644 --- a/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts +++ b/common/lib/plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory.ts @@ -28,10 +28,7 @@ export class FastestResponseStrategyPluginFactory extends ConnectionPluginFactor if (!FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin) { FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin = await import("./fastest_response_strategy_plugin"); } - return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin( - servicesContainer.pluginService, - properties - ); + return new FastestResponseStrategyPluginFactory.fastestResponseStrategyPlugin.FastestResponseStrategyPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError( Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "FastestResponseStrategyPluginFactory") diff --git a/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts b/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts index 5e0a8b80..05074742 100644 --- a/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts +++ b/common/lib/plugins/strategy/fastest_response/host_response_time_monitor.ts @@ -17,66 +17,95 @@ import { HostInfo } from "../../../host_info"; import { PluginService } from "../../../plugin_service"; import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; -import { sleep } from "../../../utils/utils"; +import { sleepWithAbort } from "../../../utils/utils"; import { logger } from "../../../../logutils"; import { Messages } from "../../../utils/messages"; import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; import { ClientWrapper } from "../../../client_wrapper"; -import { TelemetryContext } from "../../../utils/telemetry/telemetry_context"; import { WrapperProperties } from "../../../wrapper_property"; +import { AbstractMonitor } from "../../../utils/monitoring/monitor"; +import { FullServicesContainer } from "../../../utils/full_services_container"; -export class HostResponseTimeMonitor { - static readonly MONITORING_PROPERTY_PREFIX = "frt_"; +export class ResponseTimeHolder { + private readonly url: string; + private readonly responseTime: number; + + constructor(url: string, responseTime: number) { + this.url = url; + this.responseTime = responseTime; + } + + getUrl(): string { + return this.url; + } + + getResponseTime(): number { + return this.responseTime; + } +} + +export class HostResponseTimeMonitor extends AbstractMonitor { static readonly NUM_OF_MEASURES = 5; + private static readonly TERMINATION_TIMEOUT_SEC = 5; private readonly intervalMs: number; private readonly hostInfo: HostInfo; - private stopped = false; private responseTimeMs = Number.MAX_SAFE_INTEGER; - private checkTimestamp = Date.now(); private readonly properties: Map; - private pluginService: PluginService; - private telemetryFactory: TelemetryFactory; + private readonly servicesContainer: FullServicesContainer; + private readonly pluginService: PluginService; + private readonly telemetryFactory: TelemetryFactory; protected monitoringClient: ClientWrapper | null = null; + private abortSleep?: () => void; - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, intervalMs: number) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, hostInfo: HostInfo, properties: Map, intervalMs: number) { + super(HostResponseTimeMonitor.TERMINATION_TIMEOUT_SEC); + this.servicesContainer = servicesContainer; + this.pluginService = servicesContainer.pluginService; this.hostInfo = hostInfo; this.properties = properties; this.intervalMs = intervalMs; this.telemetryFactory = this.pluginService.getTelemetryFactory(); - const hostId: string = this.hostInfo.hostId ?? this.getHostInfo().host; - /** - * Report current response time (in milliseconds) to telemetry engine. - * Report -1 if response time couldn't be measured. - */ - this.telemetryFactory.createGauge(`frt.response.time.${hostId}`, () => this.getResponseTime() == Number.MAX_SAFE_INTEGER); - this.run(); + const hostId: string = this.hostInfo.hostId ?? this.hostInfo.host; + this.telemetryFactory.createGauge(`frt.response.time.${hostId}`, () => + this.responseTimeMs === Number.MAX_SAFE_INTEGER ? -1 : this.responseTimeMs + ); } - getResponseTime() { + getResponseTime(): number { return this.responseTimeMs; } - getCheckTimeStamp() { - return this.checkTimestamp; - } - - getHostInfo() { + getHostInfo(): HostInfo { return this.hostInfo; } async close(): Promise { - this.stopped = true; - await sleep(500); - logger.debug(Messages.get("HostResponseTimeMonitor.stopped", this.hostInfo.host)); + if (this.abortSleep) { + try { + this.abortSleep(); + } catch (error) { + // ignore + } + this.abortSleep = undefined; + } + if (this.monitoringClient) { + try { + await this.monitoringClient.abort(); + } catch (error) { + // ignore + } + this.monitoringClient = null; + } } - async run(): Promise { - const telemetryContext: TelemetryContext = this.telemetryFactory.openTelemetryContext("host response time task", TelemetryTraceLevel.TOP_LEVEL); + async monitor(): Promise { + const telemetryContext = this.telemetryFactory.openTelemetryContext("host response time task", TelemetryTraceLevel.TOP_LEVEL); telemetryContext.setAttribute("url", this.hostInfo.host); - while (!this.stopped) { + + while (!this._stop) { + this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); await telemetryContext.start(async () => { try { await this.openConnection(); @@ -84,33 +113,29 @@ export class HostResponseTimeMonitor { let responseTimeSum = 0; let count = 0; for (let i = 0; i < HostResponseTimeMonitor.NUM_OF_MEASURES; i++) { - if (this.stopped) { + if (this._stop) { break; } const startTime = Date.now(); if (await this.pluginService.isClientValid(this.monitoringClient)) { - const responseTime = Date.now() - startTime; - responseTimeSum += responseTime; + responseTimeSum += Date.now() - startTime; count++; } } if (count > 0) { this.responseTimeMs = responseTimeSum / count; + this.servicesContainer.storageService.set(this.hostInfo.url, new ResponseTimeHolder(this.hostInfo.url, this.responseTimeMs)); } else { this.responseTimeMs = Number.MAX_SAFE_INTEGER; + this.servicesContainer.storageService.remove(ResponseTimeHolder, this.hostInfo.url); } - this.checkTimestamp = Date.now(); logger.debug(Messages.get("HostResponseTimeMonitor.responseTime", this.hostInfo.host, this.responseTimeMs.toString())); } - await sleep(this.intervalMs); + const [sleepPromise, abortFn] = sleepWithAbort(this.intervalMs); + this.abortSleep = abortFn as () => void; + await sleepPromise; } catch (error) { logger.debug(Messages.get("HostResponseTimeMonitor.interruptedErrorDuringMonitoring", this.hostInfo.host, error.message)); - } finally { - this.stopped = true; - if (this.monitoringClient) { - await this.monitoringClient.abort(); - } - this.monitoringClient = null; } }); } @@ -119,8 +144,7 @@ export class HostResponseTimeMonitor { async openConnection(): Promise { try { if (this.monitoringClient) { - const clientIsValid = await this.pluginService.isClientValid(this.monitoringClient); - if (clientIsValid) { + if (await this.pluginService.isClientValid(this.monitoringClient)) { return; } } diff --git a/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts b/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts index 627a69b8..be6025e1 100644 --- a/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts +++ b/common/lib/plugins/strategy/fastest_response/host_response_time_service.ts @@ -15,80 +15,65 @@ */ import { HostInfo } from "../../../host_info"; -import { PluginService } from "../../../plugin_service"; -import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; -import { SlidingExpirationCache } from "../../../utils/sliding_expiration_cache"; -import { HostResponseTimeMonitor } from "./host_response_time_monitor"; +import { HostResponseTimeMonitor, ResponseTimeHolder } from "./host_response_time_monitor"; +import { FullServicesContainer } from "../../../utils/full_services_container"; +import { MonitorErrorResponse } from "../../../utils/monitoring/monitor"; export interface HostResponseTimeService { - /** - * Return a response time in milliseconds to the host. - * Return Number.MAX_SAFE_INTEGER if response time is not available. - * - * @param hostInfo the host details - * @return response time in milliseconds for a desired host. - */ getResponseTime(hostInfo: HostInfo): number; - - /** - * Provides an updated host list to a service. - */ setHosts(hosts: HostInfo[]): void; } export class HostResponseTimeServiceImpl implements HostResponseTimeService { - static readonly CACHE_EXPIRATION_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes - static readonly CACHE_CLEANUP_NANOS: bigint = BigInt(60_000_000_000); // 1 minute + private static readonly MONITOR_DISPOSAL_TIME_NANOS: bigint = BigInt(10 * 60_000_000_000); // 10 minutes + private static readonly INACTIVE_TIMEOUT_NANOS: bigint = BigInt(3 * 60_000_000_000); // 3 minutes - private readonly pluginService: PluginService; - readonly properties: Map; - readonly intervalMs: number; - protected hosts: HostInfo[]; - private readonly telemetryFactory: TelemetryFactory; - protected static monitoringHosts: SlidingExpirationCache = new SlidingExpirationCache( - HostResponseTimeServiceImpl.CACHE_CLEANUP_NANOS, - undefined, - async (monitor: HostResponseTimeMonitor) => { - { - try { - await monitor.close(); - } catch (error) { - // ignore - } - } - } - ); + private readonly servicesContainer: FullServicesContainer; + private readonly properties: Map; + private readonly intervalMs: number; + private hosts: HostInfo[] = []; - constructor(pluginService: PluginService, properties: Map, intervalMs: number) { - this.pluginService = pluginService; + constructor(servicesContainer: FullServicesContainer, properties: Map, intervalMs: number) { + this.servicesContainer = servicesContainer; this.properties = properties; this.intervalMs = intervalMs; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - HostResponseTimeServiceImpl.monitoringHosts.cleanupIntervalNs = BigInt(intervalMs) ?? HostResponseTimeServiceImpl.CACHE_CLEANUP_NANOS; - this.telemetryFactory.createGauge("frt.hosts.count", () => HostResponseTimeServiceImpl.monitoringHosts.size); + + this.servicesContainer.storageService.registerItemClassIfAbsent( + ResponseTimeHolder, + true, + HostResponseTimeServiceImpl.MONITOR_DISPOSAL_TIME_NANOS, + null, + null + ); + + this.servicesContainer.monitorService.registerMonitorTypeIfAbsent( + HostResponseTimeMonitor, + HostResponseTimeServiceImpl.MONITOR_DISPOSAL_TIME_NANOS, + HostResponseTimeServiceImpl.INACTIVE_TIMEOUT_NANOS, + new Set([MonitorErrorResponse.RECREATE]), + ResponseTimeHolder + ); } getResponseTime(hostInfo: HostInfo): number { - const monitor: HostResponseTimeMonitor = HostResponseTimeServiceImpl.monitoringHosts.get( - hostInfo.url, - HostResponseTimeServiceImpl.CACHE_EXPIRATION_NANOS - ); - if (!monitor) { - return Number.MAX_SAFE_INTEGER; - } - return monitor.getResponseTime(); + const holder: ResponseTimeHolder | null = this.servicesContainer.storageService.get(ResponseTimeHolder, hostInfo.url); + return holder === null ? Number.MAX_SAFE_INTEGER : holder.getResponseTime(); } setHosts(hosts: HostInfo[]): void { - const oldHostMap: string[] = hosts.flatMap((host) => host.url); + const oldHostUrls: Set = new Set(this.hosts.map((host) => host.url)); + this.hosts = hosts; + + const servicesContainer = this.servicesContainer; + const properties = this.properties; + const intervalMs = this.intervalMs; + hosts - .filter((hostInfo: HostInfo) => !(hostInfo.url in oldHostMap)) - .forEach((hostInfo: HostInfo) => { - HostResponseTimeServiceImpl.monitoringHosts.computeIfAbsent( - hostInfo.url, - (key) => new HostResponseTimeMonitor(this.pluginService, hostInfo, this.properties, this.intervalMs), - HostResponseTimeServiceImpl.CACHE_EXPIRATION_NANOS - ); + .filter((hostInfo) => !oldHostUrls.has(hostInfo.url)) + .forEach((hostInfo) => { + servicesContainer.monitorService.runIfAbsent(HostResponseTimeMonitor, hostInfo.url, servicesContainer, properties, { + createMonitor: (sc: FullServicesContainer) => new HostResponseTimeMonitor(sc, hostInfo, properties, intervalMs) + }); }); } } diff --git a/common/lib/utils/errors.ts b/common/lib/utils/errors.ts index 247bbba0..58e37aec 100644 --- a/common/lib/utils/errors.ts +++ b/common/lib/utils/errors.ts @@ -48,6 +48,8 @@ export class FailoverFailedError extends FailoverError {} export class TransactionResolutionUnknownError extends FailoverError {} +export class ReadWriteSplittingError extends AwsWrapperError {} + export class LoginError extends AwsWrapperError {} export class AwsTimeoutError extends AwsWrapperError {} diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 353c00de..32d9b9df 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -421,6 +421,11 @@ const MESSAGES: Record = { "GlobalDbFailoverPlugin.unableToFindCandidateWithMatchingRole": "Unable to find a candidate host with the expected role (%s) based on the given host selection strategy: %s", "GlobalDbFailoverPlugin.unableToConnect": "Unable to establish a connection during Global DB failover.", + "GdbReadWriteSplittingPlugin.missingHomeRegion": + "Unable to parse home region from endpoint '%s'. Please ensure you have set the 'gdbRwHomeRegion' connection parameter.", + "GdbReadWriteSplittingPlugin.cantConnectWriterOutOfHomeRegion": "Writer connection to '%s' is not allowed since it is out of home region '%s'.", + "GdbReadWriteSplittingPlugin.noAvailableReadersInHomeRegion": "No available reader nodes in home region '%s'.", + "GdbReadWriteSplittingPlugin.parameterValue": "%s=%s", "BatchingEventPublisher.errorDeliveringImmediateEvent": "Error delivering immediate event: %s", "WrapperProperty.invalidValue": "Invalid value '%s' for property '%s'. Allowed values: %s" }; diff --git a/common/lib/wrapper_property.ts b/common/lib/wrapper_property.ts index 2390c782..efb65a77 100644 --- a/common/lib/wrapper_property.ts +++ b/common/lib/wrapper_property.ts @@ -561,6 +561,20 @@ export class WrapperProperties { ["writer", "none"] ); + static readonly GDB_RW_HOME_REGION = new WrapperProperty("gdbRwHomeRegion", "Specifies the home region for read/write splitting.", null); + + static readonly GDB_RW_RESTRICT_WRITER_TO_HOME_REGION = new WrapperProperty( + "gdbRwRestrictWriterToHomeRegion", + "Prevents connections to a writer node outside of the defined home region.", + true + ); + + static readonly GDB_RW_RESTRICT_READER_TO_HOME_REGION = new WrapperProperty( + "gdbRwRestrictReaderToHomeRegion", + "Prevents connections to a reader node outside of the defined home region.", + true + ); + private static readonly PREFIXES = [ WrapperProperties.MONITORING_PROPERTY_PREFIX, WrapperProperties.TOPOLOGY_MONITORING_PROPERTY_PREFIX, diff --git a/index.ts b/index.ts index eb152e55..6292539c 100644 --- a/index.ts +++ b/index.ts @@ -44,7 +44,7 @@ export type { ConnectionProvider } from "./common/lib/connection_provider"; export type { HostSelector } from "./common/lib/host_selector"; export type { DatabaseDialect } from "./common/lib/database_dialect/database_dialect"; export type { PluginService } from "./common/lib/plugin_service"; -export type { HostListProvider, BlockingHostListProvider } from "./common/lib/host_list_provider/host_list_provider"; +export type { HostListProvider } from "./common/lib/host_list_provider/host_list_provider"; export type { ErrorHandler } from "./common/lib/error_handler"; export type { SessionStateService } from "./common/lib/session_state_service"; export type { DriverDialect } from "./common/lib/driver_dialect/driver_dialect"; diff --git a/tests/integration/container/tests/config.ts b/tests/integration/container/tests/config.ts index b50208cf..221b397d 100644 --- a/tests/integration/container/tests/config.ts +++ b/tests/integration/container/tests/config.ts @@ -15,6 +15,7 @@ */ import { CustomConsole, LogMessage, LogType } from "@jest/console"; +import { TestEnvironment } from "./utils/test_environment"; function simpleFormatter(type: LogType, message: LogMessage): string { return message @@ -34,3 +35,7 @@ const testInfo = JSON.parse(infoJson); const request = testInfo.request; export const features = request.features; export const instanceCount = request.numOfInstances; + +afterAll(async () => { + await TestEnvironment.shutdownTelemetry(); +}); diff --git a/tests/integration/container/tests/failover/gdb_failover.test.ts b/tests/integration/container/tests/failover/gdb_failover.test.ts index 261d98c0..2d5348a3 100644 --- a/tests/integration/container/tests/failover/gdb_failover.test.ts +++ b/tests/integration/container/tests/failover/gdb_failover.test.ts @@ -166,6 +166,7 @@ describe("gdb failover", () => { const config = await initDefaultConfig(initialWriterHost, initialWriterPort, true); config["activeHomeFailoverMode"] = "home-reader-or-writer"; config["inactiveHomeFailoverMode"] = "home-reader-or-writer"; + config["wrapperQueryTimeout"] = 2000; client = initClientFunc(config); await client.connect(); diff --git a/tests/integration/container/tests/iam_authentication.test.ts b/tests/integration/container/tests/iam_authentication.test.ts index 5b516c52..a3876663 100644 --- a/tests/integration/container/tests/iam_authentication.test.ts +++ b/tests/integration/container/tests/iam_authentication.test.ts @@ -23,7 +23,6 @@ import { readFileSync } from "fs"; import { logger } from "../../../../common/logutils"; import { TestEnvironmentFeatures } from "./utils/test_environment_features"; import { features } from "./config"; -import { jest } from "@jest/globals"; const itIf = !features.includes(TestEnvironmentFeatures.PERFORMANCE) && @@ -72,9 +71,6 @@ async function validateConnection() { describe("iam authentication", () => { beforeEach(async () => { logger.info(`Test started: ${expect.getState().currentTestName}`); - jest.useFakeTimers({ - doNotFake: ["nextTick"] - }); client = null; env = await TestEnvironment.getCurrent(); driver = DriverHelper.getDriverForDatabaseEngine(env.engine); @@ -93,11 +89,6 @@ describe("iam authentication", () => { logger.info(`Test finished: ${expect.getState().currentTestName}`); }, 1320000); - afterAll(async () => { - jest.runOnlyPendingTimers(); - jest.useRealTimers(); - }); - itIf( "iam wrong database username", async () => { diff --git a/tests/integration/container/tests/parameterized_queries.test.ts b/tests/integration/container/tests/parameterized_queries.test.ts index a8ceac14..0be0f2a8 100644 --- a/tests/integration/container/tests/parameterized_queries.test.ts +++ b/tests/integration/container/tests/parameterized_queries.test.ts @@ -141,7 +141,7 @@ beforeEach(async () => { logger.info(`Test started: ${expect.getState().currentTestName}`); await TestEnvironment.verifyClusterStatus(); client = null; -}, 60000); +}, 1320000); afterEach(async () => { if (client !== null) { diff --git a/tests/integration/container/tests/pg_pool.test.ts b/tests/integration/container/tests/pg_pool.test.ts index 4b09f224..2b4744d2 100644 --- a/tests/integration/container/tests/pg_pool.test.ts +++ b/tests/integration/container/tests/pg_pool.test.ts @@ -156,6 +156,12 @@ describe("pg pool integration tests", () => { itIfPGTwoInstances("failover writer during multi-statement transaction", async () => { client = await factory(); + try { + await client.query("DROP TABLE IF EXISTS test_table"); + } catch { + // Ignore + } + const initialWriterId = await auroraTestUtility.queryInstanceId(client); expect(await auroraTestUtility.isDbInstanceWriter(initialWriterId)).toStrictEqual(true); diff --git a/tests/integration/container/tests/utils/test_environment.ts b/tests/integration/container/tests/utils/test_environment.ts index 459229fa..a03caa1c 100644 --- a/tests/integration/container/tests/utils/test_environment.ts +++ b/tests/integration/container/tests/utils/test_environment.ts @@ -44,6 +44,7 @@ import { readFileSync } from "fs"; export class TestEnvironment { private static env?: TestEnvironment; + private static sdk?: NodeSDK; private readonly _info: TestEnvironmentInfo; private proxies?: { [s: string]: ProxyInfo }; @@ -281,6 +282,7 @@ export class TestEnvironment { // this enables the API to record telemetry sdk.start(); + TestEnvironment.sdk = sdk; // gracefully shut down the SDK on process exit process.on("SIGTERM", () => { sdk @@ -293,6 +295,17 @@ export class TestEnvironment { return env; } + static async shutdownTelemetry(): Promise { + if (TestEnvironment.sdk) { + try { + await TestEnvironment.sdk.shutdown(); + } catch (error) { + // ignore + } + TestEnvironment.sdk = undefined; + } + } + static async initProxies(environment: TestEnvironment) { if (environment.features.includes(TestEnvironmentFeatures.NETWORK_OUTAGES_ENABLED)) { environment.proxies = {}; diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java index f4c8fb2e..bfe94c65 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java @@ -13,6 +13,7 @@ import software.amazon.awssdk.services.rds.model.DBCluster; import software.amazon.awssdk.services.rds.model.DBInstance; +import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.net.UnknownHostException; @@ -1073,6 +1074,9 @@ private static void createTestContainer(TestEnvironmentConfig env) { .withEnv("AWS_SESSION_TOKEN", env.awsSessionToken); } + // Ensure the reports directory exists before mounting it into the container. + new File("../../../tests/integration/container/reports").mkdirs(); + env.testContainer.start(); } diff --git a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java index 88fc25c1..777b9a29 100644 --- a/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java +++ b/tests/integration/host/src/test/java/integration/host/util/ContainerHelper.java @@ -221,7 +221,6 @@ protected Long execInContainer( .execCreateCmd(containerId) .withAttachStdout(true) .withAttachStderr(true) - .withEnv(Arrays.asList("JEST_HTML_REPORTER_OUTPUT_PATH", "./tests/integration/container/reports/hello.html")) .withCmd(command); if (!StringUtils.isNullOrEmpty(workingDir)) { diff --git a/tests/unit/failover_plugin.test.ts b/tests/unit/failover_plugin.test.ts index 62750090..42e51025 100644 --- a/tests/unit/failover_plugin.test.ts +++ b/tests/unit/failover_plugin.test.ts @@ -44,6 +44,7 @@ import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_tele import { HostChangeOptions } from "../../common/lib/host_change_options"; import { Messages } from "../../common/lib/utils/messages"; import { RdsHostListProvider } from "../../common/lib/host_list_provider/rds_host_list_provider"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -61,6 +62,9 @@ const mockWriterResult: WriterFailoverResult = mock(WriterFailoverResult); const mockClientWrapper = new MySQLClientWrapper(undefined, mockHostInfo, new Map(), new MySQL2DriverDialect()); +const mockServicesContainer: FullServicesContainer = mock(); +const mockServicesContainerInstance = instance(mockServicesContainer); + const properties: Map = new Map(); let plugin: FailoverPlugin; @@ -76,10 +80,11 @@ function initializePlugin( readerFailoverHandler?: ClusterAwareReaderFailoverHandler, writerFailoverHandler?: ClusterAwareWriterFailoverHandler ): void { + when(mockServicesContainer.pluginService).thenReturn(mockPluginServiceInstance); plugin = readerFailoverHandler && writerFailoverHandler - ? new FailoverPlugin(mockPluginServiceInstance, properties, new RdsUtils(), readerFailoverHandler, writerFailoverHandler) - : new FailoverPlugin(mockPluginServiceInstance, properties, new RdsUtils()); + ? new FailoverPlugin(mockServicesContainerInstance, properties, new RdsUtils(), readerFailoverHandler, writerFailoverHandler) + : new FailoverPlugin(mockServicesContainerInstance, properties, new RdsUtils()); } describe("reader failover handler", () => { diff --git a/tests/unit/read_write_splitting.test.ts b/tests/unit/read_write_splitting.test.ts index 612435e1..775ca583 100644 --- a/tests/unit/read_write_splitting.test.ts +++ b/tests/unit/read_write_splitting.test.ts @@ -405,6 +405,7 @@ describe("reader write splitting test", () => { when(mockPluginService.getCurrentHostInfo()).thenReturn(writerHostUnknownRole); when(mockPluginService.acceptsStrategy(anything(), anything())).thenReturn(true); + when(mockPluginService.getHostRole(anything())).thenResolve(HostRole.UNKNOWN); when(mockHostListProviderService.isDynamicHostListProvider()).thenReturn(true); const target = new TestReadWriteSplitting( diff --git a/tests/unit/writer_failover_handler.test.ts b/tests/unit/writer_failover_handler.test.ts index bb07d9d9..6daf3986 100644 --- a/tests/unit/writer_failover_handler.test.ts +++ b/tests/unit/writer_failover_handler.test.ts @@ -28,6 +28,10 @@ import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { DriverDialect } from "../../common/lib/driver_dialect/driver_dialect"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { PluginManager } from "../../common/lib/plugin_manager"; +import { ServiceUtils } from "../../common/lib/utils/service_utils"; +import { HostListProviderService } from "../../common/lib/host_list_provider_service"; const builder = new HostInfoBuilder({ hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }); @@ -45,6 +49,7 @@ const mockClientInstance = instance(mockClient); const mockPluginService = mock(PluginServiceImpl); const mockReaderFailover = mock(ClusterAwareReaderFailoverHandler); const mockDriverDialect: DriverDialect = mock(MySQL2DriverDialect); +const mockPluginManager = mock(PluginManager); const mockTargetClient = { client: 123 }; const mockClientWrapper: ClientWrapper = new MySQLClientWrapper( @@ -62,7 +67,23 @@ const mockClientWrapperB: ClientWrapper = new MySQLClientWrapper( mockDriverDialect ); +const mockServicesContainer = { + pluginService: null as any, + storageService: null as any, + monitorService: null as any, + eventPublisher: null as any, + defaultConnectionProvider: null as any, + telemetryFactory: null as any, + pluginManager: null as any, + hostListProviderService: null as any, + importantEventService: null as any +} as FullServicesContainer; + describe("writer failover handler", () => { + const originalServiceUtils = ServiceUtils.instance; + const mockServiceUtils = mock(ServiceUtils); + const mockHostListProviderService = mock(); + beforeEach(() => { writer.addAlias("writer-host"); newWriterHost.addAlias("new-writer-host"); @@ -70,11 +91,30 @@ describe("writer failover handler", () => { readerB.addAlias("reader-b-host"); when(mockPluginService.getDialect()).thenReturn(new PgDatabaseDialect()); + + // Mock ServiceUtils.createMinimalServiceContainerFrom to return a container + // that uses the same mock plugin service for both TaskA and TaskB. + const mockPluginManagerInstance = instance(mockPluginManager); + when(mockPluginManager.init()).thenResolve(); + when(mockServiceUtils.createMinimalServiceContainerFrom(anything(), anything())).thenReturn({ + pluginService: instance(mockPluginService), + pluginManager: mockPluginManagerInstance, + hostListProviderService: instance(mockHostListProviderService) + } as unknown as FullServicesContainer); + + // Replace the singleton instance with the mock. + Object.defineProperty(ServiceUtils, "instance", { get: () => instance(mockServiceUtils) }); }); afterEach(() => { reset(mockPluginService); reset(mockReaderFailover); + reset(mockPluginManager); + reset(mockServiceUtils); + reset(mockHostListProviderService); + + // Restore the original singleton instance. + Object.defineProperty(ServiceUtils, "instance", { get: () => originalServiceUtils }); }); it("test reconnect to writer - task B reader error", async () => { @@ -86,7 +126,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -111,7 +159,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 5000, 5000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 5000, + 5000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -138,7 +194,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 2000, + 2000 + ); const result: WriterFailoverResult = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -168,6 +232,7 @@ describe("writer failover handler", () => { const target: ClusterAwareWriterFailoverHandler = new ClusterAwareWriterFailoverHandler( mockPluginServiceInstance, + mockServicesContainer, mockReaderFailoverInstance, properties, 60000, @@ -202,7 +267,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 60000, 5000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 60000, + 5000, + 2000 + ); const result: WriterFailoverResult = await target.failover(topology); expect(result.isConnected).toBe(true); @@ -238,7 +311,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const startTime = Date.now(); const result = await target.failover(topology); @@ -264,7 +345,15 @@ describe("writer failover handler", () => { const mockReaderFailoverInstance = instance(mockReaderFailover); const mockPluginServiceInstance = instance(mockPluginService); - const target = new ClusterAwareWriterFailoverHandler(mockPluginServiceInstance, mockReaderFailoverInstance, properties, 5000, 2000, 2000); + const target = new ClusterAwareWriterFailoverHandler( + mockPluginServiceInstance, + mockServicesContainer, + mockReaderFailoverInstance, + properties, + 5000, + 2000, + 2000 + ); const result = await target.failover(topology); expect(result.isConnected).toBe(false); From 207a1c992d4ab4c91d259d9cb0bdab51efaecd71 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 20 Apr 2026 17:59:57 -0700 Subject: [PATCH 15/20] docs: GDB support (#629) --- docs/{Documentation.md => README.md} | 3 + .../images/cluster_id_one_cluster_example.png | Bin 0 -> 98984 bytes .../images/cluster_id_two_cluster_example.png | Bin 0 -> 90477 bytes docs/using-the-nodejs-wrapper/ClusterId.md | 239 ++++++++++++++++++ .../UsingTheFederatedAuthPlugin.md | 19 ++ .../UsingTheGdbFailoverPlugin.md | 122 +++++++++ .../UsingTheGdbReadWriteSplittingPlugin.md | 57 +++++ .../UsingTheIamAuthenticationPlugin.md | 19 ++ .../using-plugins/UsingTheOktaAuthPlugin.md | 19 ++ 9 files changed, 478 insertions(+) rename docs/{Documentation.md => README.md} (93%) create mode 100644 docs/images/cluster_id_one_cluster_example.png create mode 100644 docs/images/cluster_id_two_cluster_example.png create mode 100644 docs/using-the-nodejs-wrapper/ClusterId.md create mode 100644 docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md create mode 100644 docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md diff --git a/docs/Documentation.md b/docs/README.md similarity index 93% rename from docs/Documentation.md rename to docs/README.md index 056c05bd..cbb466a7 100644 --- a/docs/Documentation.md +++ b/docs/README.md @@ -6,6 +6,7 @@ - [Telemetry](../docs/using-the-nodejs-wrapper/Telemetry.md) - [Database Dialects](../docs/using-the-nodejs-wrapper/DatabaseDialects.md) - [Driver Dialects](../docs/using-the-nodejs-wrapper/DriverDialects.md) + - [Cluster ID](../docs/using-the-nodejs-wrapper/ClusterId.md) - [Configuration Presets](../docs/using-the-nodejs-wrapper/ConfigurationPresets.md) - [Session State](../docs/using-the-nodejs-wrapper/SessionState.md) - [Support for RDS Multi-AZ DB Cluster](../docs/using-the-nodejs-wrapper/SupportForRDSMultiAzDBCluster.md) @@ -15,8 +16,10 @@ - [Failover Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFailoverPlugin.md) - [Failover Configuration Guide](./using-the-nodejs-wrapper/FailoverConfigurationGuide.md) - [Failover2 Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFailover2Plugin.md) + - [GDB Failover Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md) - [Host Monitoring Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheHostMonitoringPlugin.md) - [Read-Write Splitting Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheReadWriteSplittingPlugin.md) + - [GDB Read-Write Splitting Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md) - [Fastest Response Strategy Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFastestResponseStrategyPlugin.md) - [Okta Authentication Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md) - [Federated Authentication Plugin](./using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md) diff --git a/docs/images/cluster_id_one_cluster_example.png b/docs/images/cluster_id_one_cluster_example.png new file mode 100644 index 0000000000000000000000000000000000000000..aeae773b287cfc5fdcdd1dafcd8d229b92cf01ad GIT binary patch literal 98984 zcmaI8bzD^K^FIvBDj)~~Do8DeNVjw>h`34{bS)y%-OYlEN-RiAiwa10x3JPncXvp4 zK4&rS&-eFw{6p7s_FUJ@TyrMgGqZ1%6=9bzQC`Br!n*wM!M(>=Sm1apEF22Fi@=q4 zIvhz@SXZzf-jjOjjJ* zzuS4eE5W+-PE75d%s9>kl55&H?{0#hhhWj)%D~reYgaryQF$yqKiA$G7cI)?bglZ3zD0+P|KQh0^b*)S0sQ)94do>C&&6N&Dei$( z8G)}Rdf-twn5*2k%|RpPYK!gioX?4Y1$(BbV3jymw@OOA9ea z58QV-lvO(aeqtxy0`lLLZlO5<21loOo&bAY2OYBfS>~6d17S5aPo{Ss=DU45`$vSo zKNd(JK4=qKFree+RKYJCh01Q|RZ3t)q9H@bcfPem3DY#~WgnRvt zb4KPvP&(;4<`n-&iKc)9lDX%$N68_r(bBzF)`whmFLC81Z%mQBf9Ws{9}B=n#EO%GK8?oxeilIQ|jZ zFBM#plQbhWw#BGe1&)K>e;%ix2KC)M*M>^q0Mhna+3A7*B`Sd<*fmI>q$mak1qG?h zwCJ5~n(mJ6WH~X?Y_k-9xJyHJ@hj(X;~8*s_-I-0%$0u)x7n|km6bHrYwFuu9;>m6 zsZ^dbr-tMYYdVX4U3twClswPwX}kR>OusrCwH*|6?PNgBQRhHoO<>Yp%^{>8k~Y)Z znd^gZv49+X!78W4)`37wjpC5!{}zQYDTaYh$@Ul>wghQcAgKg7IjfI1dns+F-ZEVz zw@f#@V*?^jNxmQWb!e==a@9e)PuIu2XFR|dw|>vCC87axwvSR-x0XL-eVb0beFEnm z>Tz0A)K$ zuAl~g@{~kU#OKBC6x8};{gbf^_@e-kC!El0NImYcH_GFAysxtVQ7#%d!r3$k6b&0- z$^BBXy(sV<=;Pe`jZI^AhJadTl|yP-j~KhO#SK=m50oDeCxOtGh^a z+^#ZKY0wjf`s!d5msO-?S!yUkE?jKP=4viHN9}R4tf%xiB{BRXOCZIo$u?uG>8@y6jvHvWstMns<-4$SU#le1imHNA9Ddxa2 zb^*SQSKvBfvi?9b6u5w(#MAaXnB9o1KK}f~u0wo}kh2cIIma}SFFA(BykdrtOT+NO zHMTb;H7yNz&*h_e4X@GA^0bf#X=JBc6mEjeibfrWd&jV!7S%Sekq3PuvHxn`je?$T z(8XbmFh*)Wj_lReenacNUYjo7hq{!HLnM@_<+VY!XyH~3D1c*Malfux1Ci*8ceJ8@ z_)xm;&R2VRGzmnQ8Cs-DR56=qz^@gSE>kALWxJlpMCG=U3sZ)k$YT+B6j56hbl#VO zzbi<(-mc%iw}N}pTB-<{D=^jC?39MicNZZ^D$E;^a`6Yn=zDLl zrXQBn8!|4q40VQPWPpo z?)9+EW@KhtG#^u@XL2>3Vu(`C*RJd7dZ~b^3NKrjAhY;bouffyZ8H)cKr1eS7j|!h z>nOT7gM09qKgYY+!{arDn6JI+gB=Uj?e*_ZWN*B)`;d4U5xZGhCyXjZxim#7w~deZ zX+##G&k7JHdG;y(o*eW15)%XxlTt9QI$#-LuMNcEgBI6MD)GKE3^<*h#}jM# zN8$5Wr%r+bkM4&6ZXnd#0b?1Qg8;eCtCwIT4X?qg*e%QsbnjHndf9u0hNc%%&9WT6 zSGv$b{wl_zNQ2t0m1lN^SuYk4>81VP7J!K>%1z|0kmCv5P)hPh?O5BIhTG(YD}o=^ zEm>5p&@X(1wI#_GmfAb23L+CJ(|gPETUeP@uh>Yn$}&aOaz-L}9Oh4NN=oIh&hI&D zNpLN2`KS+k$L8F1BOQ80RLgp7*s?Lx6gqh4r8SWq=QQETahsKA=;&4Vmxdy#``&yN z`eGFGcUElmybik)g50yVXRJzzJ7?F5KeEL6i7)mIy@;BH6S1d7$_kOE8uVpo#GQuf z`V-ziI@&DO^vE>AUO@T>jS%%4+^J$&%1VAN;RM)ZvbWBR;hM}jAK$8wtO)v8KkAU-tqSwV za^Tm{8`2_x`9No{-Q)J9A>VT0Cv^x78V9H0sdffcJ+Nx`&sy0UWAoty-}Kx`*ZT}7 zLY?5haZpjy>t-7?O3=q^F?F1j7ZLHbQJlv0P>3`15?*Z|AKTVfDm$^@s}=Vt7QXUo zt*}|Lu)5l7x$rPM{ey##&;jr zyv~lA$bFuI)4OV~cwE`PiFMz5Zg~9EaWm4O%kTSV`fSJ4_Cfjf^%3W%$0PCqD9WHeRbQw%dS9=Lx^-_)=S5rD|US;!itnS#{ zHOdV8NgEo_bHho)g$yy~$fl!=AiA779t6Zke#I#1D2#5iTPf_e1#-;nsokbfDmEHKE($ zGHW>5swr@S znAt=E^59a=&9e6^T;yA`kvpXKmKGKcJf2oa`_WA=@$RzhFT5Uj<6Tu&^x92^)6xNxV?2mr2o9h$%g{>Tc4e(0uqX`Uzx~cW1y>an|#;b+le$n{Y>|UMA5%opVn6qHVR;=~lwp zv0;PeNoL>cu`j*ATU1MQH0wE2`FwqfWVn?oPV;%<5E4Q|<8h{y-QeW#>vS!o;K5WT z6DGk3tvDd4@dTIPwWRBmnTaFyY?!J=<1{yO5FGXzCen$?fKM;+9&zZ`dAO>&9+`MW zR(rD3CO`CYWrZQZamuu36EBUWQggII~*Id?SjQJaph$ z%f_T1q+K6b=a248(N=u{h-1DWt7rjzdq_2{zNObi{x!i^bHyglnr#!ZWp+E^bUP(Z z>PzfvniZR*nC{JpJ1aBOo3GeR4#>w3hjq;z7`4CG2f2gz_gg%x-WVkiyeJxds8;2q z_Jzv@Jhf?>PtCmCOk{4hn4HzT_gP_^<-J8+QN&hWwa3V?-G>@tuKBE)`tfgVg5%UP z^8$oDMSTKf8I%3XcyGczYMoAtHid^{PCt6=SVBD~luFO?t3TOD%o!AuF5qDMpC<@4 zSeV>N63)+tu|n@9tBhHzx!!tke)`U~Y=^D$r2fTkEI=h%jU$j(Tl-^+3@y3aCwgu9 zO^Mr^1Ge(6pY09FJ(?fU>L(O5Z|cx=-yXU@m}iZxr}2diV(ZGOG{HXnT#jZ?buQRp z@O{MA;729l&0t=>q5D)1f%q}d7SFtsDZ7jFKnGtSM@D!kbZ0`6vd&Y{$e(s`|6RwY z+}S95u-m*EO!e80FbwxMb)iI`1d7==8`{zyJa7`}n9a;l&ODi!v0pKy z4-}L6l~5rX834<&ad;1jdD*u8vDQNL4HaN8Sk5ekuUJGwc51F(OR~-ai-ls%9N@$L zCU*%y+`mJX*nmlO2rnYWdTK9$L2_X%)%mYrsr%RXyUbh)2*iKya~^SFURu z0R&}y3s|2rO>kQVlzFeF=vP|%%cKF47{(;e7gM|y{wI0I#PEqb_;;QWTo&Vhmc4+9 z;(t-_lYu*Y*d72Rq)N)YUO@Vi>Muo@2e07|+&;fTAq#B(`hAtdpF=}9-h^xY7KX7X zaC<1L#UkhD@^p7E{yAEJcnU(`_C?=32MpN@AH0SCy$J{ifD8~R5kEAk`rJSfzXbL# zhl%lEO1|}{9oJ3e{v(w0uOb?6C8}{K0WQ16DfjaKu#e`$=086sFd%*$DV3OicU}<1 zmivVtX}-o#uqaFQ69@nVdV+8K=g>1#T%Y;3?xPB{}m+K9+S7~L@wrlU&njX z{!9E%EjPuu-0QM|>B&a_lJJXJ09^WFk%UP3@K=L@3OF%-CyFDCvuVPkiY>u+`Q27d)i+|$5!CO@}F%rPd;|4WH61xCTX zu-V@OluaCT4g>zwkaK1v-7pF^b}}xJS^O`;_N@`nfdnvzeJ=N>CJ(Gqqv^~A-Hh(1_cJ-{{{*@k}fI+BU(7G_+GMXxbC$ zLR8Quj22dP>YooF2p|q13n0(r2Nm5nTOdF5^ZZy{wyd;ue6&k#Ty5uVgV;dW_-uUS zzb~QHi3h$Ywr)S}D>s!vVD-+9tDGa={Y9Je+1b{vs~kDbS6Go0D6DTajCr+mhRp@PO_Mj?~}por?3Xj)!^} zj1x8p*Ob)k*9g@**Ot`oPja&&PDv@yHQl|%t;i>yyy605*7g-Ay>5-V9Whs8Ebp-P zX}GnW9$L5nl5D{HcMtBMJI|jCe`DG2Cf>$9xqF6I+E$C9%ZUaKH4sceP644{qTr$s zq<~VuOz$KOVO{Zto=AG3xweT5J32(+|i5<@%|- zYrHSS3kh=_F0VLmu{k0~odznfKYS&~l&x_D(vil?jl_*4jiG(!h6cr#R@YM?3&_S@ z)ivOayw6Yra8)Z*=iFs{TIdV4_QUL1ARS|o48Eld1z|)Oz!1Q+lb`X~kwLkm<_VK% z#PrF2m)4ZZ(T^}yA%_f=L4lbFl1D2^P>GJ>hbvG2NVv@+~FezbLwcy~Wv$eU46>vs4Dm z*TOq2J;wm105+Y)C|xhm`fJhSWh)(jKz5=eaB6)qeQlEzM`8!9$;(dr#D!daP&BxJ z3xQy)Ij4y?hu@@91;0mp{oC*wM14I9*ze& z5CGm)iYp+@b^%;CkEN0+EUrK&2(3y4-^i5#km?;_>_)Q2YmJe&t=_E?VMX?o*#Du5-1V0df*7sfVZjWdn~pi@WMhyiw*nN$ljP zD@VQz5TPoQ!nO6?Z^%M2%7rmh+M_%O3|bUW0!)`H0W(4wfi-<;TCQGX;BHXujWGw) z_b)K7?ky08zQr0OHY1jiGqEuvrXn&FbpG98{*jA&F_NP=O2iUn>;e?x72`oBfE7y> zM9dYT0EW9pGL4DWCJEQ()>XL}=@2jIYcN;YPX+Q5tY16v6-xmD0uTkAUv-XOe3VR< z>Nc_XHYNV8L!WMb=dgIesYW>vE|{HDQORkH&?N&90yiD>xuGljxnAHq4TqZmb^T4T z|D|~dKlO$Ku~^g7|6^p)#^7zBj6y+)an!HN3;z^GFnXqgKVW%o8I2h*jyg5p;umNS z@TMN_4f|s(;c!fqNWBU$%jagR=e{JzZ;Oi&UI888M>z)41Res)1;Lj46At`}DI_sf zsISDZOhQ8J%zsJvz@~tkvLSHrxCRB~O4Z zN;fg?It{y?)SQJW&l9gaOK@E>*R=aXC9v4St(uDN181fn_-UJ%7~jFx8kY-acP0Hz zcp+&j7eM6B>>#=ikJn?lhwB@cmzP6p#o1!X=4-d9B{nXkQ^n-xT=T{EuQyL>54NUC zl8FKt<*qYU#}_Zt+SJF&Y*bB4LNqBl_K)Y%5xH!MPH!db!wU{* za_7aHn&@ks7VtOeZx2e)-k+zCcy{Zd&X`T2nA1&l#2tr@hb9eI^L70N&LxDOUcY>ot6{}2^RLOOz0X_bH20&d@fI~D5 zk6%fm@v5FeKAe7qOC43zE3wb(ezctOW#TH^M~Tg*(CPg>NW9M@4X-&4!3L}B5}xnV z{-M0WYhRXAP&|;+=bDdp78r;dUd^8g8Is^Vlj$6s*9}wE78t&5e#Ja(@>KYdL2 zhOq6)43|i0tkzGZHU^1cput2Id?84-1SEy8bfYzJrgU#4Jd3|XO0f(G>IyCbI+59l zbTE;bjhv(+ax)4Tg(nN`xfwYQL8@oJ6^NJ)|6oEHAUXVTR}+S#{Z>nCVlLF3W<(c% zs*vM4khyg8N8WIJZRGc{qe`Lu$E9xV8QZDj*j%E_gYJuLL_{E6^Cz{O66z0S(Del9BLM+lyDr_ehofrzXzBJO}tc$7r?w4KnPJV?N05P z;W+hzum8f4gw-*tU{#YEpgeP!%%k)uJ8O)L`6m3{L@HZ7*ru#`_*tDQ&$F3SmAq0n zso=M$x!qxlmTd~!^bhJu_Iq*pzTRlo3#LMJ-uFsZ-D?SJd5SFP1s?x38;Ohf1xM~; z(6|sBYlZP70f34JKnTM~xN~IuRIZS43T%tFaAhD(O|-@2*-|f)1&e#QE1dPKR(MwU!v;Q;7dEk#@<7Zm(4~u#qm2ybu$B zQl)(9pL#Ww?fxUiN3Zkyn&DIkzb>eKt0zPL>oM;PjRN>%kaNeoJ)WIwbyB$1ES1d1 zU#PMa)-*mUj&*)xuodG(8QAUn=!J7#$yPNo6mH%mVj6;prib$=V8obyWdcRI-}#a~ zASG}}pwYoI40JY-Z5RM67JUEX1?~Exg3EaTO?Vx9F0EB94he%9J6@lN%N#$Rys)U5C>k^e1iWoB0LykeYSop7bsfo;8qRMa$a=&2 zgF0__&MQKH~|*B)Itg=ccQ zA)ZM@qG8gu5+VTBH$i4%F8qbCs{3M4I{vI%SDI@39gLp%%6P*sk=b&pZ#qYyv>G2Z zK5>^qhReuLka%ho#I|3$1v;D|`F<0LMO4c+6U8tdWvQ4bz5d4iW%#`?DReU7qPg&G zn%dhYD9Fif!BDtA{$|SAmK8#8Ac@+C)9{KtGfYGx)bWrdLD6u4D^eOasjYxU-z!uB z6-r%2bxZ$^HSpfuxrIIVrBx+=hKF6Wg`%S(%)N*@6cu66Y}cwyPE!6BsilX$=L0G43T6=Hj1_aW8y1EJF=xOb0?nLJqlrCYuNBo`tcCUWPQL826XPkkn} zd=pYeE&m~U@ke&T2i0O;gi+z*#v0T;To3nyON+wBFq(Z9Rn)MVYWT6*lb zeZuDjU@m!jS-^4XnXkk@ne(x=#keVzEqY(3P2gq7!2V#q;9UH1pd8(k<}m9ufUDfg8!>VMcOv8Rd{_ zxXe{4E+3upl_I8wtN+-Gi*$gA!xl*r*U9NUck;?sAn-dp3KF5EW&x?#tWD>R?{7kq zRh#BE1byV~*2+aIQ;?`Jy!D*Bi>snWxY$W5mwK!vvEJH_9{AkwF)(*~UvZN_eC*m? zI)9IUFeCT^eg=v3n(v?d9F2oGexsw?`&1!Lk_ejyB-*=q^uBS4S@mJ=`!}#D=a5bZWRjs6yUzTqu!uxV$&6O+G4Nu|{ zw+3pKb&F7}Yn`c~i)GQZFx~4dag$XPvsr>%UO&>r$gy)0t2CF=dM(w8T($Jn9G9Zo z%jKL`lH&!;a(myfn{H^)04~i4RAC;jcglWdN;SiDU}9+f&gl-x8<8n-wlP$X z9Y-%ek;uNnqRgb=sI|q6$c=e*jc?PoVgfJ`1LawQZ|NEt>@bb7$$Lk}Muoh+UB-OX zrYHvA3fy<(^Qzr^=Vp_E^t%L4ar?BhV!A`iep5xvMsXi+;S&E>y}`57{YZ-vpd5VB z4@i+~Cr3yNt@~rhzqtyQGM07U2Uu5l;H^hKrS{|a#_F{=!K0w)T7pjDsSPMC&b`++ z*OA1s-K1!n-=}>s7+3|w>0}X|dh=d;sc*9m?ymoey;HF}Q&L8U#3AO5F4>u*^3@}o z3c@iNsJZ>#BE&DVBVjv5(5#s>AlVC9h!e-iuQLQ)3}(Ka(u;iZ^8nlXC?SK|Ha zV5H|Dao2b0oVOe=9pSk831apv9Xuj z=f0F^`O>4AURym;O#>nT_f}*~Az#aN2==2{c?8pFHq2KX#I;Y06v$Dc-Jt^tA*EPT zJuQxD2`Al32lkuzR0{?%C+5h_@Vn(rFybV6jrrkmq0d{#9~Fdw3V_m$rAEKl zc~t;eAp&JAO+G4SAGtEoD7hPdP5PLc{{XONKTT{5-a^i2F~=*r8X@13?ZIW}|1~}J zeWzS5GKa53R+NdF4i<7CQl?;ukj99s%@d*)6G8;2Psv)>xATL}10xT{8ZqQS7Z0q8 zr!76oEhJ+8aqDoDT#~eFmY<8pflx~R4@=G>^T96W;b&DS_ZZ%=22MMnmmW(mJoT3O z@wVWA2&rN;QW=T~AUSbcue-S1y%{aq>v$G5By3km;!d7ek_VKrhPITQtB2$;Zv1{7 z7C)J)r^Mm1-l@IVcFoBxm1`G8bn}`!?lZ_;=OA1y+05@0U9B)X-d5}Od22n}A-8pk zf6+c*jNeVwT*PxLQIVF7ZY>5{L3iT513eSST$*+z{0LT8JWJSmEKF-L;eXqUW&PDw z0_4nnqMNf^>FqJWeM9K!^qGf{I4gn}&lBS!2d9ZdkZf>b=*HWGwno~rL82oK<_@kK zokX-Y`?bYN1iG?vT5y|Ft879}-8_lmvCS{?5ediZ8580RJt(*PS^M3eS<kIX1$dU(B&+t#!xg{3&_dWp%!@1OY5U#Z?tS4lJo+poX&Ui&Y>%5*tvgq*&d7mqmqojJ;v z!t8eU<~Cawqy49-wkA4aZC_aPC5pkUo>s5l7>7Dd;dVKYT0HY?O<4)e zGAfZcTq0MxDg0H(4&5HM=m#Kh3*aG<^>5xX>po6ulJVZ14PIrO0z=QA5V>6iksHNr zwOlo3WiyE6q6UdCtmKdP&{2fx;fcOjc^xudFBOcKAamHt>+Li^MJPC|Zs}eo+Ki-F zTUIYHmRdK2iEw!^A`kaB88SaW+}7N%G#;+wh*GbI4i7NAX;dh|484i|639OdMcak%;#Q= z>>HdmRal(x7(CvJCbr}IP|CT*v5kHh{h4_4kgvb>OB8`>`9=Dix>)g!mgp9~6eCVq zjk!1bZJrvGys(Lir?O-xn#7%NDS@nZXm4ooX zbPLx_qe9R%d|`A~_xnLjAs!z14VxkvsA^I9$HQKtgH4fx?X}M+wp?HF6;)Q%q0AYL zgA1A(@uJ!Tu+6RoTgrLx{9Vp|mFlO5Av=E)Mr{12Ov)BPI%&~XnzlHR;Ywi7C^BBb z=cli?4YM?|QXuK-R5a=few}6> z4h)-Bn0I7?Wx*qv42U8kF_KjMoTax+wQ;YI5<_YCv0i?T7bB9n6Gta$^dk3(NhBAo zs^m2d{xhS(%MNraPl9`8dmd8Oxp9;bNb2&ZZ6`nW6X+D^8nkWoG^Bf3e#`$lJ6*b7 zS17bMV4kZv7MEbCJ=sz3Y%?WYB0`;j$p?`^S$F+zf@+ayzeZ^s|5il%EdX8|prh0H z5%Nv8ymGdQTx6Fd;{FrimI@6C1xohYibD}Ix3(Ehny|Hk2D7()@B9;0Bzl3P)HX}{ zxx7x3;q3_tcNdnm`P`nwqe2?p!%HFkIK5d?=*bs6+4_=P1he0lWnD#K)DNmRwISn! z^CHUt9zi@;{)y*|qu3XVOtAQ2i`~j(J?tJwzjN3lKy_;AF{)E_Oa1M)IDG3ZLLRP> zn2oh%;egn723rDJ524RC#XhNSBFqwf5b@jUwU1XEPdgm9woqz1!W<_{L@m~$;jX3i z^9Zu{YMOE-=0yTnIl?U3AC zQ6W2C407VEU!sXPek`c>RhVYAKopYywA6N1T~El+t>7rWeD0HzW5|6EyrcCAPF3AH zdCGYLF85o31}|YugAo_X!?FcCv~kxEJ{l6MS$gHBdYsD`H>d*&NN|yF$$U=T!WEj? z+9G0Z)+Ql{z{kT1aB;Hg_gf}`fori`HSYYc`e?))-y zFAG~64qY$btpwbO)_l3{4e1+l_+wAEm{KrdOLfKUkoWR?_tpp|ni5Mm#8Nw7e!z9O zhnPB)2sdkEz%~PuhA1Uw;wVj=u0*!WJKc^t-rT1IguCf;GX1T?gM;&Ehs$UCXnu^` zMW#tKNaV|tgxU0wQFt#wx(nXR8NI@K%d2N-{fEq47hu^Bmb$&F9yq@h;KKWsVikAz zY9eZTm2isdV82$nj7EZW{$78ZqRpvE_|X%rqe0GS>^U0`qfPI6yGkfc=iU( zXW+NbAZSHL@ISVX9#wHS%|1jJxLd-e*s8I8WIT;X5rh<3w8=kZgS)aKB6h(lMMVm0@w3Z3Ii(mqA%o^snz+b?d zk;)#2Gge?O8m7$0P^Abv5yP*G;Xj`FPhgKKxI~2H0hRBt`I5&EP!gf`Mbyc@7DL{| z)R7kN@6#F%Y~H-P-6@SH8;oMm?op!jAzH+n&G7|dJ%?#CPH9S@jqbJh%W}?zB&5z} zXFccL(%41`k$ojv3$e`3#CN~3fYWasxQ7eR%V+2{peauvM?Eqc_5-#7I@|diKG0pMIYZenxBDO9;Eq6s6Zbp+h=3REb zMeLO?-kdBga(8rJ5(8mvM@!a|&rJE$?4KQVT&vy=rI~w#Pum3l%xr+o_PmpUJIrCA z?<-7r%;;`3kQpipsJ!yOz)4wBubWnr+`5l|?#p;qg|<~;N}Ou-ae6jiF zt&BmZR(b3YF-M14CV^-_`auB`7f5-lUXzgoY9#nhqm2CnSQNoym|}{&lk1E=+mV(f zf#W)MIo=LDvY&n5AuhL1gDmCy9?R&Fnq#om@XxsKFwg zppmOEdmDcLX?^o;T8n8C!Jzz}uaP@Pd7pJXg~RnE8i>s&$k`H?X!h%h&X3R=-}vW@ zhAe>Fg~JQ#IIB;eoL(!<8v2&qn&juHCZhBC$&01bz2nG+(eB-^faeYZX2H<=90RUAY-;}9M$R)d{_^3$E>MiIeH1WS@0`U09L^LmX zM}Ej~jknR!=x-{D0Ie)|M`6OJ3&p5R!Ep+3#QNpRJY`}RCePjU+79E$txXBX^~pjm z4^MauuW+Ej()Tj+v>5xB9$l=T(;X)mjwQUh{2uF+TU2^#K5U?gm37;f(!3s&<|ksO zIW{L&BpfKvN%Zo>R(Ln}RalF1guyXN=~%|E>f6PlE4!W2%)aQ0%}IJJ14)?+kFHZK zgtg4uZw^3D!jxc~(kJ`!2dhi9Uv45~LA1q8*ghiR)j}v@g%NI}N9&JAY-x`cj2;&* z8J;w6_6DVq(Q`!_wD2tM%T#^>Kqqf-8Bm3I=b{MZ-Pib`fU=5Ta-B6B4_mFWi!2|}HGgq`{rEeR5)o}jjbqgV`Z(T#1Q;K4J2W(8uj<`bp+2z33&dlAk2B*{ zSa*H{;6S;79(zu`wJ^#~7~`S!9!gcX1hM&>I_O_O_|(C zT(AK1U29w~JfTPZ5=RPwm3!je87eO;Og)3gbVM5iM!9vJYY9Z2188mn4zp(6DJ zQ}P3Qbp=|;8**V)EQ%5*Zg2UdK4R)t`~JlH9b8P}K*pYyb*xC@^OTIqBBrY>9e13w za)ulm`Dhs#L=M^i8U^*zU8g0zeTT}j;9*_}bV2YIWS--2IU9O=N6-Y^j!y5@)7>sN zAklWmK#B7N4hArcS^m{CvG4(F%X;pUIa^^Y(rUr|xCHrXYoob3Kvld*$5#R9;lLtg zit=`eaJiX5bkn$&S0RWb;>}099cUKbn2FR-Cybi(_&qV339~e1-ngaKMJEQd(KS*; zsP^q)dM>$AbEg*he9u_{T&Hy$FV-#LtOnxk=Blt^DcagQKp(CX6MOY598hu|XZo8U z(IxbLD~pli#gt7h219X>z$}Wu?#R}2%u$87mPa*$L`bv3Sl>K%TLh}rIW|3=SNuRqIknu;-mo}UaacVQj5Aa} z{X!UWgh9IYF>wD-Y&RQ@+(@l@0bHs*tL#S>t?H*hZ7=)Qs*}EnPDzk-{);ivex>TK zTSO#mU!228AozrHu&XWMDu>})tlefCf{_k?t2>9%iQ&!dAA}9U=Y$|gc-4ygacTEQ z+p#!1*6ui@5w9g98VYcFh~f%=ag8(`u0DDfi<$8{P*q+N znkJZyxE8SKmeVZ7v^inZz0-~3Wfxt!=VoJG(Ddc`9&@R@>dX4?AIr=WVm``9P$&UXE8v1M;-L$!3hWgNU|# zYJ+!-=cz?6iGcIHmB8!)(Z#$xzsQ$LK=!c>k~B9kSbWb|E9(EIqIx>jz-Ny$qZ^pf_U#<$oR^6Y)eKXK$$Dha^EJM@DE(mFHe{aoP^&*-dLhG& zv`OE+I~F|p>D~;ndUoHno^^h_PBL zg&0Zo*!V1s3keJ%O;jp_KR)mkKk9K>ryBd<{*BltcwNe`hEyUKsask->8JN;uTO6U z@HI7(BTt=%-v>mUXt%RpqSrELvW#sDZ90W;5ED|OJvFfRQabk*#Q2=xc z;SBlNV%avx?A<)9&H3BXJ{c4uCJUqU!H$QGIImnv(+-gyWS>d$Reqf}V>tZDXm|lA z^gn>1rJ*U)1fK6z6~40-BY^%^@!`aHZGn_WAg9oKFJ+R6J zBk>67d?3VM2wP6FuyX4^@~xn88snIFHJYak5m@I*nHB^f)h6kb2v@XJByUe7a)ZBL zM&c-ZCqy^RFV57pw~OuM6|TivGwza^aIVy!K7+)2iMA49^YB)(E$vX@t6Pk7mM&ur z+Ud_cW3FN z?)-ENMAMm zQZaGfU#Id@O_#}1)qrNsUjI*yzpd)8E<)h`4pX8G{vCdx)~zt=Ap)df|2$4ei*1H! zRi#6WVS+WSXTVV8pL#H%IJWrjWfEC1_5XL4T@QbSx6`EP|ID=7W6FhU&*d?}9oikd z{i{jumyHjWlm;rjJ<6j=e3{ZU5&ux|`y=`g!F2RL51kX%#{?&!dJj|xUawq_JE!$8 z2f%>wwl=253)I6|LaqOrB!Hnl2Dtw(VMxmYUL<&`*!DN;Lw?>IzX4WYY6dgqd$d!^ z@lT)OZ@vLQ05hSR2rnW9VtzKz7eGFRWcz!8{SB5{*$_;-iYeDq8N0zQ#*JKJ+H1t^ z32KbRN1vc)r|$fPRGBrjoX2#}Qh7UCU(ZJ0Zjl47zFXu*zSP9vzP zV8Lp>Mk@*WQ|Dd_9+Q|K)BDNe@gTmQy%42%YE<>v`qkj(9d^wUy3>ud(v47pGwNj` zs)*3g_F~18yLp0JrzxV{W`iz3JBQ-#Ya~Uqsz7cT%Tc2&qx*foM6&e!tcuBa$<@lE z_V4a^B9$jQPo@XRByRNR?Vh-peUK;qjRlmil3*;~Knb3hKKXcB`t|W1AOso;^KLs= zIs`V*3v42pWBRL!3oQuBkhqDqt!W{;1Y!j1;eE~d;op#q6uL!Q;$?}+2CjIg74QIs zH7RA*ev~Y|J#+o_CYP1S*)_Fe9va$>c;ZhS=}}$n&BQ=`!AFuU)U~jQeRpiFfyMjY zmBsbspr97d6aTB*%{xX3SzE{y=pvVQ)YVb<)409z{ubzNsr^YVk|}CMbOs*0&E}Vj3N5et^|R6&3C94 zb&duG=T=Jkpr9UmuV3TnKWa{`0N{xEyr96?3<=a7GIln6V&Ik1?&WbhIyeLp+5C0 zRkW6GUKp;JHi;Zf`F72#><-0v%WPg|R*~4$eKc$JQ!%T2Pb>3T-dPJq0%uIe(sX#X z$CLGa+MVPCNNM@{F(jq;*+D0C^SlHec+Z6lk2kng^hKrM)A|Y^Cl96*}k;uz}9a!Uay(61=I;1r@1|;jOx~ zaKCV@O7<_9A#Q{x***r{zkq(v+wIG-X-!#uw_0l5l6cU|^52s^u=b zRqu-F%UlX;OYocRqid#C(Y=^}VfK<~@@EYD0j58HY_C zlTwK;^k?eX+{7=90rbp95yN z!)_^$^C%3KM{DOqO#4e~mhWT6XR)+n%+ z!lbRf{vRFuF1Iv2^egWknD$BHOMXqmX_|v-xIs?;U^g&ac0YhfuP>v$4Cs{vO$K?+ zNuRf{P@>}3n-Wj+*0f8sk5iVrvYGV-`&E3+zW()LQa9oJfwFWRg3%DZgcrJP$7O=2 zz>rfjU$KM4>3o4{u^?eD;D{-G0sr2(;8*5S8%GM_z^zzGyu(fAPM=y=#1+0S_n_iV zMEQ3xoYkVW%HV~oJHx*gkFh$`7bf#yJ_~=C9fFuhZe(qLA&aH+_3mMYilE%=`9t0 z!41GXar8|d=Mk?REm-AajGikAha;XuZbf7{Cw1g2EHY%#hT;`_=Wl`28hl1!y|14` z^Cupani92c8kMB$15IcON;MaM$>jBW$#~!0e%$%#6Gmf9XZs^eToo3d8VacheG_R( z_H+^XkXCv1g8cyrXytCr!K=+f@}24F;K&} zo$2{?*T-UPqo8H=$<#Zj4rRsFTy4(bCDRrL%L*jR|HFFu>bXp9h%dpQP+OwQMC6 z82~_+4iN3c!l&QT zA~F=d|3I49qj~o50Yi31DDq*;PRMJ{S!N_nkQs^j(+K*@M|ekuq67C6#EPgRw?=<8 zS7X!!_}LrZQPBLfziKH-8+Gh~K%HkPG|G#Qm31Fc7fJA*`nB*BaO9mG^48%gnNGC`s(d`%mQF0HXZlE12efldE2ugTzAh<>AA| z!?(~>prt;)^Gp*jTtQ%i+G|30vhBk#2lT)@~oquW5UXw z@bgD>0!=emv4OQr=g_w`F-`d2z!Yi}dLK&g6{baiv4T59%c;QCt3R*SX~dZ`pRbjF zuuw983ZwcOA_DXc_Qcdo3>#nu4~{4kEM3g^QO1rP1~3ZkuF?*)tK&jS3n zkeu^h`kEi^J3+X1(Ow}EaObygpHCKiQ!w`Ik14nSaH>EF6%n|RudD4f?4j`PUNJz( zeTEufq68RAFs>^62*pfiXytko0jCE*_&=vJeD&Opdhv{$>nl$*t^66eg}_6Z*MAOY zIGUCJ_yNTXXIL-p99hw*X^V=kdKvThy~BhP5@bKeMi%o)2zEbU##PRA$&1c&H&Se1 zS_YU^d7M{{Baby^?RSQy#@{oS34mi1a1dr3CV&k`JA6(nti~H0FEH->ykaXcZpuk* z?Adynj1qMWBchhBCX#v@ZyJBP8Qx<;!-iKwg_*li{dE3oO^KqQ^m!gn?jY`X?yQU~ z{borL@1G+vPsN7-$D$d71g_E*2YAx!So>dLVJ2(P5C@*GA+_{y4pL*RKQla+Cms)%VmV>Pk6^>tIkmvi zRO4ZWpD=W6w?w@f{(zYyN^EX&nG*f-&6k)fAmMa(sAO*3Ddn0npwNVa=aN$d1rUrp z(vO8#z(?T6aO%aUrgd>IoNkBw4@c<=PLdGvGo;D zQEl(r0;7N^DT34x0@9$;FqDKMB}z9EB1m_Gf>HxgBIy-G=@RLb?(XjHj`{Y0c)7p- zch6dP;mn+KcE0cP*0Zl%{v|2@BFI!um%AwU^0M^a@U~3cQ8&jz|5)L1EzTll#-Xl6 zHoNUpd-Ek=^$up+G50y>?W&s{eHbc%D^}AhFteAh@Vd&3|5! zdn=DCl>Ycl20wtW?H0|3KuqvVmaHb#iF{gi!;*V)smz zF#Q7iZwt?!Bd?9B77B{|K%^Dp3FrU0Q#c;^yodL}8v=*<5C1d~Ik@dlA=?)WVV>7P z7a2VqF6v-~nQ*giUTS6_5@FM9d%=(mGd4iYN9M!+?LWS;I=Fo>f62^&A3v+*@;7Hc z@~i{{^osqJkaDE+zhBmhj}%_`0{%V?l=$YGgA(6-p{tPp83#bS&@TaE^7jBdB)Pu$ z-yyc6(-}JRjU<>M(PMN*{+GXR(y>MT_IE&m{sC>FTrl~=3n$Ny!g;wEwcQE{8Z{yZ zDS?PVKtLte{|xC4GDJ>s>nDc=B#-%jzXXe^3;#RsdUqa6_upjROA~5Bm+%i{@i7=o z!B_uVr7e7dVT{BMz{3vwg4>$LXqRrh7Uyw>OwJ~b2#ofjXv<%t-UeR9Fp&Lwv;iztFC#aaJ{C^YUrPMycQx3PfmxPQ0wR-hs9fVe8DJA#x~>i zID_RnlA%vH7tg_FM!+SMV~I0*=@7UdAL4FHv-gS4SB~nBi(mbfhhIHLKYKpt zEUv@hLbzwLt;R<8!`N?Mk+b#$LZMkQd2PLx$s8jxXBa3g6Y7L#EXXA;`ZyMas)Bq| zhMNq}WkN7-JsahR*x5~Y*)1Rx&YKG_V>Sd<3Lut$ETS&*mD;RGwxZNl>}V8s-lw#ODK@VS%}cyfu(1irjYSp5oHV z$2`Ahqto`Y-_NcWrI+MoBfFS_U$9n5yHU5WmssMh#m;d)1?=*mk2i0ruJk_AsowFe z>7FwWv1u!t|28hoPJY&>gd~hSE&T{KZHhgil80RPYBy|qH^~7bWi_%C-S`FS@Uq|x zeV5|a=uS$k-n7$FqN@`i=?$KU*|4!Ov!h!yl{TcIL zFkr77Rv%_@pZsB@w-uqA9=N$Iq(3%@kL8vbs*87_Ta7B`$tYd%q#olKT4QNL0hgx) z4|?lSwb@3C_AoBOwYA4`v#<}MFX+uBN5+mY)IMu!I|M`ErVaUEq_7Mq8{SO%s!h!1 zb~?Mz5$JmxM)7N`m@*^I(*1ZfrTDz2+X4H=IB$jKx}uwvjLUSidm!6wB6=ajRI`Ia z$5d*-m_Xdd3YIzZ;Y91Fz4gw1!>taEtvJc9t1Xh{JAM0&j(xSf<*^FW{Dw!RGrZ&A zYR8zX5IPlPdsBSW;V|A|kUcTKqgSq3TW2&@Di;>dqhgS-cP`6r@PSG34jR5*Ov7$f zYC8!ueNA%EFn^iTvj3#M@7f*7GAFANllTwkweJGx*Vf7U_Z*{ZunB5w>GXZ;s18@Bc@CkksoXr{Sq0>&uW#sS zD$#ms;SuMze$JjLP{^Lcj{cpxH==fFR(;D|e0f%7ijyi3k85fgwtkKA1P&Yz@1o1b zN9-Dn6thcY{IWZKbtq_gu(OwxhvrY&ZG@P*x)Fuwr-wQ8uWqy|uNn*8)Db#x8S zTtmc>N<-Zld*|oppYm^zD^BjP4bn-#HGVxvFXxZAY*w-WL7?NQ7;ZZzQr3t`dW_@4 zyJ&GDlH0~RZhAdq3Oc?_vmx9E_O@+{;SL!{%{!A$W>PwfTfg3O_Ns6gsOSd^yk%suX$~yD361Mo$>#a5K)4{YKz_e*Dr6MsQ{3Y z0rlkUo0kgIa+XQP#{3=ZK7UEgW23fGM1PInJ};0Y2^Wsu|0xhB!=g#SG{znmbo}QT zXX7NihwN`#od(dOA28l{#qA9bN-8#C&K0DP#_clPuW|mm!>U`1ZC)=n6#oD#P=TN9jBkC zeBEtlg_tIviS6_DlOn@^KCy2?0cx|O*Ezb35|9!V83*vL9#Fe0AM5esFW9&MZl1T1 z)yxKSMVHb#F1iW&)?ZZ#7~Ni3II902P?|Drguw1JV-K6g=F2AthY9)#{AC5B1B) z_SQrWH>ptsxBWE{7kdl2=5#ngN0h|$ydaBW!sk3;Q578K{ha^M@F4`Zn03FsFOj(; z?$#;&2lf1!2mnl1a2?c2d;Y>xz`m@+Va9Hv@{5l&vs^2oFa2ZG-_|1ebQIB$tI{L( zAL;XtLB^)V_LBX=;7w(I+m>3gv!Q&CL51v^3Lxm~x{mF3VHL@X=;Q`U{JyrNti6*r zVXVymVti8`HBef(?ToabyI_eiCEB|dw!U%GG z{i66)diK#3lq3n%G$)b=CL=HMRIkny$0~_TUgD@0CPGRBl-ieIw6ZW%0OvH=!=s3z z&c$y*?{P|We@WP@BK|Z3`9PAlVDDaAc<*B&M*rvI7Z|454eyC_Pm37p?D=VLoC8SY zP=exb_V=Tt4kVBITQqtevC;c_RJ(vi(&3;552y*1LWE+b(VxPXJ2ox zs7Eo8_VFZR=ta2D4^3gg}7Jy!LW zNRU2wx-;Hd1K2>P2;%o2U4LknRe4P_f&*yx|H>kt&sX}cLjEa`9e46vx|RVg2qHEa zO9Q?#qOo{AMb9sKVN`IpDfGKtFmkx_3>RS#0vDzH*BF-?a+%g7Irq4SI{x z3R|>391Tee!H!;eQ{Zrr4;Kzak4wpK_zlD7K^$-KEAwTC<--jxO!+@zHxCmfgLJcz zK7)QRae&KO3Pa^pE^;5H2`U?vtCBQ57!*i7`Nm!PqqUb4NwM%p3oeTcXcO2TEbo7j{PhLwP_-K?0s0d)_zh2?rbsrqpBdRc z$zrGz=SI~x@D}#;xVtHelCM0aB|3N}a?n9^=3}`0orzC4^)4bDe!c%m@~Z zk6+0pI()mkI?;c{9TWI^Fs0Zm?HmPwB*njs+h<}v0r-hXjH?FE5_3RV+x^n5;7_C+R{IP#NyMra>3A82xPgcwo2c5@`MqBgB z7WIx2ys`agr6X(xQ}V45Cwp~4{Hp~(@z^}Brm$lWQ|@jfhOb4x+Sf<3;oD{AT;~rP`+YOl!Ijsv-KcYT{e8G5D3Rq03n6gAFCf2KQ;DCUw{rZ-3ldvE5Ht`7!NE&g8prw)!8*ai2Rk(LL{UcKmqS#N?BM?S$cZgC{{Q!Jkkd z?H{Km_0f+?T3;0NeJSb^X+YG^1_FA6)3Ec=H$nQ&=69ow$(p-;ccG{2zNyzNtR@=s zCyZM4)pu_~=UQW>HyK6mxQMM2xGO8n&w6aVh0SVw7IJr^T()MlkbQo8L?15L=_n$P zKgNUJ-r}?zDnY>)BNpb#a5}?cCX&3&f3Th;uxmAnNgvZ*n36FJYt%;<@!iC*m4`{^ zi)j@K9L#@=`_*5?>WFmQpD%IaTS%##M87SDDkC9mPuos}3PCF_g)rM=trc38Vw>J)q zxDBw_DyYg>wR`FGS=Nz9qRK<^vf{%r(_9u6;~RZ9!ZTfH*BR(>!zg(aSR;qFe7H-f z(i5EeibunHx*~Q#3Df%)Y1YEGBX_kAJ0qW|Vq*w|?M!U>DVI>s5WLi}#o_7@*Ac$6 z+rX9y`ARV;ti|G`x%{H*U5l&0 zBW4Tst!RVd!lBmloziSFm37MHhdrr;wC<_t`F_4oHy)n4O65 z@(#ZeSIKib@6fwPMx&nKHx-Zg6-IQ5EV1#ogbK@uJ$Hotb;IKob9ZSJ#!kQ?y+ITC zl{^nU{uR^?(=GctR>RCIqTOsy+=n!5nf;Y)gC7J@8F`_k`=YQ@BZEV-k3?#&$e+x(`Fy|E5WH@LTG;y1%x8r>AME-bGE1I&j8)j**b z{94F6gNro6l!;_vRL1LNd!=mY*BXc>1^6U`-(jr+g+j;qFnj8KC-+=&c`9$)D$viUdWvUx>OfNFvvT~La!0H@ zu*e|@E8ano(mem&5O}*Cs_xNRx%*~pUJmYd6_*?Ca=jx&Sj91P>+u{Q>$qi#`8y=S zn{}!-P`Z+$QJxE3#F#WcB*(SUjuz-#oI#Nx*v1FBM+bO}krX@adPP|zU=WAonR z)n(FG(_cW*s?;o2rdj>~ouHvqdvM7z_mLo`2qmhyDvtX^vtL?_(x@R6!|RzG44nt3 zpPu~74cA&OY1U}?z@{-}WyUi9D+7jHqxh0$pOgDyDpt*olH(uDSX@#L-kF`t%WEG! z@&RB}LxxJi!<^2rRXZCh?>)t`U^*SI4woFi0%Mr}chCS$bjN1dkj8 zDhv^MA42LdT8t31#|EWjJXG=>9vO^T$$Lq* zev^x*NHeM0TV3v)9${FNU=0J12%n#&`aj$(eC=uo#W8?NzNz;ZmXr-`aBH ztq<#JRQHlDx*cTp@e`y)~I%?chbihjbAS|LJ36rJuJgv*m() z!Z{$=7%{`nYo#Jt9})roxzxDDVJ$~R^k(3cn*dLiGxmj^L?zK>?mbIOAQ$nRSBr2TL)3MT;VVr49}5N zrlEHpS$|_{V9|B<@RPQcHq)$XP~aBfdPw}OyJe9xe_uIJ&c00d_4nUS7lVe0`Vb;lRC_zZYwKk0Wqq3>*jSqE%Cr-(HXWxtEw z{+<8GW39WvkL;7-(KHP@xsCkcj!^$cIcLn5>r$kiK_<4g;kC{w4&50s+J>Pgjrlmd z)4|)Kd&m96J0X%M`ynejXiB`$@?2c`#u2eGQ~gdv z$D5v}JSvGklGfJ~ariwN_WfJvN%GBQeWLQFqJeAbofBR#^}(k@KWMOA+ArLBz;t_n zG~I5LCc?qr1O)B27`V@=3!Y<>1P4v~pOoBxD~#t!_}0}1SE(>-@EvkML{PkF!jf+s z0A{O@I?LNikro_nDMU);(>D3tlDw!8$afXaFmHuvC^+jwgcPi6yDUjn(`f-O2?JM( zI%?wIG~SS@94&iw3L2&(4_5eg2P-Oj*EbnyHWYetsw~S^-Z_X%Jz}8Et@q9?nzi|_ zaEWwfueLB2BCne(49OGpaQ=@HU|5XH+R9l4ult|ore zP{C%Jr7g3`>uWwki91K^_Ka=`{Chogb6UaYh1z~(8+j+5L*&jp&M_ShsMFoWQ>lLE zn%`aRb~~u}wR*4hze(sM?*KHESin31={CJUsol7P!kVC}8s?2}AX)I94vuB-brbU3 z!3ev`t#?K*wl1o@MoYF$-G)I725J;}h0K|jnifnulF4@Dp|85%2~N8M^@S$XbTB{8 zy%1J&?Uj^lGPCFid%e?qj-09uL+FxNr2gGxT{MO{P)i;R)CI{*u!dwS|4z4JS7v*- zSwiGhL_5ozK2CAslytASZyxgupzoeBo-=}NMLoAyg`a4|0a`Ed#+e2tYij6^ZV{Yv zz)esu3FlIPErqLo3GH&$6CJ|fo6t^b0nF+GQeR=)-)V+ghLDH;e8A%C4(}SM^4?}X zEA4d}Q0_c4nB+F2)R6Au4d8~`}h0Z@$8Gc zflJW?=@3nD-~_Cy;B7zg<>mg59$&+vFnf{O3dgXaOs*+y5MWh6Fsw6sg%YG4(%q6MS@W z^jY_#@e{UBw})EXHHQyNvBj^&p@~^wH}jb{saWlqn;bfmGt&Zpt@xnRAFygmm4J6{ zCkjvD6Q3?O_iu1^5dxau8y&ps+d0ltqhruv`AT|V0P`CzMV&dUfYbTEE%V0f*K6z} zW!!%`{-aVBcVvMSMf$(q=(FGBa##(Yc1TqwJE6xd|4Xj&Z!gs1Jy=Y2P*WlCt9J(N zVRsx%#tFpxE@1Va6|ntO_$(nIl>j`cw2Wd;%g%!NPStp*XL8R`?OB@iMeDW$J(B+1 zee;pdG~H3%Z80*J`|I7p^lvjRRuZ7#Rl8)yzb9SSO*=m0LE%4XlMpNQ-^^tq5V z8^W2GH(snaR48T^r^sG?m`mbm6t95`*Rt0ec0M)S7|#>y+;z8IzVHQ{Rz;VoK;mxk zBH5QOUwLi36)~R<*thg{qv$ia9JxS{z?cH{=JbL*1hkQ}9kabP!$Q(%)cjlG3t+02 zKI|nemJS$MWXbNZTT^_A4Afn|+Lh+6Z0XEaH~Xue{1%8hu;SNd>rP!61Q_xfB3ABN zoWK$vrVaRS6=xP(`=8+s$JuO04y(`;z0{M|zz=N4x&cE6U$Z-Q#m66qc%Q{Nk+DwX zC7c?ImY*DL{n?RyOLw-W86HLdK-+Ls;M0~3k`rGovO*^VIW?09gc$tRy!G%uy!~c!9_Ni|{u+%}pNX&# z59l{U6ztZj^NrA(Ek=(LZ9!dl`mKheRF|8h5%SA!;}bt_I1=pf@klY{3{D~<%Xhvy z^!dU<2h~R9FmIlU2zA+S6P0z9OGmK2w#Yo1qi_{hcYb}9b$xuOAjQ1%ns+$F7^8JTp2=Ic>>R5qLI(rK7qTjvY2l{^2EO} zlzAWFIPN(|8}5{wQMQ$|?tWr8ZmP!{naX|3W5WkHwKQ0_d(*~D{2my#u9|k%!CjS) z=Skz21StArPt}vxC^QScBYZypMiogYUXAUeH+L!kK?@P!_TDg4R}^ zYyCO;`_J>FV4tYPT+b-no&R4~+ILEMzUHl20XVAx=P_%lR>3_)yYuMn@}sSA&UCd& z(bKnWu@rmFu~1W}`|*eF)?eh-&ujwXKPpLm@itTHC9~?r3{ifRy$K;tEJ1ze+KFrP zvCKB>+enG^*GZ^mMGgR^`_V_HlfgGwqR`5zkYW>e6VAWDWVz?MSNDNF z{_!(rm4dsw?kCRA#=>27phFmDPaBhPOkZqy629n`;k_Ga&{QZ)3J6JFE$gE@f1;0l z_|~P9@tw^k-gbj8%86gFtRVYQFYjF445EM)=00dYmGcTA#k8~T%{O*jMu4UsR%hO5 zC?xe&z%DBgbZF%(oTU?YyAe>Az^4J*bs|-Fl&UV8dLm9l8ixv@>YuYP6M8V$XF8# zEoF*!fyEhxcUw!xxf#q3h8G^M84+pq9(<<|$*lRMjfdfx)wm{D)`a;<XThw`?@{3qvx9_{+G7wIpBU?JTtnW?Z)Sg=Wh zz_&;qgYs;+d12-=&v0muOF~ACbVSWDse8Oj1$V(m*=!+=jW(NBc_OA2scM*dYpL2| zl3W(1exgg+biU4my_IZ9irZ|X_VE2}-!anS)jUljZ4245tt@3pt2HTiB#8tz^rOsu3U|cYoyf;-{)MvKx2Z*?(7b>B#3}pXqyJ zEL4{d=7PE|pTEa#YD4Es#0I_x8r`pd`=NoNYC58Uiy%y5$1zF!&6xsCXuvpj8+>6o z?!POmMS$B-=@HiMk5g8RQ6E6UE3n_nLKIta(z_6h<)VaXX>D-4h2uwR*!&c}*Yb&% zK3(e0&k+>QEWV^p910v#ey5WJY*g)o4f;lW4=NQ8fJO-?(N6=QzbsAJXA6^gxdwHf z1_M9b{4*b?A5dK=RH}|He{a|}x!@E&XM5*UMcrcpZa8^<6!E^4u(`xSNJ+Eq=5FZs z`wM}01n=uqYFr)a4!Q5VpU}3B6Y;%jYQQKmIbm#RbY~*Z>B9Azh2DfV8Z_FP{n~kJvzk7LliMW_XjeJo6Fw(3P#f^3 zHPXaYoi%#TTKg%3*Lz6DC;gUthv2xxG(|J@7B^088lTKQ)fhJ~%sJe-Yd7krU8pcP zXA*|Y!&})_Wdg{m?VAD#+?P(YOjtN(oVLRTu zo*3gR14u@@Np$sU$e*klb0~=ByxFB064`@xnKw6ZnnfkRwt@54OVm?su@c?ow{j!p zfNx`oDV)V=*zC95;M=UWk2b(5Qq(UZB?-^T`nm%As3Ac*kq;r{Hje>sG(fsK#u)5& zi19fh$+C4?DfV`vvf(R}&OBISO?h$A%)Egs@+A*mWw}yHiB-Ihma&*S%04c|jLdj@ zrKgks^mf|4f5!H;ViD~$9Qhcn*`G0dF!<)}43dktGY%OeD_M7&VCy5TE>nSPr^d~P zN8%0TY1&Kq$xhjdnPtF!ds42gY3InUeHZuL$*2@3_MXtXg$x}=SqEa9u!@n{CFO?& zc^JrhL-~j9ZaXf-1`rn;W1u!sI19+6rOM0~4E^pjq5~NrD)$4s=I>+b-m!0;3cBkk z#afr>KlO{k^%=XWXgvRSW_$W|Z}L@=k9D0}SN(5))ylO%XC2d>qW+TX^y$hZTP4@% zYhm%Hs8QYb3q@4i+?F3})1}f?8o3yGHETX~NuU^PjO8EQsAPk77!Jzvt%&2qyokh-lW^H7TX|EXKEM*UPvI z8!B7;tWMHuCoDNB>KK>gBi6W=sg^^PSk;?g1}O)^d|cUq?gZaT=me>iTj*e|WlLpw zRTxl5vH&jSF4~*U!~ME7Z7p4=Y=YPlPO4djE(e=7TOZc6%Z3_!ajF6zmTh731d^3$ z1%N%_5P(D8Uqw8|yBau-T&h+I+_u^;cHtDuifiur2V{(URIW|Dv53(u5p`pNnQLMI3STjhIg73f zAK{I^pk_C6cdQpsOT6Yt`g+2&g5t6t8eNVh(gJ}OS7ANm@Y7DP>1 z8~c(jrhi<|3zM+wEpGXzycw$zvsXAPqUg}^-!rX>BWZpra#Xl&U-Q4Qrh}{+PH+9hD2w zhNl1eDS6pz$L7E*SBeVB)XR$94o)(!aX)OQTIr|3kuK)~=af+`>%)&nG5*e49mUS9 z&O0m={5Onv!woyZr|E320weQPbtWr-GN@zjM0uWiUJ>f3`pF1XdfINmD3$5~|KtMw zEao(bO2LE7evRIDv+HeGuw?1fz51YEfey>HE--)f&6Ub>HEjb@)6@fD%@{G4u_?4Q z6nz8UEvm-tyFa_JzM8ncc*PHWpzO)QUGb)5W;bTI@Er&GeoOwZXMxqU2}iX zWCgAAJFR3>!JGezSpSu~eq{|r^mHoz@GtFBRs0U>;VV)3sI)ceQ-c3qUuoXrSI9|@ z#oUJvS6-!cQaSu~ICanb+ZQn3d z(jV)H2H@;h-M!;{Lg@0{(A!Bzc<2Y|ZL&6?7-TxP-m=bB|N2bcbuNR(F)b{cWYAds zntob7J=D!~bv(RtU`cBJDL*lCr@(lYPPEq`J}*ZzXsu}W<1MmAgTjSiWo5jV^$W{$ zHa1pHj!k>K+%MVzaVmAWM#M=>gzY*_xzGo@=UbeSm0ZtuQKA#6mLqMmCDnT8hz0K^ zD+t+sjBJ_}X#B^2`k_ruyTqUf%uF~7_-x@18>fq;ya}77*sxwrD?oSjq0V77u!H$f zI5|YHvr+U;<45>QC}3@KnV71xDI>xnomb4zAgW7L#6A-2@yE{3f+qks%S6D63D>JF zrsUOVEgOJMah0g>$fkTq>I@i$zC7hTQbpIs-L@h*>IcWHj{8sBs7w(#4sacM)4sl; zB;NZcNy``=)e*JA%0@{DsMF#h!#Vy+NNSnwW?gX)ZD<0vU3S|viY|5KTQ3(>A&6mr z=8I2Z>)75_32DVnR-`|oB?rIzMt;mJoc&yRJt=N5z>UQVr)RL5{5i2aH|(XZ)%y{F zg+>(4)%BjDNxw}X#@e+}&+`L4@1wkFxBR+nX6|61rmEHY+Se6LZl%2`|NOM8YcW?m z1M!BIp(Rt_HSSb~Zd%U3M93lx01$N$2#_3S+}X5rOADxL$57kgfmXqoTvZN2w^T^t zn|AVMO3@(JDsC+_)+0UD*Qp1eM62$~vBkZLDC^+vs90%`qtmX(kT~@hT!-d|B;iJIKH7INP=TI?_i74yAi^<)% zDU(BGZyZbxd^+_N>9S|Wn_B1<&+zzoSF1U>Eh}Yz59v4^n?4gSSlaSLDRx@xgy7W@ zQO2yN*7U|d?dqiF zPz*jAGNy4XG5@fqj`v9aCtfA}pcg~Nz~QZtU&)4wlM!^KbJXQ))R9d=kxP}>yH*q;UrguzfyR;a zSG%>gsV}akFgwoMD&z2Vdq3}0Xw`?RtMarOG@ee-*sTjK<**zH!<-Vk43Fz>dr>T4 za7i^iYo)mqhC?#JgRFS?PtJ6S3dp$+>5o~Mpoy{W@DT2s>-Y1SgJ}^3nM3zyzVE3N zyD0++K{*IA<*p@$KrWoTOw9w~5rL}F7jA-GKYR}zAn}s3M~$g*upSftt@p6TOyc?( zEA@IY`sBW%cOtAYw>+9p;=szRwn3NKFCV$zjI=t}YUD%}wNtjuSkuOTp zq-G@V+eMeSt}msK<*NO>Ln0UK0qw4L%bocoEc#$S-h~V>+1xLI>ENc41*hbXm^9r( zI5Yth29ilX|Jop}cGldp{KFl=V_X>n^0dQQqNPp7^~Pff0WKtg-9K`dEn!fGN08;& zhDRw}c^y{?KU_Hycb<3pI|21vxUx^MFRbd+x>sr0)*v*Tx?6;IS=ag08z-61>dFU} zg?<%X7Cv|4ug9yxURRVgtz@U6*$HHJxfV|y0c#xJW3k)J8u&-$r-C9{ys@sG!6s&3 zO;vrvAEmt&QMrlsps`bS(BEy>Gtd2Lc}U`mxA~p38qh;fR8O)S93wGd#djL46k#4~ z?WS=GU-Gh78Fgq{Veq%-(nA-@G2fRVW$aP}_Q_09BOqj)!8O&v@~oyWcTv)l*39#D z+0J^Wx(MXq<~>QLq>^1iHoq~s;*fK@VMA?P{=DkL{e~Gd9ckOOUw<#?7rXvf(MZizDu)5^1-7F zxOl{#j`xR_krDjrXI44+jU!RAF#-)FgX~umB5t_-_UaMuI(0L%@C=H10pSEzHhubV zfC-mTOCi1N{_sL2j)}Z-FOw6wRNzeX&Tn*o$+rBny*Di~3aS|GP+uyvLS_LJ{wbZqBWaFd@MwoWxs42XwYf4f3v={zS z)3iN5UeiJKiQES!p{#>1dnGIzRkOVWwdko9^=>v3|Mc58W-!2^J3LcILUPQxI9uk& zuFNWV#66#CPx*r^EcjF-46AZxK0N90hx4wF^=4dsY zvjJ|eyXO3}iIfSFd?#|ykhm~^oI&*#^iHAfo4(Got!vZ5N0|7XZ2N*st_0c9~ysMr^+{SyNN zXlRf{5MG6-%GDPuF{KDkUj><}Bi1Ie$7zP~9#F^aCQQk48lE;D3Pz6ghtH?e|CA0> zdNM-Jd}bgom`2#c6Uz}5FV7Vi*80oyJN!^H>evcw6bd(%@NwKTV#P!Siit?w)Pa)4 zqUdMq^A$uZfoeMZE4QGD+4%1(s;b1Ly{4!Ow}etZa}hr%+A0{tu0%t{PJ1|^tlTky z$1*5ZjPY{gJ)zU|%FmBj_X_<5VcWPt`+E%+&(L4H^8BC zwVJIYC*}Kt-2NzFo7Ncpo?f9(GQ)mi0L5@f3?PAVjW|^Io6W6wY3B;=zefZ}x1KnS zBLb+UBul&DS?<7>wb-F&80V^9@=_I7prKrbgxVqOm-;dKdz(Um?cI^_Sf zS0VP8bL$Cu68y;#(u~1K>!^z{-#)NYm%A5Ee5?t4^lif=XYNXLl+u3_=*=E^T)}NQ z=`o1ReT?!YFL%>>yz)~A4L_P5Q)pKsl)|VHz7*@UE7!?Lsnq*2cJvKH(_%bx_*%sz zM=DU%@u<>iCb};a%$Mc^VdYPx1;eQ7w`amrb5wqWy1w+M;42N!7EcHv6zLaM84%0L z26PnRuz-ox&L)g`)3%6<$_IJKlR?)l zKET`^wPQuGTfV-b4~|LG%y^LeC3UcMi*R9}uv-*28~wM*&=}3|V48EoP-fVMH zysqqfHwA9enIQvMm-;j4>0y?J=2^G}4Ui;aP4Q0Z<>t_zUH)cr*Y&90>z&NZmJYA; zWP?$y=3Xml+Tyh&3Lx&i2A*7cgpZF0@=(S=QIBw&u`!SCVwe&jB7M4e4uuA3oN135T_%x1#$@!rev_7h1zw7|+Y z&7C2RE@WOXU|33MvgCCKh8XG*#>$sB%h}?Y%k~-G4-Yps-8G2#6#i>aM2Qg}hiNL)kkQyepVl zUa!U&ifcCuyQbQ`wQn1_Am$kMZGi8;ZrEM(^WBv;clI*J%{slR+2~pZdRNL>!F73; zz1Q(iXWG@XEFKTr)oFg1Td`gZT8rQP@!YO;#CnR&J)S~iSnlZPCxh8xl2DMG>%sCT z&bhd8YO>;P#8$4HtS62aYJ>d3)aNCm-)!ZGp63+c7%Yrs=CTt%VGP+2p9F91LRQvA zA~x0ANC>%kfCQ&uw8RQ2hT$U*$`D(~pg{EYfK@=Oz59Oolpj`kbSIp5GwarSCL$LQ zJ$p}s?JV_IBFY zWC@kd>mNdnwuKg;PHjW@4?2IvvBSQN@pvy&e%}Jxd`z{^pzAc`&>Fbg>!1iH+v~gA zzsZiwUG^lqc%Sm~Mq>W+f$DAWBYo{2*Rz@vguzn8%zDPWWRHTq$4}+xrsci>R z?*^e~!;pDwN?gfFV%`YAjvR*D^~8u{?`QHeheB~6v$;sIu?ae4CP)R;_E()93@1pm z)u;q%z=w?~t{?>(Uv!jfotTMFT_@rIqz?SZVj4tTriI(>Fj5SQQ7Xuytq4O?32Ukn z=XX;$n@6>P!Br38etLN^KEC-$Zrf}*l6+N1QOyB6Wpytidbf?{Jg*smfyspqYv;Tq zJB^pZyho#4=;FuGJpD74gX6pwMM`IDOUc*bwr1WkAzBG;7 ztdqrzO*ArTeo;Gn{$7M~nhP^FsC4}XKM6cHDIFT*kUCI+$wwR9;LVASq_JmoT^qDuRU~5%TwsV5WRk!T` zHNZ5q-N}e9!S{P1++Rsib^X>d9wjCFqhN0va5fpdsQ(!{yf*vL$BHa>(eDMLW6u-d zj}#_^;UzYpU^Z?#P{Vb96*A~RFn~ge{#GUz8ox@ zJYAK0peJ10^h-h$*?F2IVfD`wKwX;`f97U z5UTY;lV6T91JKtHuLaoc)tcjv<_Zv_LbWB%tp1LBS|rwMr!vF#d$g8gnnZP__FB*D z*{0$Kh<=acBG^`orrfa(9EW+Ak}c;}gg)>a#gSUSK`1mwIm$Mdt)O!p^{&;C|8nCt z9NCN&J*78n;o*N1zp*0P>NjM5+I#(AFN$Mp%;;|6%C}6(kEC+&p^q`Qe~nr!J~>bu zwyN&)bDd7LY}(Og*;O_*FVvmMm!dg|SSo)u9cdtbn$EP-KkAoGS`O}TFwvdMD+qLY zBaqI+x~R2K*Bo6Ia;!E1b#r@gAdqTYeVvzoRm$w>O-JjRxXG58(9@N8>!ZS}aA;lZ zFGQZsc5cDz-nQ$82=)4dtL56iORe2hvr4{?A1;ZqGB_YIf~B!`-S17qIS^$+L?@Px z0z7M(KinxkyT1_Ftl-?J_wK9p^38x>#Tol_Xx-tZ{tUsYY9qP0mL$E(aOnOLI*x(>zKZQ6H%2qp_R5>;rDksp% zOUI_}q|MB9$`#2RM<9K>g|T*ONN?in1HbaA=*bS>Ot#voCBO2Og^hJIM1b!s;wv#6 z5pjAt*LEV^T)rn)^J)F4pqw~#ZuF%1q1Lc{UuM_7w_75>PkBTw5giMlXh+=8gVx@xqej zP+T11y}xAuOv-VGSd8@4hwzHaX5&N18W$J8iiTYXY2Vq?|EWcMffrS-R&R90*rde;Ux2hRXeRb(bDX4Nw zDWDSP_E^YvQ#Fo8?BPPkB!Af2vob{@!m)IJ|237e*9+Q*QKz!{>-uehl`hIcTg_7o z0v{$_?s(;PGty=;dQQHt>%wrzQoBmUUcCrqU3HB45b3{FfJ?Lv@2=*9Y5>J5-C=kKTIwCBK%cyJ9gczfe0(%Z` z(y@D!qmmLQi82Yz4}qU;2c;idZG@}Fk{-@Pjv-8J`s{2}u1X486fccBR?@sq8>Ft6 z;#=c)IzVB=@DhlvEgq;eWZC&gZ2W`MMu>i6giKc-#C2W2wt3FY5m#`O^O|C#zVE3I zr)$IY@3F~cyz@j}&jeB$90N+9L?~>)%W8W;Vt^F4bs09lIJ z6e$$S$`|KkVriL;FcvuNKXynf=+~hxa+p|QV!Qt)KoVN?W~Qblyyjg^-lW$goMBwp zVs4a|=&DF;v7f&$zrS$n@=)tzBCK3L|F=Vkla_J4aW1;Kn!B!smM)SU198?vaK8`R zjy2NoW@iM7x`!PmqkI8H z2~q8X*D#8M@TK_Ate1y;J1fFcbQ%y z;Dkq-m>x3@m%%Edz&L(}=!JrRDawQ2lPYq1*QvKrWxZXl-55S8m3a-YU-B zhvM!x{DX8~-##J8c1fq7`VmCC(Y<+f1JJ!Cx@jEl$BO?yw!Q=&s(+vnB7^wNi`N+m--TU%80ea?3tO0XP~sXzrBR1uN!1ioki~==fnZ8`=?f1>(l|Q zNhHk+vBq7LPn)KCumEz-R(n-4M7Pa7fl?RQUyHN%$QAdcJ*^JUbA3F=#w;=6ehZ>?0!z%R|`h z?OXb_d@%CcOt5*3QGD-GWPA?i9?4bRo^I2Hsqj?cL$q#0%J>vMz(Zuv&NmlZ0jb}Q zCVL;m6g~tGaKzzQOji#w6N)+$2QYGwkxJAH1Sg)nf%}S_aiRFboJ&XiLdo7Bq#6<* z5F?3fVNRFv2(yX02bDl}^KPsDGC$`k20qUWdD|AGvo(asf_OHy8@t)q&I94+Pv`E*!c2}aGaaUayS7>CT0HK-qop2GnGAgr?DJz7%cg^*+ zJ9;8;!&oMFC#E5-O20XpyMCWRXwu*df`qngotT+tRtV|urgyz}k+3MttQ)Hpx$&)W z#3sZd(k_b}&#gD{)_1ABwN>Z+jM?>hk=*0w3H?$u_hY#D^6m+8o%lr|sFwA2vJSh& zf)wN3;MrT$+$v{L5Gm~=O?4Hy(H5o1KfNjV*1`RTZxw0U zs`R$S!2po9c1gW=B}dBte%7#WXMNK{(qRGEgz{QEimtyTrJBcq zXu$DemH=M*rTaF(ze&{g!Cg+omyB~L%#`jQ}3aqpJqn}wyanYRqM+L@Wehf~uuhC-CeV<~XhbNFHO`85CX!ye*F3>EZa0JbzravXX~oQh1Wu|1p|%oyAioqv z{o*(~=b3R1PKNQvK5mb%OYu}lNeNXYBvdVr_tT!?ylhH*{lK{%u-D$SIoXuDFh}TA z`uId8ZuVnd=5nHtrgc~4ldFQ9Z&~9o?vZui=I*SPGl}eR>pHN=K0GwMFOkTTrjX2& zCO;;4-=!)bYQk!qO9r~kCF7gRC1YoEj=#?KoMq64k=vl1jPq>fyQoNO;fV14HK8{* z{hN*KF&HC1Y~RCS6WFsW#`Z7s%F$~^PAk*J^JB}cKb+1-$_<-Ue)RhMI!#?MlQ(xTjq%mPxvU-vrK2hsqqUQ>nV0SFP>y4;C;b;G<%m*G^`v~RU5+n z2xcrWv!zGO%vB#PYAYz*j{N3`B&+m42e*Iq%Ac5*QOgu6C>3fv@* z>SH2SD9XyTk9HD;-jp=gYxbtoCpaTUj-`h>!N z0*kTZJ6_26&w1BcT#iL?f*}qneELz^QK(kMa!Qd*E2cb9Fv>dZMU+D;2FVqVthvQk z$a5S~Tm+6T7mEGl92qS8F($_5u_mxmjusC)k$NQesu;^~ z5m&Gh(@4<<=2v+WiS1&XNc?e1-NvcBrKe{OAgx@*L#et?bzo}_HFX#x=Y<9b~Jm)K^NlrIB z!fdPtcxD};zi-%~ll-0+wMssb-w7WmScy!+zp5~WKi-Or(^*9e!tzI|X=NCvo~NkJ zt}0J1!!?6$tlpn@_QqDYCc7e_lJ1dif30P_U(r6@*>ZvOLq(1V}iBwmKzx*yLGN36tncQ zk`3U(*1tU0qdXZS^iiB_rw^}Gef3#NkZ2Ia=!UZe^^Z7wY`oMdki` zJOn%}USYjeT7gy)8Ikboa-S`gm3A76dwVpEZMstqDyq^#4aHu5zN&Si{lFUDW0c%5X8+Q(!)yQ%z;h|;lk47<_6bGcN><7 z>xOXrlt1CNVkam-Ss8UFY!zSE{PTPajPP0y!UW^FC}|)I_f*CH0NLN)NQa`Qr{Sk` zOihFd7a8RG%L;yO0;y-3ZoMf;-Pj-UZYxQ43x-c0)s#EOSu3JUK!?qO!}2aw<;%** zF;Lh`6knDJLCx|ebO_4pUO(MJRNM4eo`xqX7VlLho`mDDPM@a zeW&Zm1--l_NIJts4n7&a=&P89b{)GHm*Or@YPJZyZJmEF*HaDh)-xgtziKGn5>F|f zh<3wNvtiA(Dz8q% zK?AB<{@q4pMWCzES{c52#Y|W+BgCc*GUlRaFuR**cGp+q*9yHU+kxvg#QL<-yxEa*YI*1>89A7!ynuz zf-%a1Ex(+5HDLkdr7QSH=k-av-3vt5p)xsH*mP~=hHjz zHwemA68V!q*42#SfiF$u&Ptl-4wX08GV(yJ4yQkNI`+P*uw1+1g;#+Mn4-TQ7`pjE zOstuU>Xf)GN+lE3idGWs#|n5OC4zb!K}G64ObG5U%vU4p9B9NF`Y$@Si~F>0$7C(U zg_{hXqKV;auM9ar#sK`=$ii_+$&Rmj?+NN(T)N7am1= zJ~?b2&0u!PD%IOX7-E7$ctF0l8eXi#e(eflT3rKu@h)>QofP|)c=)Xte9LK$4SB}{ zo0ApQI)~7_Y8TdABj^72~a1$;nCv@Rl5+} zSPZ5+G>24Jy5Uqh*X3&%kUttOF(TwT`4uc`YV1iKAd@d{D(GRPBs~DqFY-JX;nXwI zt?BlSt{tvnt{+rq>g&VKxhBg0k(Q5p+Mn;R@l5IC1&vRGTCCY3R8Vs{m&q~7gno$( z^UU(an-j_}AD2mew^!Qt2#3HL0AcNm7_i^0jN+Wj%49wnMu);z5!(_l$}+EMDUlf+ zs5+lCqaunF2K&JZZ&{d_8+liK%g@6RwPJ}1lWEV1WJwjSxPaFIPo zQ}{VAsJ+_Tth|}gzxdW4*vR&Ocj zd-v@OAh}2CW>y^~%l{z5@-q@2r*9nsTQ)#*&nN7tHn=aU?Oa%%AiNIRPUJMKd^Oa5 zS;2wzMUJlKZJK+jNAzox?0gTKG=|XFZqfunds1o0Z}U9b73&g!R75?)$--4Ti_(6c zMw;&v%cZLjTY5XN3y>_epq}d>4{x_axtgvU?F`R8c(m(^@2A_k~C8~1*iVn=j@IVf37%wrvGqzkj+ZL z7m#(sH3L@5z4({M54?J&EjK9iVK4<4+fBKEkMDlSRU911eTzG+$u}lfCAz7Ra2u7d zZ31*>tst9}b@?i_E$hw{UTOHJId`Y|P7HZ9tEB&`E=`U$AgPuv80=+>K6&QdLLu%= zcqr6bFB__ZL6tqF_a%SqAYKIM)nIkFT+5?=#$SqUN{8~iR9WH};(dz0cbjG@>bAFjhfQk{FVQ4~U zet998Z+O4v{L)A#9XgzGeIX%%X4d=B<7$*`VPr(i*tZFX3YU#rz>w*btY0ll7OuCO z5G3feqinPAX$DL0fS1P0@@ljO;aQTh#Lv14I+Ax9EkKlh;@?A%y`Gr2YI3mG_$CER zDmOnxoO6%KBpD-%3Qd3>=lOIpu@i=#v$9aJeqg00esN2YX~OUh1^t2LJx5JeNV?@P z4uI3?9PNZyA1qHaaRjU>aH~3}7MCtA;*KiGhAV+&JV+wViPKbegP#xeRpGep1HN$5 z9C{Q_{LH;7!qCx|m}cG`sOQV3)aGq61(1+(VncFt$ZR0A(=K zQ6H6zfnFQp z6pSB*9X)SLy#k0$A(nlbr=asCU64wQu7Y~>;bb;l_yYlN;_8o1^ai$Jf_^kskee>zFRFC1#kv)i9DGdOB#cuW^AkDd}FGR zJK|5?z0-0Tf3N4f1rIa#9+ZSyl;kgGjV8NvuJi^kmQ`bnhCw^j6GxavhKc_6P#c5i(BGIy_>lG5po#qH{tQ3-VP++ zT@ipNiE=tWq&vjTPBTc)`oV-6BSySPIg5CA_$S7?r|Nw`GuoP@7&{h|Z7uYKfok%2 zbAlAOYN?23Z1JgG{VKL9H_6<(pXj|k@NOiq=jonqiPvZ~g_x$k9gBVbG*3!QfJ!O( z!jpyY{y96zF{5Rs!+e9D_HQz?9?Pmrro#cfBEy**EKY;j-`RlITv$VoBrUvZU$_XB=7%>W+H>f^J&<5-?B)-p8+~QxrPTTh%{O)+t2>7;z#8 zg%R{#wYk6=SLBHS5&0<0pbx5fJ|jXYrN&SFNZv~m`>&T}Hm@(s&fnK2!D2+RD7TLG z2394tCH!_fCk4xW(ZUsh?Us_Tp~OHs@z`@5#L-zEgMA^~jjvZv42YPgpO@t<;NzF>J(+`vm7Ew3S;C z$e1x@6H4#**8Go}bykrfMv}-0J?Hn{+-Zf#4Q2A<)GqpQOS>X{7@MaPZ#`U$y0R=k zWwWfhuDl$yZoZ0a`Z_EF{zW$2i-?GDZa@HYbqCIMT2N~9;al9tMqb|#kj6AgpIlsp z3%44y!Y6nITi?M%l&t91d{@$4oX+f+OFu|UMLgvxN2I1$8%|QYNf&Wrn2GeUB4EQJ zY0M?!c_`g}C~pUsM@FUge=+KiCgOSKUozn|z-==c5leUT8Hy5|Mr(3mkIG5P!&RoO zlm))#dbsN*j4g}&M}_`Pu(2L>UX_& zzm-ohm^RG1RV;2aaL4^P8G~rJQib2f1ybI1OlhC`5^Qr#O)1Y&QiEN9dQ0yV-35HS z7M80=@e>y=$`c8~CzE4&8~O5n9Db~1#xM|R9g6rdtV9N=Ncq5%Wg_t${YE1lf{zmX62D9L8=5M3VCWN1XvS1@XbTP-^A^Ec}WCWuXaw9nAc zo#-E9KSPqx=z`Y@^qJ#GTj}woYLv@H7Bgu4Kx}+%wDeo)4F8enpBIE|8-+_CZpw;@ zlOhc>KgL!omd8EGrHziwd~44Vu$azKYXCy=AqQpnwkJtk{nrgO+c zLdrqMI2Ey}g19)TW+hg{`Ox(!?65J$O0P&9!wK{z6C7($d-xCvFwwL7p%+C?#ta}f zf{kD*&-+!)IfC0*Ud?jc?EXAf4O+$Kd4&Z6AVRvAJ8TNEsWwo|EW*G2+&8B zM{KJl$-;$@$h3`o@OJpOG04aAS6^MD6K}!TGN?gC(mf59j{BM4zaf?+$EV;1PV;JZ zDLqvqnEGy|$ic11;~9>}(w9k0ZKfN^F2)K&6>ji3{?ik6bFwgnpbW%D!=&VAj|m|s z1g}~ZDu!3pYN9XlapCg_`WAjFBprUX?Pp!!1W#KJ8UKnz`HHNvnf{{)u9aEk3gV8h zdljXH6v4+qI=tuLZM~$@Ma*v5Ai0TQFjwUp$1;-X;6r2 zQ-PHw;L6!yxQV(44+Tn8&&oME>m*h}_@Jn_N1XYrwsKi=&y{CjeO}G{$=JXhr!syQwQ^HS-lYG3>udYP7IYDB>q_v?f~_ z+Qb%^NKiA}z0Dh~e`57#PYn_mM0&F!vQEP)Ly~WmO+Ci>zV-iX@O*Z@l7dB=0dt3>xa1u#z<$S_P}__onJCK1lI>6 zvUv+>1E@~n@-64jI*s+Vowylz2^D3Jl*1=N(B5u|@)NN^#M_ZClIqMG)mmb#W^yx% z+?HHCIkjB}1^_dzsA5q#zs~cS*Q(bKa+MGXwIoBl-Vk{`4qLE#{|*iiFO$NBm8+pr z<$%k2V$DjFH{aHNw78UH@#10$IBa_~_&{YG0sdNNCf#4lM0D9?gWiaho{lk-ESD_+7C?(Q?Z#eK5j2O0LVJkW$%AOw}=px zc-r;gMhuspPLk|*a_hgwbKQjEOO=^>HvW{$K1nhb2&(bR!-^DN37YN)X|9*Wtoys^L;$ToW74?m6GhncSq+aOb9L6 z5*qgsZL8nyFX{1@e=Q|#M_#i4_Ny6obJ@yr)sWkkF$IwzU>H3qr-`Z&x=rk{aMvx2 zk4~ai61Vh>JW{|4E%T1Ux9U@>T+P+H^=v9_dx};@OPx`fN)^KP$tzu^R)~pvuvkyU z-b~f;Z`^EC{*{g6m7dWnxNmpWuGGl)8qx_0zr0ZrJO1Rf>ljBe`w$Fa^ z@}VnXlP7>s%&?gPY4o--yP;OK-0f{WvB?E3mjT!-N*u|Fk0K+hzz5}yBdFNFm4(S^ zYjN$U3s#kmad!C!Qf291ZfQex#a*wiS;RxlQHltaKIQ2u01&UCH1vY9G2QP2w=Nw@ zE`wI(kE`VO;+0nImgiR1Z3BN!{6<|&iyDjxm8>?G)^;3fJv>uZaXfBiqJCWcw?xA)ZW(yK?{D!D5@?9mtb&k3riT)|m&gNT-&FJ>$ zW1`1HN{D37CZ*bmXnLeGI%34!$4x2mpuh!+k16ivvd?qFDGorL_XeT!U0xLLde1z` zGOX#hs)Qy)b=<<{&9J!z;bCn2$P-luXT@c@ILGI5_#cM|JM>k<*N=3~~#c)e1Xd zca#chh1|j-m#=wj&F~Nuf?3LA{?~CzR)%xR@(|_^A z3a)N)>P&?=#i)DgqFk0y zf2mXR)Tp~i@K8qXmlKmofV%HaqohJw#x+O4*S(!i|Ms$61yaYP3PHZ+T0A&Rs(<8& z9Wp5gW~i*}``UBFDMLNVNYKw-3+O)%5kV><`J~a|^5flMm#Nlatj^`{Toj%oFBs}r z8C;T}L>V6*0aew555*>2PU{}k7k?zlQbY--JYx!e8k#D=wm*fxjIXy>(i(NBKcHYy z#^X}L{@80Pd&cCKH?cp&yoxd({;sc!p0E+06}UL64}uOVg)))Kn`uESsfl@&iOhv}h29#uAgF-4*1jHfsm*5voNA$`ijrqssDyC0x z_G5RZ=y>O5`yRxa8TqAuyQMRB#@-qGOAOm`@Q@&}pXT}FuxYsKunwWUpUq5sX61LB zqZhQy=_PKjW+iT~9F~|^RG2W$0h0|{5nXMlV3%F>LJ1snB*k+^3|!1QE*D=Okt$3) zK&)SdiOeB1Pi^HNZ3=E|^;~mhTrpW*6|k4wNpm}qGq6J=063_(+;qd2#77O6o&%<{3y*zI7wbAYhh!tAuy0%GQ6Tx<}RcAtZ zX6K@6`*HzTO?LBCz-{T9@W9Q>ko=$03+MMj`LL4b&T(^Ys)f>Gg=EvHemyJ+w5$I^C9bYD!`+y_q&{%e>O`2BIBRbaBG zgv(8$X1%APHQ;wnKAhos2KWqJFJO0@Kw2^5cwk7GpgRnslVm0}>NaV|i)M{EkCvQE zN>P!?lnR%X5HYx9-~=tSOU2HqD7cg#LRk~E4=ZkMwQkMZcLDz3ymdVza!E%v!V@`jyW3<)FefH2f*Zi=a*OPApc;T)^y=X7=VkNrV z?{KDJC$N_vuC^H)*l)RaC=EK$JGOYR)5s8ZVPc9-%lpsQhU_zhyR_ryR@bzc+D=ys z@Xx-jE%G|D`x+;_?mS?dAZY1)WBvYW8^5a6h^TLHV5-<@u}S>anL~}(Ua`%yhj)5z z!t1RO?LA;8bGRDRBW5^8Rn$jS({fy~PR9#vu*ntR`hBc>xe~^q6p)p_5TuE}I|3LT zz%i3}J8fq**;fqksYYn-L|?vSis4pxc(WnZ@slI*wnOqnAC6WBu+-oeE=g_Pg2tqm zk?iw9OgD}OZ;+n9N|f&(%ZRR^po1H2B4OCqU5Z1>TX35sza&@1OZ^i`y=7gc*qLRO z=G{h`ga*+8mno6SYoBChbBDog?C;}3!f!G~CJPtH@bCbT5&>?9d(BwJo!7I#9i1fg z9bA7B50$r=n1B)UhessG{1q3O8myKH94^Et@-z@f2>SDuRNcOdmA4X_-ggC79)#&u zq0S_G#DW@7{7U@Z9SzkR>vteERIVj@CNZZ)1J(0X43FAjra^`&g(D*Y@%w>>56Aav zpnH^l*D-Yp2aX?CD5J0&Na6DeB`F++omNrdth9Q^Ry+0Hvp5ldVh1SH;e5`xRly*- ze1F>lfVgL@7F3`8pkfND7P6d$Ph@7r`MehlfdC+~pi|Bf(iDII z=?B9AGVVllRlDWzjmT5@d$O;U>!n1+YXw1*OsE1+S^k)<34m)!4k<;AO15h9%ysIsZPaKfXGB5*#Q|zLMWo{a@xlY>?e6n@L5`&KpO}{Af7aFGiDqNz37Vo ze#Wmg%AaU`w=8#$DQEO+-I98|E2M(U&mYn7z0b|RSQU8<2qv1Nnb_)`PV1_cgGccm8LGBc>_}~_xa)cZ3j6O*=p+DzsZXw&Mgd&D zH-tKz+=LvaH*A6?m;m8@PRzUMe4smV?3lWTz5Zk)?$Oq89%W^p*9&S*$96ND8WwHm zcLlX(!_7WPd)%LF$KbdIBB3S5XFE818km0%3#yJw7l~pP=(^zVqnyQ~&S9 zU~jS1qwqH;SkNsN01LWIUz@}J@27)XuB2K-{&f|&;mU;*X!7CM-*7PS!&|oA>XRq` zeGJL~cPw!gMQ{Q|1n00kx%|Ji0qDNgi36MVY$$7t<%z}ref6Z>!2qgigEDyQn{5LO zVVmUJlk-62t9O^fQFWftx}#cQwEmEml_YHYR6cm)gW(h1;0?|I*Y+g6Yl~usna6Wi z3UB=YrjAh)zS=$ctFzf>sEBOPv->=*_J3SP%W+4Aj$d|K(>076{HGj(SWWMLA6yX@ zTGQ~^1lIdEmy_Jvs?72Op^GzZucpwqTn`Spf9Twr2)@{o`a@YDDf~SoZV+-ODS;2@ zDNXKKfLI@{1eZziIybK?%P?6(HtrXvGacJ~jrf4dzvnTS+wLW+-?sBOE_-7rB% zYxi*qd!V_&zHdqgHsERYB#qR% zZPiWAME{Pfh?UIF&{uwY+s7-MW)ig@&Mo~cSRZSY+tdC&D@dFy?a|#`uRP%0S@Ta9 zl=Pv`7}vC*J>h`tsN37FNdXslbm6s|vCPp`-yTZtm|Xbd!c{@X-JMlaPXJ!~O3j3I6LQm1Hv;5ik6r*^jN{4BkR*Ux}MgbfGeDnaU%>48j|2=^whN6%3 zt1moJzSuo8e2n*-ceEW?m_K%XBhwFsUV$BVxL+=|N_agSEdq+d=y=U}sbBYmu`-@n zed8$x_we0%T$x+HD}0>5&q&X4XBPK)D|uxy&aJ6IR+pQgr@r$F&43b^)T&{IG8CCk z!DYq1Awna4ft0(A!&KgJ7_ox?XymR`<)}2j_ZaC^>akf`p4QRYuLdM)Np6(VB7AIZ zCd|Jg!l%2;BfVZ68MU4r=5!X22_iJFpE3kPT@3Qw?A}kmYO1%1lU9b6c)h}9Ua_~W z`0?+^NkOfn;$;=>TxZ^JUv)8oJ2FGZ4AE8 zG_%Y}9aD@5+I*wBJ=k;TdUa{O@(qr8&!>H8`^&+5cgdXq2I&vJ)f}5)wFkp%iTZon zS5#8a{}9#Ugs9S=pQ+yy&|tVhwhwL=c9_T)o$$ra1(KU*`6QZtg=nq2-=6;Pqtd-P zz>y1?&vFtd&B&-2GLsUAt)mwE`8U=H}AM$%f_j+wgZKmQoPZP zyX)-6ff}6Xn0K6c-wKU5Uq=2(`*8&b*+tWgrB{?}^r*Fh8jA}RtLT`C2K~>7mQ{7v zI$y+j_r96ri4wM*b)B4Vh~L&9@{mUCXwR|hTdT4dH?Or5@)cH2#d&yo&KMf zY6%mDS;n178?<8Q&L(xO&gyN zduqD^5^N?*Q-Pqqxw~*P9t#orVTXE9#!3N>$ z5Gk^16$j5K6}w73=aH-Er1_uDw%_r*hD2wZFgDvh>Ge2;+JLfP(B|x%ug}k0pvjdM zEG%YjPQUg<(SFf7W%`y4zKpq2@2Pv$#jVI7!$oc}huhIqN}k6waGwl2)>n^p0KbSP*Gs;)Uy}T)#gBuS#u|I43{nT>Pc%;roq+uMODcV_(*oYkWyl zn#GD{it9-`0&ecJ0S}LNo4KR4gs+C}C|Dds2W`f{q>G(CphRwlSvm`cZ4*2mT9I3Y zqUdA%3AL|Nc~;*HFpb@{(qJoN2;+*8*?VyL$mUrQVn_+wNKrrJ!GwuYxx}1Ru>)Q<9cl#?rYIE@S524n*SEh_`kh7!0Gc z`&UvQ_kT*p#d$4zu8gW5P8=tH(aj}27fF5nw++_!7$DNCZ!u${)Iw{t)QU#lw|(W>{855vVAIdX+D|t#T{*hWW19}- zg77O$v$Cwy1|#O1l2+578}|N#1qc*ue%iYhs!R`O(^8J{XPzmIZ*Z7iNyV87(~!%O zJgEs1U%YoE9j?VG7JLf3%^z;~7g2Ip=$+332}$drB#UJiNqOZe4h(k(p2zg=6kL&d zF&y6pb7rM5A+&(61)tuR(V)ZABQb9AA!YH zcdBpYtM4GDChA1p6%uu$`DA*lI(2wpwc&qpz`w246XWroAdu7W>w(GPEm@5dO8h@Z zwjT9)ymw9yUyjuO=PFC^d%;u_wv)U!hC*uUzdQEt^GA9bAYlzC6IjXELq(3pYv|g4 zH6Jk12Vwsz^8pudgGf~Nkp4fF1WR9d67?TT`NM7V3m@^nr51CX|GFbMhwaIufA0Oe z(ZN;1Kwl=ppYdJ@9%FmL@Gmm|yEVWE76zai%@7;x*PoyNrFQ?eX#P3Rq}-mMJc-Gw zV$W9wxb(lD1wKCspzaU^N_luv%ID0E3;#>44?I*~0nK<)&&J8C^TdKH(8%Vo|s<$dt4U+daU{S zwIq~2JL5l9{a3T~wFG8jva*+dUjDmf|8?oFhC-;XKcUIF1F%uflc)c$_hCWp`2D0& z_auR~{CaWAG2y?O2t@IoME;wM|D+lK#eS>wL2PlouCF_yyza{u4G!36bUtxJ02{65 zhtgLwV)|*XI|9LstxT*%a~KgM7vbkHSfYC4ntV_nVJOpcrSiD*SpJC=VMrYH5+nym9_$s z20;+-t8o_2ap zOr@%G`r7b%eif@cK9iAq*d04HR{O3qzwL0O!evuzc-gFMsMN?eL{b?#*G zueyyW1{W8T-U62#FCCImqR{nDQ;);00lkug+C}*jc|rToT`*spZoKH=JYov|P~({y zALMbb42Lz3Fsl)!Np_M`Zs8Cx1}%bm1K=6uUh7xRB2rT@jMPlQR)Sn+U?5H_{4?&l zFofaU<=#&EMF-xxW-5C!^YOBvFbySEI7XLxA~prM8K^G9AR4wsh^oR-yeoE`o-i*1Bq5X%fyp@mcOleRcz*W*lk~_{%>pEOHCFQ z9$TvQ=ka${eb7`zK_^?w~d4wU;53i9ew-UUy{C9omgp{IQWHarmAiB6N7-D z|IJQ0(RVFu38^>rMbzje)QqEE$p2>U_k3y7@fPvg*eq9<-icu=@=CbDH1ngrbn*EGRQ$_mPDARV@Ssxf$IZW(cX7<O=cLwstkb>=h(1*GQB z3exz8lyul=h5p&UJS&)7(gEf)%i^QvnSSX1#B$NQ;5o-ta5GYc#FH(NUl4iQi@|;4 zO(Qxp(XDEbwBFPgvk^>=MB{HEeN^1`QdWWj$yGs7F}|+*lhR7wgmtGLoSUJ=wlJye zCr?M3S%}^%f*XsD1nfk{+J{pH=g1OI4$M_LL{n`k{6d9lBWR_uj0eOO`Rv zNq*8WRkEz|&N%VS($qZ((v?aZ@uQ_dd)4Rp$9ggw_PxVTzc2d|dv`%S$UCh8@|L&x zVQbawUJS3FKi^*O&EK^8BGy|!oI!-Ha5a7vlHY>#eKmO7K_LFwdvRU9n|14zegWKe z#~atclkVf#hE10%yf+!v<2b}d^@2P~?bV^10)BuW_JLCJQ)N$Qw5NWvAfCbFEgN|g zWEN9Y+eO}{tzst1tuVq?t6!a@*8v>$M4ILT;sSsXnI0_#u+!hrKxs6++~@C^B*5ro ztZn<0?fMmnLkd~OkH*qJ!|ZOz9L1&_7QEyfY~DUhY`}M*%teE6B;K2P$P%;5*d^k{rcUt zzJ^Qe(w8N9$2^V^OSzfzmKI``R;RM0;Var%Ixj(HdsVQ~jdM%sGMN>_KK}LHteHDd z;_TEzwL6U;!W9txAJSB~nt!Y>iT83W*|X}`=yJe_7pIgX;Z}nBe27$u*+%L}f8w3H z3L&jL5SrH?T6LcgfMuyWrE9Hvz~T$SBfYZypTpkdg%?VKUfxwzw7XU)D!AZgOzaq- zThc^t#vT2O?Z$gG`q_F#pg}BpS7Ui=?wK!e>}K-eMCixGl6{7juano_$=W`@_zFZh zjcCIB>}Jmxkz>tfq$ZlwZXr7l-I(Mr(a}QrOe2L)`^DBNx%6{2Q>`~?c{&cB+e{>D zq#n4VTo$Q&5pT;kBP&c3X2*XaF5nna*@ zHM%;4{)z;Zml=>jHOX)*XZ!N=pY$zMRH6YAuOU%puP0wNthIdllO7vRUK0#Be3@Ku zGk7*3#c~Q)$oiNi96Zy}@-bWum&;~!&XZ1wo*30WSvDGye=u|{_qv16W>~wL!}ltW zD>r`I-ZAU533)votVzx2Hin+rKMV@I95`z9_(ru|KTfp#YXj0IRmU-c)@AL}>S^fX zT8IG6TybZI`TFtkdT^HZYK4KA*FNLcPG7FQZ#!OSIx^}@SHGK%a-V#M}x?+ibPjCiyX)C+fO}@PkLVR;d(V!Q(Vbr@R7de{92?o z{F{HF#M|Z5U{RF-V`5cIAf3yUAB&nhBe{`$F2&vWS&LK=zeLM5Mjwx7EWvw1jhI%^ zwLFDQ+%`2iMb$ z{E3b7DPVgv{ z*gmd@oZ|0zDag_?Es>`uKWM3+%>d%KIWf%)-TP9mo&8H$+cNiqd2jt2%f>G^jp&nt zcFmC?7m+PP_Eys!)KKkVKj?NW-_=owkv_K(z54nNqeyJk4ZG`)EWbLfG4Hs))MDSg zZiCtqC}KfTo$3+L1!kN$Q3I35T#ZyGjhkR_Nkw%I?%X7IeP?O@*j)Vf$=iL^r%Fd?( zTJM+cUqY}YSCySFjNrWvP|x=18LumeP-=PCpaHLO@_+ye$0dWnh^ho zt}lUyy8HgmQpgiZQdy%yS=y{)86i}*Bq2N5cV_Hs)~s2xi?Z(-vKJ9DvM*!bmtpMt z^1p-1^L&5**9)&P^O?`*-gD1A_ndRz?{hhSw8Xs#d^6y%z6Dn{r=@j=wDX(3u&5vK$BxoXYxR|f+NdQL+qE_;+K-r#q#o0`65 z&_EvVC1$b6tZ(aRgH#Ih#<)Mols5fH;TBB}&CLg}%{tc7A3R!5?;`Nx>L-zLRpsNY z1sPouy#CDv#-d`ncXIYur@FJG3~$%iQN%|KDfT^#xg7L4B<3>bm!xX5#vD}#w!F8y zUaJeeYs5BQ-Rix(koVtQ?oZ>bg++2qi`aG7jD;xPae=-@4}0RR)V815ykE8X zLG@>w!^F3*v%tQ903#8biG0oxr69?0zS_4sS^O(;(fghiOZl5_{qwpHTgG&aeW%!$ z88;3RyKK|$+a5eOIji0Fv@FV}a8GF0!gsk**Zm1CuG8xa)U3EX-`wGv7IDig-f)9O z+~=d_Q3|^C>~sB zY3Q01b(V_S$kb?cBiM7ZaNYS+*qQ@ZM`GOtPX}H7Aj1E`Zsbe&!J4o&DUB$;c&^9q zTmWVITj)TC968L~{fb)wA=+4DMC3i|XIr7iss-9f{?5<1akdDI^+lA>mzns_5F5d! z;_hkUOWr<*o<2)gEUeB#y7!5{59%rDzV?eL_EitsZO`EfH)%;AP3dx5p+D7w z5pLtg3}U;}FrRY%tN!JW>lx)oneoIKNW_ptJ#@ypi)T-bUObKt69uv3M;E3bo=qrf z_4?*lU0%ni)zd12PlN`)4-5@og?kv?ZwPo0vxsPYL>rkH^k&hyO$c|*>beQprnDp3 zChe7hYnY}KP(j#{%D=o+hR8gj`>HS4A>ai_J%{zkjVWin29I*-UE}_Raxgn$EhG>tc7uafgoXUe;|EH-+ zsl%G>luCbGhPHSZh0oZ?s!0Wf$vAaT`JeB5Z0E15mLkPJBuVx8_74$$C!%Fpl5PpA z*!z<#xA|#?d{g=BbZu$InK*Pq+4V&>dArjXC7w7pCh_BTOOV%Eu@%DMi~`;1F;y2Y zhe_GzoL+0A>w9tqAK4&2d=&%|wn$&bVgD(CcRaIO$Yxw>`_JB(-}xU|b^HvFS;^%} zrv-@^d;01zlm5FTFtZMk@VoM-%U)>I;0rif(%q-gulDj&Z5}Dq_FwW@xGQL+_RbPe zNPS$ls{A+Tgf=7gRwG?L2kEgr*mq|wd-eShb`pbxWP>QZ7B(Kd{Fvf8ZoY^s1krCW zkz=qq(AhI1Jb(x*L*|9Fr*Fr8&)NRzPK~zI;Q!#@FH>h=E_Il*Nh88R6GC`oK%~i| zrDymoaQVRloX$hXH6NRX?}9f2R?^W#Z<03s}*9hs-!Ae%=x5rJebf z^}1b3_WKQjrlHD$!AwO@2C}=jwBpL%aO0i2(~$LmA#o7 zcK+F$cpSZ5HtohOhzXc})g^u#_KON+82pZ^GdEpnY535x8C2`uj`2#y#sKa-Hv+aW25|M9Q~uE=0Q1({ zDzbI1O=21F(X` zO}_i@uo-<#Fdi#VG;Zp1=|lG0AK=xv_CMvG1 zwji<+P4g^M$f*{9dNU_-9Gfs21BEt#wR6aj`H=p^@GTfP>hnDI`U%DNd#GSYZI?We z9!Bb?Cx{xFEqO*g+~`--*W?rr$MOZ30T~pPmiKP+-6DEm+W!q;+ec&N@UlcAi6Xl8 zsOR3>6|K=&XX=uaww788G(A%xEx4I<5%R98o6RwzH-ZzR(zW{^4h+P*vkZW?|I2}S zVT;y(zOIV(^L%J=v1AMw-i{m1*ZA>l=WZ01y>OvvAtJ25x$iUGYnt|(O2xF4Dwuu1d)I_3JxTy%*?Qlu0x|B} ze9v`u2Bh$4ul-cpKIXu12J+CFG^ngan5Z1&&o`=SI27j=;5ocCv64$Nwwib*d-lAK z8Ey}bnL``^Z(|u2J?GE_us8MZ@{b4>M@`fW$~#Ga`OYo8CJp?8SGc9JEMSh_5=+7N zg(gb2Uo?8(->FI$c4J~gXB82KT_Q-$wKrH`Eb8+vCZc%O3JN^CQu88JHHV0lw|YQD&dm{vzE}Qv726<}%TK*pUl>~qJy4zF!>a<@q z`0wXjImvg@pv*gDI~`B3Ma7^z4z|!TI|q*e-~|TY6u(t044mT}g!tb@QfHjZD4X>!&{F;Ho;^w}NYSG@HfWMyQqp zDv)E+lHbvGP8tQ6vxc5B@5P4S;$P)7ecc@WvD8ym4<1?R{qge*hFYN}+bMOai^a+B zqm58gPenD&j@n+11xaR1*3d7A@$jMptJN>5uH=@^uA=(WUFl8NjZm<#qQ}V3*ttz^ zWFSTmv|VpPg6GupEr?%jet}$Wi+`rSa6Bo z0HCSAvqi-hO92XIY>WE+n>JYfC_wDm5T{-NtboAzMwQ`9G6{f{nm|>?9Yt0UI#ylT z>>Pp|_@3;&5(Fs;d?_c_N{Pe#2oTh(XXG?T&I6chH=xUuAMrPQ1V2Rbc>j`LwLy7Z zQFCe4UcFK=xhLM6Ab>eUOW$}r$q5Z! zhWiWET^kDlHBye|tJ9gm@k^ZDmyf9%_Xkn1%M0}`mXArzNb`t$yk|${wttYhI?0>G zl<}bY+n=FG@(?uQ$_`U$v$4+6A9Z832OQM31h&<5!HTc<9u2eMzOYSFS!2IZHdwk* z+F^bHfw_ZxCVxix*-B1VhjI)JNpHTpBP7qNaDJem5oV&Sp;GusWw5b$x7j1>^g*A6 zTf602YS3jDHK&1Zsm6O+=w)Ig`@w*vVx=a!L!-6AWUqeVQ4M;O>Rb#ZRT=McoR#$u zvgkz~6RKyz;s-ARa1#yoKh=cb-sXq(_?;zga{b7BJ}v5$y~jSamHivRBrJ3o;j(c? z1oKO1)x5Y}oO%_-jpSOr=p<&=IwaL}uwXn#td`qq1{e$Cn2J5!`Xnk+o|TpUO^w() zYDa}<#(XWlZ)(Nf#Y(hF&6N5c?G9;0Q;-a(I}>ivw5dydKmJO$KW02E9=Q&?TPHA9 zrFFLS(9>a%jl{p&sXKB@1uQ`J_@MfNfmBSY^2(*SEZxk7u3JSxD%9r+5Hyco?nsYi zY4@e?--X_Msp_DnbH$0hkoLWzO0KhM$oQq|z5W6In>u)2LniM%OcC*WIYs_z>TB># zNNU{cc~IGer07NVy}b|Vc4O;ZC3hMgiZ=(&bL5FQz)HW%2hO7qUF+lQR`7!<6v8H< zC!&*%N3W9#VbR-!aBdx9ZDQ+JblFbJj<%_~`+odJedg=Et`{xNwLPZ`fT5^c2(}?k zfXy6?#%i@}H!yE;e}P2t-d3}Eo5js$)(r7#clDj?T5I3E4#SeN9oAV<`38aWKi=y> zEWEz>F2u7TuOV#S&-WtwqfSj>UQXPsF)}{Lu|c@B*lSz9%F|1xKsQ^wE#QYt$LbPn znA}~+Se^{dHQm)oL7MU|MMGZbS;= zcP_wnsk%f1P_*t102U2)p8GS^Q3MsD87O&T0{IbiWNjOGvJq(wi2xD^RLxW> z8d31N8Kv~hG8(RIA>y`C`g!g_UGIeN!Y6mDS6^bzsye*VXAO4H;hPFCY;ro|*aRkX zvrtaJzAi~+Z?6%0SvR6z6W*qM&E6;RE8LVXTK3{jlXL;@`pQ&h|3jDN z{(Aqa5-e6YM}1eZX&=qoq&7>f)}(g#kXsWF=fFe8mAh4a8Ku%qfzpBXE%dCW&s%dd z!6uvG+{~9ssV+WwWzN;xJ65s(NGxAqu=sXU@95f#VKgz!i-rwL?M^HdBEWMfcPX=C z#pKoSG?Of>2Np6Sz0L>H4q}X#oNMZqw488B&eO%Ry{>;6QWzlS+4BgklP|z-x!z8A zsi$K$Bf4;e-9?f&HbJ&U?zZ*X4vdF@Zhrae-fQarlUg%$ZOEk_ht}MUGh9Mtdde#k}tJPtxkGDyx(Nj}lsqRw2pVx@tqM z4y#oMd1bqRb9E6ll(`GEV0IfX$3nBe!yWBn4OuWHt8B8rQ#AJ-`O3@QfmXxS@moA zm*w6!HtKqgDwq^OmCp(^sq6&~*#xg_8DX7WgvHDgB++`gi8*nRjpsw&*5MeOQo^=L#0va$v&pi#G0L?(vSzH-j+Tv zQtZlA9o#`%Dbh~lr0gnHP2hnX6T!IKt67+2#H7dvo?)8nwfsilndTXTRHd z*>q_jTdWY-l_d$OdH<^<&C}D|m4QI)EA4KVh)~w;@47E>zfHT)_LIHW4u(g^q%^9*e?|Vm!oqirCQkCLTgFXgV*JH(y48K&(mbS-{v zgKwv4sT=WaLS?2=>F%7Mitu~QHCYZ4t09tZ+P)%?3wZ=F;Fwp##Gv{GOV#{K&j`7_ zEQb66$@_Z9(oCe=i$x}K;KdHE!lw?2nroJ@tAK!4%3C1B1zJ7NaZGGY^1avAyUOhpe!u-v+WM*%TPV|9|w zil-nCql@*;npLbnr$nf}nl@};Iporqe@s~~>@qp?(hzb_7cP-$#PuE#gNUHE{d0r@nVJ^Ol%D)>+-pSev%;Bv!mi zu1kGMU`JKEVq49qUtxaWAg0Tq`dpV@bwC8~;!Vofm7lyk+a#sm!Ea}j@y?xbz@^c? zheTec^K$Cm-dPkVPAl87-K;!Ey??}`2`s6P8yX9WC8b_6TPTkmzT}y5g$zbL+_@FCVQM5jx`8W#?NnY(5p%@5Qwn1s53zb@Ujk+!TM>^(*>+I{< zQ+XWt!P?lKWmU{^CD5CFpv4lIXO(3Xw4z#4-8(wAUef)stD<`%!c72K-q<5IJ&&|- z@UKbUa5dwjT4SR$eGp6C|MLg6ZDC;zl9sfNc4)ykj#%XFyoJkD*uoTA9prQDVtA@~ zCC~g*FU=Q&j_tzhZw|s4^coyPF9;|Ue@B{|F&DDW?hS zwWQTSIiVxiSz;g&HJrcGf(_7aj^8?oN25PWm`21PTZK)zsSjpI{r4I_qG@^Tdly`> za6x(<8@Ur#Bx>OvJN#H+1UL2`R5`a4eqWLJ8b9pG?|-m4%|I+#Ui{nh!!#d#0siyo z6Uekf6kSIkcudh)YCnUl5dL$JJUoG|J9=GU;6_?6Z4FER^^^_{oX+1U!cHvwY|WOX=`33=tF`ARNC7gpH+Z3H}xL@7sf?b2aUaWR!r)@2uBlK!-~TWK(Wjv zKi;DOJ$?|qAW@p)pBA6PLZZ*S$;~)M2-4>hrWODB6uj`oV&6qWOSTV7Cz8qjcLcze z2xx@>^aTZj?0FnjY<0K){J7bI3S{k7*E8?`&qv@&w@s!*T5!HR;2oVuWc;Tm;8ui8 ziB{k$TWnRY@+U73%g6nBz)3Pz8XnlP@5FgdaRR+sbS<&`Z)eCjfWG=sN@ma>Gi|{G5QzSWPj#6`LbiV|DK^&-Cbud9`+9qynx!{Jdr$k#Rp&4K8D|j zSF(9*e_%xEPRD&DBgXG|^e-VZdw~AvLgm7+U97!_Q>6OeCV%h*SF|$t;v0)6e`2ZZ zt`9^5$9Y#a@^6o@6Il!2-OlPxP~gk2U{eTuG#dvl7nV6&WHgLOmP z>7=N@>o6JmC~kt4>1Fv@Kf>$xaMlF+uv69Kf@I?Q^^b9;0nyh{ZmJ2rUk5A&Zv;;t z)yg}Tmz&`Qjmn?p@3-!+OdGy7iyF^3v@_H|EY$dWLr_3{W#e6msa&4Vbio8|pOmr> zKTi1N9enq#f&`&5J8SfYgaG(h;vyE*;g^#9IU!xdSwA4O&3LHm3PQKHLay&I=Wn~k z@CbVPaQCT*=#I?m!vwzS9 zNJ-SLBX^_h?yXFCMQ3IL+M?U$by@X_cVp?55}EOjiSkTJ=F`+)7B{Wzk}xCXz4v=A z)*gNRxJ9nk^dZ)+c6T7LQIu9l{x59}))#LOdOnovh>p_tyUK!Vf!%^Exn$-KWAErG zNv;`)>qru^3}iZO@uycuElMM$@d@OMV!VIB^>;3S;65D}H%DOWm5S&?IQY>L4`pAzoJL(Z>$xj; zXtqwdX$9fwoh5pbsF3-h60^0nwVy`%yPa;)WU^qJ8mPK4?pJ$;m+IHK&aDA2fF5h3 zp$u+81_lO|8(+SBIsT519q#O(^%hz|PS{2Mb^F60RU)>|;RkFd#nr>7$vw=5m@x2EPkT?ju)c7kUF zFXH24?V>Ybo~g!OHJ|&M=>Wm_3@E zWITf%+XComd~3;L-7pk&t)F|zk!Ly_C!NLkQl=yAjxKh0(g@g^??twmL%fK(L7Q&X z8<2#3%)x%kp#2;hZ(>H!GhWfeX#B7vG|=Jt@gd<92~!tA7P&jP6WvR3QJ1;Shd`QM z*x?tE4O-^gYTaB4a(T6kOx+lA8FJokPw2xV=P zV9K@6YLT(fXzaHz+GG*Dq|6f}9Gs~wbz2h&CumrYJCDz-Vw;YrO7)T*7Z)<G(drJW2&Dan z${lJyCzcozkjD~wXt1Cn=VdF7SMuMJBVPoVqn?m(>5rqr4j1?1_JWV; zm5I_+kJ{9VEiXJB9-w@}M&*Mxo#54zgSFinm&EC~nQpe%5}p6F1N#D6gLREZbQ3Ru z|3Kx|H5vR-S19wvmfB|2ke2i?Oq zZ~uMfNhiup6x0$pt`-1wCj*YVFn+e_A5mj##Gh=34YporoZ#$;Pg7()WIOnBVnr!{J(wh z5yG|TV+8v9Ja0DQe>4EvOvoGw`ZkVrym%vS{Wd$ta{e2XLH$ARS?W%wO&<656#n`= zN>BdN)BJ9Amx1NwW#d|(^IR_bj_hI&c9$coKFczX5ltihu<2 z&UraBVKYvop`pP!^;r%{2YLog?m4bjK1aIYINQhfCs?9(IYObM#aw`KR&B;F14M2&QXK>=F>aW zYJ&IB9()GcK!MBh(!7`T32Zz!Ja`J)revpUxhw<9@%yB%2BxHzCsqi*ueB1m$d^w& zO7U00q4-z9*$-anFN*x2wM1!9?2!7f;FiJWmyt;yJ}{J_3!2dDD6`_;LOm3R^AfSP zVrvAO3OXY*r$EK5qEn(K1hcb&;h14sN~ywB?y&lpwSMqis&Lp?*e|i1cw9D*K$RCV z3keNz8TV-Lm@~iO8s=wMFmu_7*_(3NPtAQh=0Peb@s1o<`etM0a^r0$en{}HWO|S| ztzB)sES+>-XQtu~X2NM$<3mdd(=Xal2^lrm3r~#oHw&AZbKE_vGwn_enkzMQ`@e*g zOKg|4suhBcexN?Af$-Z8_PX|qdATz~^taA}MZ0=Lv{cyR%vPK*@JZ@;EqF9w zm_ZZ0Sc^02dm)%w^7ARgfwkcj=GChU1E}o{3gHGwDmh5dkoXeo=C`txp7}TAzV_zN1Fqb!+g_iix+pgpvi>xNtGP!` zYKLC;Y>H=LJUNPhq^Y2q?OAsHIa9$I$n_C9R?hTlzW`)#u?_ zBOPB`J>+joD7KGrcE1{fbF`j$s1%sHSYYvb#Fc@~P4(IP_wT*%{A>f0WPb>KyBsfb zxKfZ6<9FOWREuZ87K){7bg&-E(5?1nHIgotulxzo1uT(mDD!F$TUdCwF2AKWBhi#x zUK30CKZ60aVT#UhpV5WYXuG%YulQGelV)9Z++C`#gJkN@ z73ek@FON6umkNp|RuZKN{uy8wu%67Qvs(XNEaXI7#Rq#-YZ*>Gi=cwd^;k@A3GVvxl>Lz{Qs0(`A`;%4>Y#yd$D#+bpE1yTLIL`DN4LifT)d z7Q36FW1t{{pZqK%X<6u|w2Cb{2(cIlI2i*kzDnO-4(oo@NWUn-U~|xNup8uZ1>&z^ zHJhHgO&KqLCP_Sqc0YGFMEqp*o~Q||W5=C-g~OWQ*DvGPaL1ksB&R)BGxCr>Fmbr` z-l*-`)t zAOJpMS060VVBk>Bi(~AEFJJZ^4H~E5BwJSR#x~4@v-&=4uzgb17GmwH-APFN* z9JIMfAha7I(_OW)NS-tICl?8&CX=NUrGJR^K*=O1jyw^7s(hHJ6ncrPa^pQ>W0>8+ zVWx0>*!#egx2(W%gKHjXIpi`vwnDnFJ|ltQm=!tnYZ9di(_{~)vo6-TFIeq>9S=_$ zBQDb#-+=N1lbI(&aQ)oufy({_JrfnvodI~Id7O3ZZ6Dx#$a4QtfecCDXBgtjTOQv) z0v{Kxb#*FMhGH#TRW2oyS!zI4zTxjUfTSUohW;}6-4P8H(ELr_s(^=XzvDFNFaNo3 zP%Quc<;4@pGX_~dAoY)~)0uxWM+}murrO>J>P0AxZw~53!64zjk3vp9L=y<)8+icip6k__^{SIa_h8&!#bhTpm*iil2 zOs461X0gP)#VC`W{Km=bgx7Sr&h4La?URw;K8NSVxL=kam{lOqvj0gzap@{aDx-K% zt+i0fN}Jsb)N$t2%8f~>p9yCN_%k{>9;+V>;|FXmS&kl`Qy4#rb%>yNJKFOQ*yglwO#Zq_rm5P;$Vy3& zBC`32_cJnz`>u-??RQ5%yYSbMM>1{2#t_pXVp@pnUmYqi!?fD&wK@g{$XB;n`CglZ zLR4}CJxnyVUs6gxlL#_7TLi*m_<7kSWCXJXzqq4T#cZ+<5c~bsoW|aEfmdA_f zpGpt6)#Zc}YnLukWXmy%(z_9wb1wwZ9pv5>HoRzvrhckDkJjw&o$<)YlM|@Y@hFux z|0?yY?AAYinrHA-Q53N&veZxQ=>?8*K_T5Zli=9cD51MS-_Cz8_uHaaVVzVVdQkCP zQ7o|$KTH;XHWY|!O;W|i7YyN3_x*c2+z~GNK{k@}2I{oi=caU`H5!^#=w$wmEY_#D zA-r?)hCjvjBY8$vGmI;rF;HQJRTvxnZi zXtZnN;h86dWE$}lXh*Gm3O5FYqZ}3V9M*c{={GXqa^9$ZVK&HHT}JR|HR@TrR+tR_ zY=G;nRqHLZYUT+YTdJctEYXw* ze}#Qg^Jo1mf-9UIluS^^?}(AW20<4tDtHk-cdM>bHQFQoj=Bl&GEKd;`0PZ=%m*I_ zCUEy)RBY^LJ%ik*M*UN`8P7mS`5LS#7Yjeg*JuY+BX;?R#<|!ao5|SFPb;(TRm=Ni90fI>^_AM%bzG9oVRUL7rWw3`=nu9bw`Z z3A_wwoIW%*XQ(=;6~9etyz9nu+JJc^G{948rqi zXWc%0_%PP+03GP@OBi1(@RW3TDy>7K^7W#;_7=eb5?UVp!qk>~g)TOpH2mm)MkQG7 z%xm+WoSh93DO7Fui^N9-Xl=FK7 ziQmjQOe)!c%*>Gs0K6t9X69pALT})^i;pxue$h1s@K}tyxx?Y%kvy@tcpPnqWEYoB zsFlip#Ln@A^F)x0KtA@I7NeyNza5uTfEg#A&1iNAoF)jmHddDW5^ z`7c~T`z=%6-rVdfu3Q+(uVtr&-C44|Ou`m>d+cUlCVlD^lH2?5RDx5#)z)*gvCm$^ z-)hVNOgn=u>GM2M<_|1rX>Fee(@0j%*V9=fn|u!0#O!aDjWj$#;+YB`@J76{W9rx@PTpoKrrIN9orsyhsjR8@)Z^dabAA4 z7ZD!b577HW`&Vg(ET{$68bd4ntIt1jRCuUs;~D8zm|b2aEgsVoB5F5cr;jpfSLe*} zt|ypMRCVbjirrs{JeX0xMElCyXfaBi%$sp;aA;tT?p#WLW=_?;lJOw4@vVWt9p?ih zk0P1G|4-XxpC+cIrEQKACt$aKePFme0=A-=tW0Fm{#$dCSCE%t=~hpB6U-37_ZOuZ z)y?A`#DWS{PGs%{Kg1Jbhp!qmhz#@lkoG;YpS|L+{4!Z!#LnY0wJ(#&43xjyqk5yw zz#TKe8|Pk5{bVZDWQ62FVO zo{#snEuvWRKLt__C2R-$V9)!V|1Ng=ThVvHHr8AK*5UF}rug58EbL|?2Ha!`yTP{^ zk&7)xd^m58{{F2dhQ(R}ADue=?BB4=t?z;?$T|Pc&w@IxAQtkb?j$~Z&#jap(eh82 zgIz*E%!sW?b~RZZaAiM9kkvZ8S2C@Tu!vPV%un3Sn z{ns!adT7{xp{nGvTMT3|SbtFJOM2(OzDtjLP7JzsWDcIc2M(<10i~QoNDRfwFUVG% z?CJ4qdN-o${$ng|x>-4YO4AHO=rJMDkVMngs z@4ut*u6)0-EklDcx9G@iKii3<+!XHx;7qLWmw^AGmngtF<9%VrVQVv#PR zFH2^l|1C-A8X@x$00*jr^*fc~Z=FOu(ci;=?)Uo}AkGJlEyo%!1k z@Jz7(02<;SV|j=Ws3DWh{@2G?yCs6{&Na>S^j0xw_W!YA11iHtLHT*&Z!IL{9I4db zVLMdET0ES!YiF>H^>`>B`bYG~iK!ulA(%aocv@ zTypsa6O33B)b#umc%&P(xZqkX;m3x_ps!5ktH16-!dz;?R%eZ;N-Gd+D+3Y5 zwy7quh0R?p{aQw;lfAPW+`WF>+)Zxno{FWw`ApicWElP(UM!Jn`@jmk*kYpEavKqj(TrT z{2LdrVa3{1WVPqtj4)W>LY3>JnyBWZ)|iIYjr%Y=;zBdIm=eJY(Wd}1P6DGXa;~HC z2U+l-xL5My-@8)!STFp6gs=&j8~zPh+pJ$C3%te-mc+(885I#_`l%!SN-M%SP7AS* zp2Rm5Y{}#xdV{hTrLR##h!4uUUX+)%b~%-N=W;2nhq*ZaBGI8#ZbjC0Wzjb%dq2xo zZ3>L&y4HNJ6kaI!UI1|So>JW_tqQm-^rP-6t>IV;wE-_n{^=VADM<`A`^^5IcUvN z)$Vjj3p#|wS=SL#2^jdC6EX-sceS@=``oOFfP>!4jn$m4FHV$3p&f(DN`9P>U>#4$ z)v|TE!ht5z&Hbzc{mg7zlh9U~?Y*VFCjrhinhjQSVNSo?*8wcoD69krVbWPZ5|&wFCD zy@^c@0zD9nw}&8-1)q4m+l+bs`l2Or3X+dJD(9<2>Fa6)1gM?B1uF9izJ}w_D3^u7QSwdQ4W1A zl_eQk*Oey8NX*H=!}8_e37q}nStaI+1&@**fg0~Wu9mGzMROrOvAYyrlFGJgnvCek z@VrrMYQSQmqGafOF?o<=WNW3F z|Gk*Ko~b~L@>F7sT^F1U zQ$Kc})54(1kL8(#cgbXBL=yuEsRe9*3T;Ph^s-~no9OU0u8pAE^PvQheFc#P;8eIW zw5)W}4a$$o-K#lVIzOS1@4SIMQr2}8OemJAp!|IGs|(Z;cv%N&gIzVeug%*&W7@i& z^<7t4NtqlRkr!_cn$;3X3g$Fw{rRl#rpgYxN!F8=z%DB&_Z;_!;&U`Y)phY)Kc#tH zvYciz$>;m$QhU-D9DhXNbg@;$+)(j@OMi8lc}rznC-Nlj)r;O|OmwtsDD?Pu#b+ZFt%;`PKLYh<8cg+rgDp8sl4Mx1jm zK4hkeF+f$o$?Nq?`&{WRgtk!?g!xe^m3*^KH-+O!zn%nX4ex1}@#mIN~tFq5`l7MgG8F_;8yyZ6_uIS zCKH;;P(gRYLH)FQYpAHb`X|=oH1Mxepmm0)ra{_9%dI-eNt{*hD{RcOYb8O=WT%^^ zV3pl~YwKO8apMhgV-$GUG=dDroqmcV`I#j#{J~s31y;c5Y&o4Ozh~R=b zyc8rB$gk_;lJ79-I*5SJR+St`d8^?23UA!)U;vEu@xX2~F3kL?4& zHMMoYyJ26hR{YbwV_UeAr3jXB#Y@|2+WM(dZGtL>Tb#j(#-LF@jR}GNoIVy66rZdvN5+!ady~a>c zXIwq(4AEoyMj9ESTNE}b_*Z?Zk?4vp2Z={`ojHcuzR)4L3gfw1)%=ceOVq6j4$1&_ za0XEY$7xY@a(5{IjMGEHr*RkEqHZSE)!)YYS_%cTk%S$=(5|)s&Y$VVzM>if1yUBB zsxwZnd<&6$j1{@}Ki}fi)yE&%QWR)W+vZrQ}hKW$1l>=>oA9=2<# z)Es8xfxCwsPoKTPh#|af;97BjlPKDDmhHZ8Q_MK! z*quCqf;5FEVM_S3@1T}*5!lr^fX^f^&TdVVO3*H^Xdzgu3^AWCM(1K`wm-udvma!K z(C@QUDZ7qBnD%R)-XfSSki1pqg?~f@GAa$FT2|q4`Vz@xDOUO8iX9_-T)k#ii;oI4 z#@rQq3e#W>vFV}*Bmc*3Syf{)+`}WA4%v!Ts1si^3$Dv}b{NUI8J_nkss+AIbZv*) zc^_O{li2q3sp=fA-7p;3teEzEle$o+SA2hWUuw}>(5T7F5N)d=6=O5Otq`ZxQRA2u zB!dc{)kpZ_i5%1{HMPUM?AIjbX~|tqoE#t#PM*RZ*Wc7zZ}Ho~l$IjSq3pC%6X2W` zv)zz%ryy{;-6}2i#3||#BE?NCOT=5q5{f-*s-yD8t)vGyCEqBKi(bEPLIBo-aj)HA z#%0l*AH~*RwoT7af-{W%&l~vYjuTZMo%S*kL>)|EDJ=Md$nF~Vcs+&-`*rSRHhEu$ zU|IYQB@lz(0)bNc+2Alykl)q9m5SlaN5nnLcBpzl#e)b65U`4psNzIWo%{+Jz`pi3p#a(CYk z8Q5IL>ayVsu^tYlprBCb_u;8{Ts7lTId+P)q?$l_(Vgn(h(AiCH!00}|52N-%#C}* z9?H$aFIT$kH23D$^YP}M9I^Eh7^ClM_UgACZKA)ZtybpcN$-$lds2RJZW`=<#MgMp zZD1hF#)fV$8UW&ri~v9Sfr&AT$-8X%jrA^V4aKRj#?K+C=McVAL5TKTo?he^SV5fQ z0yEVEaAK|AC#AV{mP>{gEs>vaQE2MtnUp!jVsvc9GO^mT141&z@c8$y=>)mkKUFNJ zH#=&(HQSlxyUg&CL`_(AU7qP?8_=hEF|Le^W<%K3a2@mv1(vHKjOJ*;`I-syRwUH4 zLOgi~!B+Y-Fk~JhY!xs=?N_5HXxSdz+!ymiHcadn7wmtr&7u5q8h55ae80v!a1+$+ zT~7DjPK(^O>vvl}Iy8}E{?i_DD*MJsO9S0*4t%n|fZK9HFYJbUp0*rvc*8nxd>4y2 z6+3bcKC?$6MDlSXq%&WibsoPWH@o7XsovKBqgthlNtP??JlcZ=xgDSy2>fvDF~W7& zIu<&TENuR&jByUIJA)jpW;z^H_mXnqvH?PRDRJB3`Rg~PQuubF8(?+e6@^DT@gQDY7z@7 zzqT4w1$!?m^J9ury`Lk6id-u1GzmJwyec+|gXZV;KA|8oGKIzQ(T4mP)$AYMEsl!| z#?PfN3T*w5;SJ+n3D~N1oJ|s({jMnr(QiTY(^NWrF8omx32S%W76$8(G$U9YW)6>+ zDmeqrn9M93!&;B_RgPJrzZ~ZuSnAt&K!1o32tUow?LlRJQLY}I`ITC*O1mdQ+A2Qd z)RFn<6}POt`nXH#|D)?Y;Hhr^|MBPu*(79dB9gMV5ORPI#l<4e}3Qp<5BKA=e*zV>$=|8^%~FD>-Bt*{+n8cdxGr1*JbU>AfTu~ zUxVzUmbsm|>c3?36}SY~@YRsplg%uRn+|^xY)`}2cfN;ftQKZQ8@xmvp4aP-7&2TVu3f4^xzA3$(=4SK+vYFWh|CcC$@j zwk2S_EnRVIGeuyPJiu4>ykt-z*CH$W^-jjFg4mygls_=kCEBs#BUT3rgVMdyWd7!bDnplw|6l=se+f3*G)Bt8 zM<#Gb2_Nmq|0xxF#DIF)-l>llSO+WN#Vh}Xskpf=kve?0k4SJp!dpG^FI>qI?V9y( zgr*P4)3Wj`cC!9um1E0oXXGiorte@amzgR{fqd{YncGer}i=&+zXD*-eY4;hEs2;kDm9L(SrTPdZX$!%t{wW{u z)<*)DRhAbyC z=bF_NUcs4p@cmDMX`na_M?5$a^0{d$bl;vs%y}1&rPdNjG(U6-odnt(PhGig{q*Cc z`bJ-F{@HYqOJ2fFS?8jLGf|hW=f4)`KIF|?3=WZVmK%bM%K~^=7rXan`B53;pqBHx z6ke#n5dU61z2ZqZ5U21moD!IK6;j|@U68K8p>s-#wXN7jgdwwlVPux4K(F7 zyVVF-qQx>+ow~RqvVBJjjYriNp^+Z%D5YW^qi)3-DG zU5!HH{U$gb#VdB9U({6D7NAAdm1F3wuenJF1>x0MHzNwx4(m`_ZE3W`AuaT-l@@-w z87snq(F-o8KNQXmqY3QvX@zD)Qamn=UoIr}A9^$T7DxoSf^w6FqV6h@!vwh`MF%sL z!gUvH*dYymr&VRXVA^g6y)_3TbJ5H;CatMhB>r;cl+Jak`S`KwAF5jRAyVf)Vm~qK zJwQS2{P5TBAu&~aK3l$R7RpVR+lKC)nu}S>&W(e2GTPEjDI-nW#9nZNi65<*qKZ$X6|B zvxr!?VCZs1OfZhy4%>>0i$DLYEu@#*7Wc#a`EL2%205Z(UYOf{Sn@z*AyVC=@$Ica zS(#(b;5bqOZwjzT4iP_vzRGoaaNXk52Q5t8R`9)vE0QDmJi6_iO>BKnlKaODG$=Xb zE^RGjYugwo2v79z{{*>lG?f@Rg5H`Ekd7bq`3~On!t=_7#5?(N){5}RRdI|S|B!p$ zi+mq;Wf9pkTal;vA`zR~JQKN+GA;}_cBU_aoMrj-zQe>Os!MKM(BzLi*TU#Qw+ae@ zZhe-yGA_x95fLvxE#MjDwMsQvLIvrO^eYb(7i#*{chX~=pfK$eQ*r&X0!^2iWst=# zsqi!gnI+3h{lktu8p!f5d9r#7qem6V<2Ced;tubqFMGq_w?aVnUJ5dqKZu~m>O!Uc z327lwQD@ZmvB=26Lq@Pq;-0X-zo^0dtx3ez2$fp@Li?yd?xHbR{#7|3p?}36dF`KX zdmxP}@vZcWWWPrgs4Evtod5d53^E02>rEEb{}Bs91~Mv;2D{eX5Vg-U;HfL$t92EH zj6QCjEkRwdSB{N_3CZu49+1p1IT{q0-Fehx=2GKaHd89U=8?L2WT>xg{>gZ}&{JL& zcPxgk@x#q@@f;7&OdTD9&#zas(nK809OvB zd+-H;SaRaQPl=SaOS!QJr7>d*oMkoB%Z~C9dr73DoTt<62pUPOW|~Ah7uv6mf$Ayd0_6;4O+wb7dU**!jDQjo4m zbe99yal0KG98Jb`q!u6#o)7xCId&|3GF)-qDVS6|p(H*V6Z@-4yQ1ghFd)^Q0d)^5 zMhv1beMm@Sg}CT8unST+Q56w;FzbdywWEeZAUTJ(iCg8ZZ&&P1kXw%`Oy>IWlivX< zX_A3L1*DRVN9WfquVp(nZ{D)o86Q0TnjE6-RP*7ugLi!=sC6way<7PjOwt9a2zeZTVUXX~ z`Ig&@+0@ksJ72buKEmG6?W(SEZu%9dQZ-lkBQdEkSM+pWktgR|$6;@nH&dmlV;`xC zR>LORlO1OKlh^mlO!i^3{;vi%H1}t?&*5c>L!Twcz40hCkqNBEm{Sjh4_X)1p0_dQ zCu_W49SDvXYY}{G79-*~xmi8|CAG<$e)Jl9ip73mIN4~UPYN+oq919(XUa;UxmfJD zajR^rH2e-Pr&l&{unel-RhIh4-akIcZR%e3-?=z2w%{W2SO* z>+O5TQoPJTAIU$I4RCIw`?VJUJ20qR{&u{^r`)=P?`x&(s1KN2>jImkKtdtV!6B-O>-UsqNyJ*p6k6ojRw$*eXF9we)xz1qE=j1$O4na_{rZ9{s&wy>W z&}TNH+_(WUv%y%y{u!$m80%i0`cIIYod-IJU`#31`rn9R9-6Lk2u4jNEQ=b#{K;hK zht`n-7Yd3@F8rWtQV!2@VK=DT3gMqLuUik&>$M^P#eEdv<7L*IMeNqhXLC{K`c+Q5 z2qFY3Wd}>d6St16(CGO|g1xqU#p678L($U-*qO~rjoVyR;FhwMogRZ~AVy!<7}s7bqu?GJA3{rC*cnjU$lf86Q-|P7xGU966jq{b)pef=*8zM^|UAGT*@3dDhHY zsIw(Bi`FJSl15?IM6~IBeZp;LIFGRrE7(pEs(v)A@O^WoiS^llRl{e{E+wXlm6f%T zqY_j@^_U^j#%C4JI;tkI$Mp{gIrW+FIEnCFX!*<_RE+Zk!X2slEvd;GO4sa$$D4kp z1^0u;=B2-1<-0^zWhRe$4vPaH#na}mhPm-FZ?$7!&-aIU42HXX; zlXp}v#;bq<$6(!LO=sCmB+>f77e3ToKXh1Uaz3a15jK%W{$;Zsec9G1uxiY9)X&EL ztWn*vU3Ku}cJY3(#cQXd-TX6~ZU1PJ)ew$OW|qBJGo|a^l>^?cyOq*IfvgLdDHnaS z@XL_ahUReUJTli~{oV_Wfu1$*kWqZJ?VdMO>AKw%eVw|douYeRraxYvF}>kgrS!uv zfi~*?^rZTmfZ%F7(>u0rK5sWmYPC!WOk%G2md^B~LsP;JpW`fH9uuaoGPhL=#nD)etgZ|gt%m9x6N{e zN9_4;)%>>tQHsVNV>hg4*gTJSv{o^A1Yo(krK{}Mhoicoh`+q@nxaol^a6zz*Ir4V zD9!M?ykI3S>ru#^3fTSbyjID}`>_fJz5X*Ob0FbVJ07tdrd~q-4FgR+U%ml- zW=)ez3R~c5n&Csb>$`CVUeXgHah%<4C27X-)IBF@J!y8Op@R;*)Bw~72~$ymUOMp) zkxqP^q3bcVKIHvmWRk33<|3=6QQO`bdx<``#!vRviKCtS0KiS1&lrCVz`9f?#@n<2 zG|``{Fdccq>GEe-3@d`le%E@C3%1037Br!DRVRWWQ)!dgnrD{*TDnpu?vijoB_-kV zH09ufh-(zXeHyD9s+>RnU;&q8H!%Z%hq4qxti*%T@C8&fzMDu(wW92=`J_4ADy=;_ z_}$?_(OA13&w7xe@d{hM(U$N0ZwTCHD=O^>U+n8I3;zHl>lA;u#hnrxi{Ar$M4j!1 z0P%9R3YZ$}dAfGb$?KcoFYf1U)(XWNvKI}k_vFNaRFQlt*SD79kVP`zM|g^M3D%9K zJjab|*RN)tqXA|wX`<0W8=v;*+`fs^UKyVCpZtYO0eXdRm#>YMnt2h=QPZSL1QG!H zMRfi}XKYIh*GbpFb<}@1bE)`(XVsl(gG)qf( z@f$Zs_BUT6v3_iS&KW_xEQ|!ZhfiF%d6lysNdRn)X?iX?hk}i)RmXhSo*e?G=MZD0F-^PXD~hg=~P-feE9;-p3}H_ z!dQzYvIGY9X`=2Dr3E14A^ip4o8nr4cO}*TiAQ^#NkjpdO~?ofBYfOv7`zbH9P@-<-hd1O&t=Ck#A1EKq%Ht8|Y7}Jojf_FxlWrYO=vBsGP zbN9*ZWfjR!G7;7-k&;GmJ+guMgu;PxL;6wb5ncWg{zA)%#r@{+9bdskY9Y~H5j)+e z_A}GmJ3mTlpK<)k!--ye9%Z4lK2+YiZL?VP|8? zg)U=T`+uSCA>KxYqxyeYx3Eo$>!|r@@aEhdzY%R;5OUH18YoA9?rrIfUsZ+cz73~R zo}SKQb(frs$Tj&5egCs0c;*tw#_bJs;bA}d%}f47KRmh1r*U28T>mfPwb9jz0GGKJ z6oQOg6U(9$Qz-wx@qMqef7!j)IKTFa_5TRo`a-O{h+;iHn*Tg6G3LC)IpWGYIeKX(qlf}z+I~V^up@nVy8P7(xOVW+|Ypn`DEK85( z&<&ao80EH;7albstmoGAG&K%}CMZvgqAjN2hc#|zUSqZUIWFqQ-E}hSOWnH9Yzw-c z!*3D`Fo&E>wj3|clxWS}luiJu!;XJ7_NtekIcv0i3d}w=y2R!a0>F9;SP|djX9W?E?%s0%gXl|I{7OgNOMB6t7X60WRPfo9s)Zn z4w;a7#=DwV;Js22Ai9qy=q+5&@tRt1Q{!Y(e)y!c?@iEUZk!SckZ)44w=XEV(j;P( zXQc<|egf0Hr~vou1^H(0GJmd*Ueq+jQxd(Q95#p1rPY&>Lyc=gtdo?lXH6*I(N}dh zU3=L#Zts+uZ^h5rKZ0%aN|wxzy>BCw(i)mxA?0ACUjy>0bo2F0#@-kXW*p#VBm%DR zxZzggiGog}{Sz$X1&tk}V?zrgAVAlnIXm+3qE6~3PDY#d_qlsDL*JVSKj^o8>Q>gm zINc0$JRR+PY-Nyif^2F%uS7NfL}X8idT-`#9A$x*b+^cftb+mXdNTE&2&_|;=DCa@ zE}V4fUvZw^suX6|%gD|z0y{djIiCoB8xWJ8*cTGz09i@dQC0(D%V#L(f*^}Q|8^3R z$OCCfg^IU%IAhTCjB)3*IAwSv^;aLe2;aLcGG9L3gJNufB+2L=sz3#C# zCx|}GI!QJYsS&V2^&b3v8&E0Q$K>JyVqnbd#hU>y%d0kerf<+8?T_stvxi3@6S`0S ztia=6L780-1e^tO79YJkxXU?>8FRsp3c}z^Sb*O-MrMh~-qSf&e~H}Q3lZxDh*3W~ z$0aYZ+4~^knyfMpu6C?$G%>CxR1}2o_u?H(RxMGVB~)Q@TB6+jqg)|Q;wD|xyBJR2 zC*;*Hv2`H&F+T!iqAK?ir}zg=xWpZRx5iS1(TD8zu>$+~{&ykLUxdSr2y-bM2~gDY zBXO{o)hkhd;0mNw+mFE#^ylAVbI4>yyQJHHi$r6rA%TD?`Y+f|14U{fX7{JO$lt|$ zz({%ezj#Ut@-go_p!`xX&X_4Y|NAl0-QxmwQNNfDD5kmu8vgSWrr zj+rJWdS5vlySK0?u078aQm@}MSBjjg?^G>zN8k_df7rnP{Up#s8WbU?# zEFHcKq>mW6ZUE>1pL@5FhlU7_^ha*uKI zM)6G3mk>j*#ebS;< z*@NGoercv(gxi>^Pu=aV?fCI+B>g6uQbGeqCK5w-66s1fo_504<9s;eLJ?{4f6vaI zyesmW<4o0`7dg*Cm@Kb^kz_GZV(GUCym?UN7hK*3Fe`j5e48r#_bM}qi>vsSzm_T| zL!K6cNedy_MWPbZD?fwEyn&3Dx{_YFnKYgpe`^aEDBLitLy-@|xS1OF`v`pl>&V4& z|A9J$PF$yT;m*KM!@);467OApyyEKm<%ZjH=4v62Bk7C-$@R;)UkW8sRx~M=NP%KQ z3IW?+-$9yL;o19Qj6Ql4)yougrK_lI+@U6YH0i(Vr0NNfa_mtkAgZeuMFDWxLQT7riJ&MSl+4qyF#bD835ugYQ;Oe0TI@b@F~bt$3xBVk7(0QI#_tVsDOF>T zG*ke7RYTqzq*|@~EqA^YcmSBb3ce91e*L|q5Rua4KlA)YD!;;H0FQa4m+{p{EE98X zeqJ4rsj%O;9z^DR)T%vD{1wRy6QUk|PfiW04tO2-f-9d*95|iq4>yo#8|^H1a&lha z&HujlYj@XMWC}>gfXN^Qxr_v~a1ZL!>Mviu`0A=&uV1zVR9mAbn^oFDb8>%5?U9QN zIKt#WARG{Hfs9F8V*D!vDFSZ3zSynp?R!v43PEuU3ZX$Qhs?=}-GJ!R`tJb&XZ)Z9 zi;K7q^ri=c+l*^zZpP#|qAv>sR1)K6O_T_GK-{7iH1YXHAMtgkEqkSFGS}xxuLo(q zow5w$F%BcXabqZyOIOZX4(&)Hb|zA*Pu31HB1xV6_{Zx&@rXZ4Wf$e(c$Fznf7LVZ20i-@IK&3`KGcNQ?inl?#oBVH(}m-C&HOl=oDVZxNM4{ zY2M8EsrSoAKG!I_1v049lmIS0)#whvykj88&PLg0;n~SS$d9l*cWZRa6e9hnaLvo^ zU$iKC|7;QVq5t+sI}8g z4q#wtGEC(7*O2;+N{cTABFd?-<^X05fG^@js2r5r?VJjRYS}L{VsX-^ELNUu?Cb^J zPMc$!S7MqkFdFU|$kfpH^z__sBsU=)^qzf@w zq_~_Tc&yik^1cVwqZ#WDs<1p=d!tM8R2DVzf2|!*u8Y+Cas-T8sC&HH|T|RUYFD2v*h}#$i)6 z&vpxS7!taN1ol#E(NSF|KHvmYIA=*v#23o4Slevk&7v8>j@+=mWTC&puKSUg@$aHd z2JjZ?WL1!?9Zw%!N$$$;6j{~AhDum-oE z(GTnn2h=tAw3AGYwl$kmJ;6gZKJz=KSI~}Zg>V*e4cz&9wlGFJo%gxV)n-k1O@%SW zORe%_pgp<0ti_F7DQqmvrviEeo&E(BbydBfckoex*ZqPY_aQAz9KS%TL!r6(TYKGE za6Z7-M5Be7qWyR37qW;5X=;89+Xkc@1Q?!(y z3`#|`{Eg~eig|puP~<6X^OnQ$6G8plOJ{x3C&S|N4s<&FEHD(S`s{@GzqNGWd7mHC zh=2@ecJVz;z#b+BwMl5>4;85kQm%s+ zmL?%<0HpG5#=C&dOCAbEoUcW?WN0<~2-T-h9Tp%pGR3l1Q7Pj~diLbyd$zY|hdqoz z5wFqljURhHX3V=i;}wu~?yY>)<%&d1X5w| z!2c?1D#kdZqig+hL~oN@d%=EV>uBR_!=}n%^I-P#_5`>70+MI%ZHL~%UWA3Duiqw< zrsGz_G;h_BLRv^GmED}b<>UrTB^m0eVNzjL|DnuUKy`CiKU(2zUVto|XkOAGK#!ly zCEaSvrQr*)F$_yrfcR|rS=GI&=Hrlc%hez*J)@=_JW9)^-gLI)iLNfCDH}=65<5ab z>A*mPg84`cm0Pj8y0;BhbHgr19rH0Y3Mws|S{61S{?b0ivU249+&;w^@{(}qrARec+5S@ndE<-C$CD$b%N484Kk4pkI6noBTxu8#Wm0WoEmq_!&? zTit?~mcl5|TSnuh4>HvAnea(la5sl?>5*JYZZUElvtJNkbfmX#_C$P{vAjX9)I!k1 z)o2;UvTBy+hO;tiQblPl#y04e8!Las_ZXtj=VW!ZH<+yQeE6E4dY50WGHgFrO1zjn~U)wu406{jEsnZ!pTWfF$-t=}8OC7BaPC$LhUc%e^<&)bzfW=vU_epjK%)gedJA?YLIbWhm*T`38Akk&030q`J-WTU)LUU^= z7tgk?&`z00>5#YC?r8$2Q2ZlbKatqcjL3rt2hMi1mDoCBmmR)t^wVnU(=WNxSawTl zp7wn(vjXkIt~Twp;Hq#q?JAF9@<`SJ^6ejoxkd)KLOSSlrk2?m-h!P*L~D8opEjMU zcFVC|A2IFVUwoYKK9eJylqcO`e+?6`l$z(A0U|SW=+#_tJ)+}XL#YfCeUmY<%5;CriF-)Sbs^202-I#DIgP!R~p|OU*uyqrqO~RESP=rQiWNP68_blCIg|P`es`I$Q zgy7-{orz+ry%u@AI*>WbBU^OrX)(-u>@&H`W|17Rb+i-@BUFT9FC3v}kA5h-BXoYc zlj=Klr4z-9>9&vxAo5b^#C$9YR^TY>W~u@zwlA+!^|elvoiOLS`czK4 z-S_q3{DaeSBC~a&+Zk)5xDslkGj?+&8Eok*i2b4}1&Xf41DsBrDr(z8yT)$jUz&^P zdWrW}Wd>?C06}1Alpt!W7cQk7yE$l2CUTF;)V0ZsC(Y^iiQ>A7PybS1uN!Qb!c zZBj%o;Le+lrZr`RdyKy{=}l5d^=*GAcSCfDkMIWlqlY;t)LRb*6BYV~zY&GDgg@Sv zi#3|A^+cgY(q(kj1?cRLV@ysp`q0={2Y_}}@vW`GcgR_jTF<{Hi zRccSY$oFk=?HaR67Q${Uy>mWHM2HBS#uN+yvN}$CQo2HI|{=Jf|FVc(FJiEr)R{@gLEde#G~V z$63HB?bi=M+(j>W5{2f}7*g1*>)cIM_+l?)>Sonq4i5Lqt90qM@4q#Qjw=xOx29@} zRtet)UCB17C+%GoOVG$1{7OYGr!^YSQIkxnI~9m}keI{Yk{!`NdNrJRBOL8rOX*UK zd?cx^z|Q~=d39L3Ce&s|E`0u)y-;`rgHAqw)P(IkBxDHxYhqK9KMsQ%I;yDJOEamF zKJEdoC^DN+YEfrLb`^!zcOC&c5yxN5*3_E#l*K1?eckw#uA!|qCWfEB&T1Uexu)Wd zuSca@!I}(C3~_DJ7I4hnLfBFfOHv9XRVeKLq1U>;9jgs zE)#Z(iCnbaAECZ}Td#4=wdMM=Up{L$>7RT#nf^y<6Q&T}g+FDSsemw;-b6tVN2oo- z?@2^GFYUnmc*+UaQx`Nq^hdiyCg0KOuelcff(M?EpF;+gJ2srrq&QD&JgA+~3bSC& zJ|bn>QXiW@=B)y>A>?vZX5Wrl4Ij;M+wjRsifq@QQzy8h7_<~K zw>wUcWSPo+cGb-4v=Bd;+sPpG)_L44`6yn#i-d9;ztP8Rw*O9xg}O6OGkwZ0(%4Qz z4Tu&-wbG}+&iev~63&A6ZiZ}~##H1UiyzraRPpHrX-j&bx24Do17_$DEl;+S=aJF# zd@9+1vonM8_$ZOni4{IFUa}Y!4sF;fW$NVYG%IWhtqtF|Z4VNA#xeV1&qpXBOz}*J zlU41vAw>AC`XSi5VJuvGCtqDD!zfUAf{eZ$s-2go*3l)i@)0YIuf=rA;`1~ZjbI?) z+53Isd^rBA1fe=AYBqE+x8S!|U+J?qXGfWpdf`wp)8O~a_S5S$6lr(0s`aQ(RUc?! z1a4IjxtkxpDE!PZ$xDadV8}aG#8z-s(YDCY!#*O;c0NcAHG5);(@yo3WJ@znqwl@Y zIzJtX1NH4y&qHZ^f5!!sNqst)Ljs5Q73RZO817jd4l2#qAkZADi)^zUtZ^?J4}3zs z#w^;i$$FBvt=Rc!mi!nYL7`2)%FbD&$$1-NB^JGG5vqc1C{glFW&-mZYS#*wd z7&JbBk=XT3^D~bdusS6~pT4>}VhhZ`x`$^hCvJY$kY_?Wee(yzPMjXlIqqN~USHvZ za;@;DA1BWejL@AbNB74!oq0_e@ujYO(I*iJ#i8kjcO*7$+bb-yBlQ>?vou*6AtM2sko8f6Dw z^)^$b0KO4!Pd0?VcAN`QXnG<3%=sr6*pcL)ABp-$a%g zW;I>A*Z4g7JED}D84ph2X%tcl|K(e3t|(%kd0m{=UH!0w$t=V5aXL@8?#>rsDBXiu zS6phQ#qfJvjbd;&1c7hewc3|D5u~D(7Io?7bG?et+G@r|QjBjo%E`W0Zk)ZE^`Sy} zywXH^)s1b0hV=7?o2ZnKTNqT90oub=jt#t8W$u!`W^-42BIv}udKIe>d)d8)89$g$ z(<%6rAtWnrVmI%)qS-}AyyoCe{^0>JZua8d(={r;)tDYQl$>$H@lI0+e~a)@ghXp0 z*`UKl3A0W>;o)6tDo&`)G`cmJS`@K6%~;XxHjlSfa8DIgc)~hc_RYzp7k{vT@_YDC zrysuL7L{{oKNiPNCq5@7M_q3l1Ty%5f(MUEDAcxVl2rFrCn>oL8D8`>Mki@!n}N{- z!+^X}>upR4)B=cC@oSGOH^ezdW}>)sL>fp{z7c2ya;b%Kw^43vr5_9Js7W?Qdsh{t z0viu$`7u2b+d~0npMmSO(JVQg+nC^GVbrEmVi;_{3t&5d1hqJTI?Z_ML>%&LPo+=a zlCWc`_L`Yf#Hl30{D{J}Hwd=7B%|$1gtyb&Ijr!Fbg&{LI)fa*f5XQD6e$@_WU2Jj zE6>k1ldC?ejwE7}55;k)ZL~U$yspC}`gq^@l~-XGBkZ1$sb0YI;P?Qv6C>|jajN}% z%54@&uX{Xkr&gKe{*2kst|pt2WX-j2UUz!>V*OqshICig zm82>8Y6I?#G>9Ax#Ze(FM70D1osOmKDzkU6DoP#K z0uen-#`+qG#@jbAAH}F)EF~k&sZ>Vry51kX*F@S`gG5s|ASmomhTQ2jdoL>`(H+agOFBPJ%O7-4L`oihm`m zD?~uDx>`k_@BQWRPx|EgjPcU{{C!s{CU93Tbj3l&yA=>4g}*}L_nu<7%Y}L^im5SO zuwYN(|M`%{4Z+-nE_@%}hwkx?grN%uawxX9C~{DOulv}1tAPX^@nqzUQ{4vO_!m~^ z)*YzY8_)>o4~TwM!bp|FqbG%l>h_rjHV z3V0`Mbq|5{|79SFspMUE9I`*goWn-lV`n1=0OT~2WX3J#KhM#5P23-hG{xLfnPQxf4nE-$Em%7HcR7}-sN;`Z1FmTleZVms$iRe%<-r84 z*Dy{K@AGUzqWRG3WpP?Cp8|_aZNtJ2NUcCTYm>^O1vo&kyqKdM?_r;9JiS`O0?n-- z>T2h)K{mwgvtpGdvyQESBA_(|BZx#`)01eF0pW8-rbC<0K5_fXt9 zixH6^hkNIC+iWy7!;niuw_G5l%?;!1#V*VO`eUabA-R2*?`HO95wKGC*a$yoL<0P7Dc<2 zB0maorgA=$%QvKKCnSO7qEW=kom# z!)dS|@Cu}qJpF`{^OndXCLDL##bRJF)JIbNfgc}$wu_^MiM>y4#>`}9Lav9OR;%&s zz#E&8o^yM0l>Ke|K3i~BdxhOn9B2HtVkt-Bzlls*j~W|L+~m}XXe7~|&IQ4h8^sdY zH;$!gh$egm^~NluK!cO8wO91{G*e2)Zd$B|W(~w@36-<6!PBk;^TQnLIwp0WaDMK5 zoN58BKE93URf-%h#k&z1_@e7xv02lLJvR--*v#YTMB5?Rf4MqB-e7Qe6zjZ!w`{kw z!058NaMdDYD`C?2PAATOiW>8h{zl&sybkXw$mk5k>J&Y940D0lZkNA3wcSYZX$J?F z-Vw1At*C3-tH6u)oI9io=uuFmzo*;eicm+Gt*H$tlTnsQB#1T*JVVao?={!C*qbGb0IpO(`lAv=xq*#dq7H zgdG@YkS$uigkQH#CDNU>d?@njJ6l_{sHoD{Zz7gUO1J5j#Jjwzue$M~nx@7aic{$g zYxC4Ww1y(|UABIR;qQYZ(=YQK;+i9=4UHCw)WV@)HKG;&=1N1FEfoj7`x>An3>t)v z_4-+QXzn*5d^%!?B!%!=07wS86OPq+@oYtkbSWBDf45(HsyA8b^s4zgv&6&UTtH8|ZNl-ZGjQqd77nuWfOliVpQZZT&Q*-6fXc! zw`QWd@cVY3UMy*$+7Hm8MznqAN>WpQWb@1DUfs1|1yNqmUjWZyp^bml>V6A8PXQ!g zD8YPE_SKHulsgeI2Flm+?nFxH$`tuB6oXtB3RUIp)*X)W{t!udY;$_JRT`IM2T5k} z&E-Nw&`nncrfD`NW7GLI;SNBiUKsXI!A%T{q>!`gqY*)Tiu6zWWkqHI*0^e8sQOH|9|5{7}=vK}$d+Q5@ z{MZH_aw}9ZqXfH#^cT@MMuN;vY1RTxidQfvnO}zxG0iupe(p0?3&C-*LrwWXiR!3F zrW@7dHHd@xI&faKEaz~_15Yl9s~7T+PgaWj5w5k_7ta%R2q8!5QXGgdJ3l4HygOe_ zEJFB^zFKmMZuOp*0rRBdfK>}6=wl#M`fM%eH)~~$f{ZapcsoFm4DA{V*BKi8aB%B3wU1_Rp$>6;$JeOaR zUS@h=En;5!{{b-sFor%sq>;NL%*zINh7io0?z#FW51!X!qVc#uSchCg#BXZ3HMg5? zd*RR>8wC&Rs7QbNY{*)^@KERW;9~gQ0{N2y)fK2(n3^ZXr>zzuaTK1}{=yv_-@K}b zTho$L5!GUjn$7dEQlcIuE~tn)fc<|4h#7E04t)- zHtew(nRC!8fK9w%uxK`}_2-$^97z<`YSqQPOU**x?p17Y-dv{`#oMC}CyLJ3T6n0P z2hC^eB;6s_9H|o#r`oj;sjTx8I&Bg$fDG*VTe1%6ap)3N>9m(~SdA#mvgxW0G>M)v zzzTKRHVIFEomGeAxh&mAnmpIsT_Xd*YIdOkRc2K~v*Zr3kG#ShJao84Fb$l%x*U`? zYVx;4zj(DVFzQKVtU~UIaQHg=`l+Sy)wn6N3_W}TQ=lh43Vjg1LrPv6;Vc%C*NhRa z#?#H!xGVR{mn$%Q?QNk!IE->9b_EG>=a6NuY`@2+mRgjX7{TDXcy?J!ytB>RUqlP)Sn0N-ZD&63Qa)INyO}_B8c2KD zavf75_q%l`7OUQSjmO5u-q(ivlz3_y70^4{_?XqpEHDclJyRE%uh;IY?BswaC1gBqbMz3k;)Qk=`=tOc zrKnYZCKwK9xEc8HQKA1%ck+iq*|ITR$J4Gj5i9P6XF>t(*%93Kg~ z%G`Z=B2OkUx}*X-##nx9-Q8K3qIN7zDe_LisU=N9bL649pfAcki)1v?y+3%leASA| zVJ){0&yt0n545~FWjK?1V5H;Ym^G$?-dvdLX}l8PgxuP6x?zj+jYQ!5dE%QS(ifaB)Hu9}I?!JG9yx^e()y@y+Su7`J1(EN279wW} zCIzbcA$fU|eKL4XBMI1)8}iNj1_3zm(JixqX$!qcq?_`d?;GoQi^V00U zh_LlO0ui?RwS}w~iiZU=^uj*B_z3x}<4BS7M^u6KrGmT;K1PIq)opAX97fu9?Y$XJScC0Cb{+5tT#Pfx z@DR72`_7&d^BdJaH}R>WYt+uEeU@+i_BM5Nnv34o6H;>DB{lBJ7ya=f*oQvveNhfn zV=1J$ct-Rh@;<2Z+!u=S)%#g_5gbCgC(Jrn7mL(o0v^9N>ODe@X^H9PMeWRYwwDvY ztoZ4H;lRhTjZ!xJv6NC7Dwoe+z?%)&Afa8_oeNO>x`OMLv_4worQ7rs&@?;qkJ&9o z(xSV{H9Xr(xmpA|_jmzg%wzOLYy~jn60MKev>jc(Y$_4FDLq;UucNdX^KbN>x*^hk zi%SKSkdy*-|LI@1($W>Y5nQo8r2dPzLTxbtOl@1xn@QzcVw&ufXPzH9@4!kRvomPf zBb+8sa|^m&5AdIWyl*iIzJ4lCxp=Z5`WR{TLws8v&@7cg58Toc-zjBZKY@7znw2gQ z=QrbyjG-PJ2fKcT!5|GRk?1vO+k?>>du(Ef4#MVe(>4eyYuhz9GC}V7K+bH zEtemiy?3wf1@K*61iyteF#&~;+04UYV0Nn@6nMvP9CKlW3Jk`JT59~A1~4xxc3Fbu$#vDm8L&WLPG6c^6^?9JG zXDLh+VS62V2L#^m-%URp_=H2=BO5=4YX{L!snTn`$+1i<4rN~-&iATT=QEuWtA2Ky zVip~R*PR#mlzj3&u07j;6gFo4dmRP!A#h+7#1n(q)V`XB zt4KlgKYjXC3%V6XK~c;UMZO%}=mPd>F-}Ci?B{7};5;Vsm9Akg3S6I)~Q-?@A3A8!-^uGGt+3_B0wUq1P=p@4Wy$67*oS#TziB7qlZ9UcK zC+~!+L#J5LxA+XCm0&8`cln$_fy(|{!u@05x!SETz)B_-`r17`BpN?{srRAgR+&MgV84Q}Q(zUlg+=wJT;*+{(` z=!!=8&I4>@o)~p0R1axa9kgsM+60Yd(d^7vSYPDznTL4Hu{l9k1;7}h&3hnT7`Obh zh)`v$j{1e%+4A)}aTB$H(qpt-COIg7(j=VhW{U>9f}fnj63?O6E)ob;XRBSc7&bX> z#$?$}vuZY!%6?%&cTj*rzVfEyrLOOCkl;4)ABj^6bEN`!5#bUs5{uAF*i}JwlBTIT z4%R-S%kePn#AI!;qx`BvkZiu>T={(=ehe8vgm~IMfT=u*2#(8u<{>639In<-LGRDw zdF+&``}zeu22oS_$-tRWWyl*jVMGtPODt^j;g;J*UPe~?TRVtD-sJ91X3@m0qZ}Lc zWYr2_+T3@dx^?Rof2NeuI*1L(ulJCM2t;Wcf8;LhCwkv*6wV3cqE0pD&?RsJgmWVe z0LHNSZi2C^$hgv=C;!-H`%uCNV8V#O8W6=MKHPZX%axwM#_SLJxc%sXc=Iu;9{44L z_z?XnHNQDS1)>PVuw9KuXol(yn4m8KeChzGG32wanbTd52J^duBhEEaJCQS7h){HJ z48LBL+uGE!<#A=?3o%;{dMDRd8da&)laV@YTeBBzjifEs7Q>(~oun_9DJoXDdi?fv;uC@c6p308gwxjp`w#ieREhpfF42h%~ROZe|yQ80IjP)@)RYr79LA-6DU zA@bGzxr3WHjSlq%zNkZkv~qsED(d}nZ|r+0Cmesjo8$vd$TFR=;N{0|`Iw>fe?J6T zsuBrHW_t~GfYD_VDFIdS-ysSpfYBv?es8i3DfsuSf8Rq)8=sUL-bW%_Km(w=C%P5L{bPG18`V zOnw|>R9vcfHO!n2or>;sL}$z-&mHi%m~0>>SJI~oxC!4yRx`G8U}jhBzgQ*iLt5cr zXxJ1ef%i<1F$+PXIus7gztQ+MNgZwub)IuPCnCQL%xhqYg8T< zu+S72!?W)&qr3zbzk=9fpK`>l0I^N{;+5!^#Lggvb@ywX{m27|<-^Mr>bib$(Ti?X znTuZ>hk$iPP--{~oI{xU>*T9!4MNY07QF!QY5&AJgZ=NCr$L@;BWI?3C%pNDboMci z$tZHkuYCvCf5k47X6OosR3fqwQirpdE@R;xt3Us&qo%y;9@?sJmX%Tya=rx$)nAB^ zuy$O1Ux*kr!Ql#yHcq|>Q61o=IZjct{u76C`@v9M>W;kf$8KFAx~r=uSTK8c@VF>a zrpUnRK7zye69mhc)4}*OXq*>TDqZkrv|`#szan*oYry4F9Xq!#=HLM~M6V94sv<;e zY7`3(ae`hu$m=Lv4vY@M3-8ixf+^;JFq8`w+3YqaOUH>z_v=v3#Uw0|Z(Z{Ng#f+V z4_ec@PvoeRuMS&8&%Q86zkQ2g^pg{F=)#8dkaP{W=#`t3MH&$`s&cXMNs-X<7KS>9 z8wzdCmbMA@2BK&-LrgWTouc85QC z?I!A5rS zi`QKGRm}QVwF=X;Dyqt3oq>$T*?ohX z+2qZ8@B96J@4fHa**%@A0IyH8q7{H{dg=j9$a2?Wkx%*}9|&eyl3(gC7JMG$ScU7x3=1<@!dDP%;dm;tgglHEr*f8fH08$T3l3Y(^s!%d z6+gZLmPmAWIRKQdr%tthQoS%AcqUfnmB ziFEv7i-qEwM<&6>X5BlnV8uK%`9@C+I$YG1LfSQ|cu8yA$=IabPp#Cv0r*TicbrNr zCt`1WpG#g9pLzJVT)m~Q=aDyY#2?#a6St9I>|u2~t=JGZz*7_=lL%|L;!cEY~R`TFxjb-SsE7uWbK8;R$lQzjUB!2IF7y(GGZDrzw2 z3%p+CeE386=9t`tKJ^crb}jk%499_p_T%F?Qq;)R{ac!jV7-FUg->IG!J!Dk z_^7r@uD*}*QSFx8>A1qFK%I?QjVO5M4)7FAv~a!h%WEdt@_`)Fq!c$OFb~jOVXacY zY|u!#rpaKsYoSXOZf}rd&N_?wC26FH%1zEBoe>k@p&fpmbM~p*Oviciwe{v$AJtBY zta44qDwHOoeM~+sy0f_IjxhP zNDOIjN+EXoUq$6dVZ@Vb6Eyq6Z*416^b*ShwE?2`hxg?&@}5S$xFE3es7Fd6$;L^P zHG@>H!Q&{rbbsMb2BMa%+9bJHcj(tTp7X)wnA}pvN3ciNYhl+On^6w531G7A)h&X9PvV6&sn9Psaxsz@spp>rVt`qfEstj8?ZB)AFjx=Y3K|i z%JY!sMSOcTfrgo~??7xobHn01GZ+7|_NG;3?K=dKh|C)1oB*O-!A|eAa}T+6C=pr! z${dQpZOjtx9bc#93o1U5J7kW+(tPMUR)My#{auw}D8ib9d#H#w>fpcG_?lxrg( z2on+&5wDM$j|Nc4F4cauOzWW6dU{K{+k@eaU} z{aLWc3y5$1J%<>+ZSmA$3_m*$R!~9#<%R;bd<{wwgYT}KEmpQ4?TrTZ&k~9c7|59B z|JgRtxQOyxzp5U+VBM%elRX=X>5R}a_9Ir{3PCW;Ug>`ff*=PZkvM1rCDJt}E(@d< Of}++lBd&$Bj{FNm=%+IP literal 0 HcmV?d00001 diff --git a/docs/images/cluster_id_two_cluster_example.png b/docs/images/cluster_id_two_cluster_example.png new file mode 100644 index 0000000000000000000000000000000000000000..ddb4ba28d77326c2308562a73e876a2f1d5a213a GIT binary patch literal 90477 zcmZU41z3~a`@bSeiL@Xc0~7=VBq!ZSh?H~-(jlXnbhijd3raVP9?~G)qf@#@4H)qs z#{0hC-~ZZkja}^7dCvLV_qpS9-zVgik_;|3Irgnvw{YcTC12mVh2C)M7TP}MJ=A|v zpc9AW?x*b7m~dHQbYgs|wv>@1(Oz1n{v$qfq5I~7 z3YhnZInzP~GUdt7nC;IB@&<3(b-D90LnYw>y(&tf9S66qAbdT25Xqq%j zz9P6yfdBtpvLLk2>6qEXK6(n*!5=XH=Mt4C9L$yyMVAyskB$F8sNd~w&cpHV7>}`U zAOAb14}dnqUz2b~Qc>*Y2a>c9^cB5dpMDMS>tAJ(5a#rhJT;)@1%WJm@^(Rq1|R|AY5dBBK?1F2{uOyRlIrRQaEb14j>8Q$p0)z z)DeI1>CH5LUGh<=TfTPcGlczKv}iuTpy%(k{<$YZ5;a?O`b3`Kn?Y8jMcx1J^YDEv zh+eg@PD%bOIYnRjz3ji^{~3h#46DsPP~Np2zgYLrll}{`s1atD1_~>|n^ixMar+m} zUxEt8z3oQwTVy`ss7DnHc8&h$G4jbDI!ON`(XKGmn%ss|l7FpQbP4}e_W#8(^7X!K44dOBDTe zz=zc*H$TdgKs{x&cH(p1fALpBt9$r&wzBf5F&KZ^jAj2tdmVjL^?%fbFcr0$5N`~U zCQJaQPgC{V$@8Bk){~>mHqD6?*1MWs)~h*d9|J8E945#I$%C)RPoEyGCAGbk4rwlo zPM3^NN>8KGHXw8uM16bnl1=0JUO_Qn|Dp%3fIy7*5`rDshWsbakGE_NR=Q8%oix)0 zB~VL=!24y*ei$zaV|{jY-j(P*9u^Ja4ETa;7}rp2EUsf!y}T(bVb(?KC)KNG&) z!A&{f_+q5*^4w7exHlV}H3&vXq%*?pLI4e}_}AxKB{prr1fSBga^qKcEBo)V%Sc1@9A)_DI$?2eHvw! z3eoY|hC zqVo8URlkKJso`Er9_@&*y+`wWP?r!)IAIMILNRj-p z#(0LJ^2>rLexoJLjGMiJ@AE|49CvM;dfVew&U%Dln0l!cJRwS=S%%f+ z^Pe4ovIL^}gt;Tz=$Z?nnRf<0tj_h2Q;>7Yxxyc0?m?0L4}?z*O}31!#S=6MWC_tY z6Yt%u!V~XIxWh0ww@w$hCp4W>)@x(3g7-aFFoFC3H&8WnqCTxBI}ttBI~NG!hnfn6 z`b5l1l@OchhqdLbZ!6+W2xRHd)ZF1hD@d||kt+|YZ$&!KS_0k_>6^QZ2K}GOTB0nd zYTO6#i4Gsw|FV&Xl?ne#+Wwc%rW?aw$Me+-649ULpwPR^{2dX!$1vaHi(t{}IU0-G z+1oZSF*R`Mjw~MfkEz+kd`x>YWm$K$7sxXMI|T!wCrpPC?k!f$zIVrSRTvsGgqf;e zDoJJ7qyPB$WEmRe$$>Lbu~NuAt^o4;x--^AbxikXQvr#hrT>}_+DXh)bJ1%BvYj(2 zp~WiCGsgnGdQj!^M-c*iA9}(CWEY}5c-Caj#_8OcPdebo@vI8X&9niu|BIlkAsR`( zHTd4qfd5dM{3wxr)Ad#TPVHLSP^9V=Y8|1nA8uAj4~lZDZsIL*G5I)~^#%BFO~MPM zv$0{{+n#W<6|K@~d~=lXScfOH#v%7dhLl{I&0GSUrK;Lp5X(qy>9-m%&FnZ5)(LID zPt3c|Z2>wZ3~w)pN+uYGV`@Uu-6TUPt}31GLrkL5OkNoz9$s|x_mCoDJ$_6X>7m<% z#KRh%DV5Hq69IWC8z1Pr$6V8o!WE3vPuT8k;BTQknYpto^{#Axe)3{zAMS|Ph`wj% zu#71DvJsc~#!gs2Qa1~MhEyGBoJ&lf*m!&cCR{pWdv`kh_9eGb<#HHG7|R_I8c@$O zi_RxtDTWMv`h^=gER(d4i}2M9g4lGL|41S2dzW2@&UlC7?m-fr9R*&g?r5IgEXzj@ z^RR?sOy+!FNV>NGZ(%y86`fw8*v<7hh3*aInh7^(+q~5QqnWGK(;gI zud3HBZJO@E?*R~S!@+sy!=0eBowO|E*{jFz5Y}PP_F@ zm}wpq^PSUxt+Hp7%c-}wASTswzza9g-a?$!ZnqcM4)dN9aX|l$s;DGWQt&sgv*Rcr zY+W4Xe>boIGw7Gx3*#aqCWIa+Nl!y;Mv(Q!js+`f=%LaX3Q1fYh;oa@v6nYGCLeSw z!Rb)cSL3jFEcI@1d8dd0%Jp$xJrS(8;beE1Z%GvWdKYCpt2g4%6oXCOt$D~WW6 zvi2L55#ZNp5Wswyj=Z;j+Ak4kiyA8(G`gbYgjGNd~` z`yqB6ArQNZy8_MFMha7eiow87g@h47OE-a=k1O*6k?Jhoj^TuXR~pWQ$cnl`fg$eZ z7$_v?C+m2PKe`+Sp-J`SeEZ@GnH)!~!PrT$rL(2Fi?F9NS-kLzkK@JaeGdmtySOwX z${y@!^Czd!0LEm4V@&H;L(|yd?S5waJ_#6ghCj*hlCRGZVMdw&3E5C|b&8w=Z@zhY?f*tvnojRJmqUJ;l&zb9YE_|-N8t>HO zI{IeCVJ{b%kX~<|QykBtUMh{d1)FjP!w%xDbdBSAM86O(rZ|gH@WO^WfyUk`72aAs z&9EUSnsn+5-W(%0%Y7ZAtI{6=B4j4Zz-RY#Ya-hcIklE6J&bltZDv*oI{=Z5(2UL& zd(+59_FAFssgki2#56^;C-$83Y8c+sS>!X#)i$2kID(yLnYc!mOMZv4Enbn7w>Q-lbOAsHp9#L@vG3dGVf=&?xN*tCj?bMDYzGso=Izf~XNr35D_ zNP7g2E2KE?UKl)7TuJ4P8OLn6Ks@8W=U%MUgb+GvII9_=yV7^;e{rc>v+Fc=2{9bv zQc5K@^DfR3zHxE1*yyjh7I|EDb(8rWzO*7{UFIi$#7|d0=9i)b<1>d)5UP|th$}+{ zPwRnNRhIE)Z{JQOa++A9LOyn_H-wxu&0(iv7mV%s!F2h5;(2Ox-)`FUr}TO4twggI zcjm*ja9Y)M0L_BV)`}IGh;B^kDOdZiK0%&@nkk2`;@9aXedErw=}KNj#OSJ|fA&U7V@r7IrxG4)64o_2m?awE@h!T1e%Jf8pxWByZ3VTSv%=u)OvJ3g@b2rk-B53JFemg3s}L__CO5g0z=1H*qNlZ+p))L=FghhZO{3 z3!`J~UM#5DkNw$73%HSf7469>n8prYJ8!JelD4Pgiv;+r^I$aa_hjZsIHAI8k|~2I$!;MbD4> zmq-*b@S6+xQJyB`F*W@4*eg zQy-wzDOO7JNMJ<0X@6hR{>mo`nmx&n5 zTpX)1z?K=g?`61HI`75kv}A_abHIdm0rx$DUanm~>Ef5ew}U)bIAsrF_UYgfH0S3h zhA-N}(l4@YQ#YkWmcF{P)WpY~oqB8}5dl;7dulM(PWPYsnjqzcT+f0$V&jjJV){}K zFZ|o?;$6?Yw8Ozmf4naUKH`k7F{C^%yNBOD!~k3K1SjmhOil=y6lqy7LAZncv8+*F z40J*5QsRi|;jIEP#iVcT=2K((ZqLsc@Hh8Vk6;)nE~Ou)h<|^N1QM9i#$L0R~$ZR4zKvV*5g&vh|?eNswF zJ4-rZMnleXDVcMXQJc{WQMH2!^6g6=y(S!HG+n)oiFD+HsRtspLH4# zTJ^%GI6Q_w!?2D{v5D#UW+(QBvvajCIcU#ru-Z-ngQG*~<;#i%FYFE}kJ7%*;;df@ zh3F9GsVP3*_KTXs==N6HIZxkak>{^D+wUZT7ff}uvI3oFkd1?JiL_#3^BMcken@N_ zY`eI|6h1oZJIx3InopCP3@Q5eXfJ_ zKHFgHNlJVYeEkLXj3eJRbeInoWMlYZB~}s@hwrs0UU~bIs0eE{ybox+oJ%Nf`I$L* z$IW|Jj`&nwkmYqLGo6h1hr^81osxM^zUDVB!amCr)PhqNQ#dEMY2d`mf-h>YXONOx z*i-hqM0ZkTVpF;cWO)Ug-53=eryj1>J)?9w{|S^Z4s?+_Zn{WG&8KskT65_mH>6rK zPpFt>eqwIQjN?@w6@?6pWSB=@?9xDnh*v*OxWuhIn=U(Z>MeP4Rk$=znBqXZ<{|q! z_N+Cm%+r{4jVz$C-_ASWnAzhdcM6uwSjQ@nog)-otLCP5U(Xyb)&nkgok$-w?^l$U z7pmqc*|N@iA!g0hUfhi%KdZvj;AJV2 z9VXao3BZZ4EXXj36_KIws!Vm?%FM;B zEy{hJTWI#1y*MzI5-;#T?2bSCIA~JOdjpJ%+U6&ylh&r zMXwm*#l;8)!HZW~ZEk=?I#Pp}DQMqZtW0u=AsuK{6fJ;FuF}ousqr_Oz1Ig=#8WbX zu{Pln+r2&*dd2TpgP+N)hT1_ehdV`b55NCtWf6>h@uKoxKq&FNQg2&7Wg?CjHhtU0 z`UfjL!o9T(r&hf|zFf`ePZxE=vZBNIM}0Mz=QACvP)D%b)0m9=9l|I}s!=>c{oX}q zZhxmhY2h|0c@j~)6Xx2wt9>W)T8zKE;4Z;dm`C%A#HgmIbqfYwKWisuK*&^W2f_EE2>Q<8V`#%R`sY``3*)hRCjIr4v>sKWh9LT z6|#$q2e#d}Hrn3P9AsJ(@JwPwunv=3Oy#CIG#Q~$oL9^lcDMoHSQgu-#jX@{eF5oC z>pJ;Avef^(1~L)0kl3JU%H;{gy*D!KG*B9 zX;<1$Qw#y6PGkbGNw)V}@mw@1YrXdx5i>kAx>;)}3=_?uBY^-@tT9tm=v{HLGhM&B z0BKu}5}9Kxc-!~+CrgxUK7SUP=jmxl%@Ve`UV#SN)^v4=hvp+J?w+flmHReTRuhm$ z#N4!c1c4^VDf7V)`N%vDOEKn$vp84!v$0GGjA@s7uZYADDk_`sR1$glb7&e>2(#x3 zV;qyB_sc#T)ZqS)lrzQOH76kp|9G}F`>_UbaUi7P-u@9T;A1HzSO zh9;1PPdiqO;!hjEQv}p0Zo$WRy=@98#jPbawYHNm{@I_5Jnh+!frqq4?RcpYbfCOv zZA4$IV4)|_!Glg^5lO@W4WZGA4YQ}iN4b!k)GNwbX=NZt%xLCbrfKTLnLo)yvy*kH zRC>ZG5-ms;B<#KVT>ue=d{}ub%>N|_$D{&e6rNV=MmHIvSBb8)?@Mt(7Q`^I{*Gh) zO>a!?`IXzTNn~%lL#eqgN8#;`zS=9N5eb<}PPU2~A zoE6QrK>oqnG6u$Jem4TBY-wqJht20@*x&((cd{Eavn=ab*uc@fl;#ffYg+t~~6& zf+a^>f{F!F6RJz8fzKcpk=*9};TGbrgF}mgBWfSx2&P*^CcS)IEMJ7uvqf~; zc=O(%GC3|nD_bw8k+>8V`Q*KkA{{=>M_oK}a=Vjdozz?TO@neb832kOQu<)}5bs?K zb(&Xfvu4X_kSFz#03z%4m_!xE#0#Zz#<#nb32ZnFA~OIxd}zj4q@?l+i85$dX~18ai&*tS1^ajgp6*mmH~9@( z6xBQ1-guVmlrrvnSd}?6^&umrr4jKCt)YWh6YCN=1lPw_kpl3A-G~d2OmQPF$~y6H zC{x^ro}CIH#2uPIgA9cU8B|*#NNEaj3f>fkwg;sB44jc|x2Bh{8_0m`K~_g%dbVqA z@?723BMHa-h_tsH(w9E#-VCwS8wemLVV45+B(5TjVi#HBKE6@mS1 z0Eb5jLG;)C<_y-HSE$#HXTwJmU}Oxij>}cEV^13V^xy+t`++Tw^&MXsV~@=tidL2m z!z=s!Ztv9%6R~SJ_}V0bVjLIIEMv4rx?4ze<1!yJ`FxajRoQ6c#bT{1fzw?aQDH5|5*Fo9$sF$Os3gD;&mhgOd73cSvH z*!=(;pop!KC&{C3>u~PyMc^AE=}?x^nxeP2_x+e%!}Ii=l?OQ*176wrVTm2#k4&C@I|!HjT0Nw4qc0GmL31OleOeht_3R*X>Ou>O_|(UXwKviAKsA0aB^fq*@3gUA9B?#WBa@xE&*)udkh~T zrCG->>7>rYyWc+RM8N4}&IUbuVVJAX zRVm))x(xWpo-ygACn{s~LgeX;47ve3_HC|Qw{S`8J$!LZ+=OJd;mQ;(WC9cN(T}`B zS;G6s^lMI_o|G{i2;E=?7^%?2x)B?-PW#PT!3Z)}N1J zfV)x{f5&q_m97KBMs^v=5~8puJl(;~H4Xg;^B3=7YfRybI|BDEhl1dP3~e9S*K`}x zc>S%mH+pzl1O)v#6GbRXzou?-=GV*eeqqiVK(1OGw&*4j38Qj>GZ58Ht$|hp{WZ(t z&+2A)@#9z${hEGkMr8{`M(xR7wfI2Qyl@cigj(Z9`bzfusf}ADiL{~bWdNx@S+OoXlkxGG!Vq;F zyx}xESy_^cCRU}nOzwFMAUi4wH|?y<82F_7<+T!fY}AdAz;0M3ug0XLsJp8Z47?|k zh|+;WGIQvoEPvSVo1`F`82w`0an7jYL>`jz`q4{c1-YW^n)c8Bn1JU#ALU$QMh&Ow z9VK^XY0%>Bq1F+l=GVphL52bUv>5_OmXTM$9ZbenLs)}KkeESGv$@M7w4oPLbK{C z5_u{%z$0$(9@Bc?hZ=tMPNGyQYT*u*ufY^5{hNaOiA)i!c6<8S=m(`aZEplA4vJjX zN^|^M*7a$@r#1S{%MlrKuEN;*@R|RiEbnlL=sQ9~4+bk(PwuwOT0pSew;OqKHy9QE z-# zf6iQe8UZw&q@4V!$ojm&k5>LW%s;sMblnuCd!?q@02#Uer_%e|zs07DLb=siSPdk( z{*l$cA5h$9#wqGXU(_(z!-KW|Gm&7?1N=dS-^0BAgqqY7k1ZaDzjN?W17ryNTP_!6 z!AF(ZNp{Oue-$X`qe*X9h5WaS{s9y42-Or2yB%mLNRLm2c0m81eSw1V*A?;-Rh0If z15W;*pZ;!n0Epb~d-mj}KY_SfCGOu<`~ez95N&=lJK5(qb?YfA)dtj{0ZjX?DQqW4 zITSnJj2?lG4>$g;tOj?GN8=SS6y5&y^2ye}^Fjnp*?y>&Nz<~F;fHEZuqPr$2}jf# z^P+Co4v9cg*esz{-1)ro?)pVh+AP>eB81EWx6K-N9r;znr1IW>U$JJm`xk9a!~?1? zG~u2k$;{O-SM?TFUfrMS=#m*9vG#-Dap|U$w-pQC>@iqr72JFGNu!wfs*Ba^0lEHP z?fBcN5Gwv*jX5pLHEpRjbd5EwB6O+X`T6=r$@AQt)5OnE%#Y|k6P?zOtf@USj9tw< zK;Q`3Z$STjkFG>_EnYecT`O!+D1UAfzt5t0POgJ8rA+8B6MgTf*^k?ySbG1XyN~bP zb|&%hXA~U{TYH*TO=v{ukhHH|ZB*g)3=|@CGR^@Saa!JiYBJR`ti{5O3ABm28y;M^ zgSU6yom|1%A-0#_Bu^^<#l|Vm!39;!-f16T?WeRKYETA3RAn9F_rJvcj3{|H zMQzqIB^Oi$f%I)|6PFUxN3)>6x%R{Z&D-#Kc&&J59dN3z>d&~nwVC@yI!a1vOnIUy z`!A}$wD#-E52TZa4eE6&#MVbO@~#o7&>ue%re~FezSuN>Nr=bHzU8AD1)CrKkx;Rh zQ`9SKEBmQgde3@WvsHrJ-x4-0%s z(z?LjV!q<;f?sS`l+5t|3ZH+e{^LhzsP%lV_VgPoY1HI=TU<;Zor1m@03~%wFaxZu zJZJs2X?<#F2aEM6bjqv)FX7YT^~mC6T?M;_lLm)~G*FI0IbN(bpc)nMm{+{LT3_vP zfG>R=&+AcvIF4-slFPm^-0abAanVD3p9ZWp^%ko!JBq3Z&OPMUabm{rCmrfd=HJ%0 zd2Id|h+rxox`8I+IHpVM5-3-fhGt2fFRql==12DY{!`$m3V&m^q}8DOhb zUNO9=+-IHQD8TLrlSI1+v1dt1%C2jDm9oipj?WA_TAae!F^Hrz^Vn<58OzrjDPW*( z#JivsSYAXG<7V1u-oKx3qQ-J+9dp)>YdEc>Ey0&x!%|sLBTgAh5>PzMw@&X zA{0VDCAZRWeU-zzxrL{TROCCU-W5{fosWzljg{(5wTGlS{&493#1?2R0N|AL4L%8? zrnXp1ajLTt+3eaq*(&MLLy+_2Pk&7~0;%hux*A;S2xvTwfA%C)?*=(LPh2qwF|@+! zU>RaU1!`qqH22<{HiWU)PT3rrhA}&NEJwJsmSLg^tjfTGh+^7}H@cmJ_8%cyJiaz+gn-1tjjRWO#Q6$!6 zW)R5X+plEwhvDmx6~YNXvMb(0ELOrdS9*qkf|JRd$RVZqjPP?%Vg2?5Lfpkv$LLvz z6tKHWRatnh%EMrn{-_z2>1VXo$%Iw42UUDUsvVq!NaUHfGtm4hMLz)S-Vk9c^7Xn{ zqyX9k9hEKlK`Y)4@DMs$=e$^n?zaUp!U@cypFjxz@k=c?O<4QAud{;Q{%Umky;&#G zJsH(bem=lp$+nmmdsJr_Myqo-fnD#dHMef{E+QJV27ZUY;+~i1>Pp9m0@P|0>)KSt zDHNz%A%WGtyeSw&GAM@5ui)wncDO0=j(+aumpayJ0aqYP1MHh@JgNE{wRdNbcvg)^ zP6e7J#XpbsYz~00**ZWSb;sJ1uriUl&=OQx^DqnaXQGb@eE>Vz{+h9KZxnx3tbb$s zMd~|qApIGGk)%gI#E?E$CF|a%Ma7o{=%_sR~ z>uUwj23#3CnU+x3At^$ZcCx*%q-oFd8-013+w}X}^aj~WoLEL-EE`fBJp*B%rX({= zF)fHu$#)wwsvmG^UZU@>qd`r=SROTlT;VUqq0Rh`ipC+J_ke(VxYR%Lrj@#Qbk1P0 zT@ax;WgfF1&nJ}BiS}l{%sxS*5f=FARc5n)MjBY_VxrP01(bW*L^X`WuEA_7v6?I= zS@doOf#=qS5A{6uQ+2{S!}bXJ8`xWH?Qwbhl5|oA!FX295sK zN7VaWMilsBa@|V57DEe0E58X|u+aczF#QDXM?a4%=>(>eE<%_ocf2E_LEA%abe#Uw zi;Vk#VqY6EDjO7NgA>z^ZG7%4bz8-fW-$<%SpO{g-bX8NAF!tz9hS6sC_82CiA_nP zAS^h74Ap*c`HsQ+2sSuSvDC%mKIL@&mhye&A0h+=Xs8kj-Y*rZ$^?(`Z$J7c1d~ko zZl3P-GrvoTe}i(lQL6zQhOmd*a)=O4<4nGlZ6_Yn*Dv{9HMr@Pvc!KJ;jd{KJ@)j! z0jaV02bL%LNBzLI&2cL;>$@+0eq8R=R1|`ziz#(3T-WVE=&#-BFByNM(@vk!Pyfmi z6_v8^9Ff2ID{WNAPS7o#b6=e^V$z;6qU%m@V2Is`U4>f#m88v5HKFy zEaGqL(ao^Yz}r>-_Rfla!~$5iu(EQu_5+K8cnYeL;~)_%(?{vdQje5}W{b3{T5Pw$ zF$ByG1oIOHo@(!d2;7%w?u8R%YO(xwr|3)>ga2LPF9d1ni1Y!j=V>nG$;Thhz_65f zot+l0q*#d}*0My>xM@tHx1)l_F?9FACspLsbFXT`^m^KM|03XwVEhku6%SlIE8&y)x+khz! z`$(meX@VF|Fx}4=aD*mejI(BOmcdtxK#h7Op_1WW?8l#7_m8!peTQXX9KaYIjaB02 z^cI%ppjv znKd>-6^_u&N@$*I{;N5^r~W$n z!?#~Ovu^k(Fg2+U9TexIs5c0Wb5bX#_0qu`W9)7`Hk*IEA7O`U13f$)LWI|k*N^T?^Q zpMtT(%)EPh{kN|<4Z2sI|Ip6=>fqn}$eQ^E!Eb%qb&^C_IXff4m*%N;$9m4B@w%AZ zMX;!Rf`5Pyl@p!)uhPAa78Pe|5OmqGRWmO0iYA3Q&BKhlkR>oK+2c?1k+RdQ10owN z(5cFu?`D|+*zEarx%jOj%cXrWZ1HM7Dz^hwjWMAJ96hBFkhQ?&mi7%`2h6lMf8FU{ z^^>(l!%Es;k~;P?xcokw1X9eeOXwE4T)*6H)UZC5pa$>qc&uFM=uvREf;YcjYai>* zo&BWz32!c8Jy0t4UQ4&kq4A1_7qh>=v7&Cbg*fdapzkUkt{P}xch*%I?F~h;ZNFh| zXULel*{k>%=;MX+#th8Ywtm9;?+~~K#?^Rw00O_7H+KqU#u>NboXIf^NKu~QiJ0MX z{ut!wQh7T8Bds+3eBOBZT%VP|FR}Rc_@k`|=g#w&AsnBXk}A<163BLMO<^+95yOk#xaTMA+Q%=m)np-otJI2?Bln#Vw}6rDT%dn3MWo4d7p3*$UZ;|oAVs_cD?0L z(c3q^P-9ZGyu+t8Aw%;qSWIF3gMkjJw~}Hj^0-q_A=2*S+n7*vLf*yrP`;N=zgxFu zh0!*&bo8VqI@s!K0Gd&eRYl%?&!f$O3&gDOUXIdZ6VPMJmZef(}yP!#9*{-a2*@L`OW4&(= z?)E*`x=F^Ly1YL8os7%MqO~?@St)L6lo35BE&ynX&o!%IyCZewyvy{pstf0Lw7pQJ z7s;^iGv)^6LdkpPiq{TxvARRPEFx0$q6cBlUGAoO54%GILqwn4FeSu`Ru+GZE%U41 z{mF+KMe=Sy)9cOlqwHP!R|n{CstXOq9lE2638ZKRuu8oO#fv^`>~&Fh3yAfx&|#HQ zYb6v8_@{jERnod?h~F-nXp^GlMfaXA?bg3E{^TWSt73gG`d8;}!>Pw`!+T)4Nf96^ z<{JvtH%)~m<4!F722KxsEC?6wE?RX&RtGXuk}6goFrul@eXle1Dp$%{dZi?pfsP3< zZQ0u55H8=PBA@=O&KeqEaXt{Yy}~d>I;E`fHZavi8;>Cp4ZYN_8)bwc6i92jUB#_H?akU zWH3Le^4wU`l`t#rbeW#-xPB(6Nx{>`?Blhi&O>OR=J5nhl8%s7i5K>9>*3QGagTo~ zf-gmj)(-eC_*RH?B%&@%^al~_nMvf%WVsZf6yYAJ z#(V~c%e9k1`E#%%gfTCp3G8sW{SPw~r={cd?u(@#b&3f3!x9e*4ZI-*?B_uR3axlo z1WRoXu+BU1CT;NWx$UNQJ(8CHi<_>owYedGfsSmtuqaRGS*30 z!kc|(rCBo5%rxBYVTV`YS4aKUNyD~EtR%dYG~KKB#-Ze-jZI^%0ZM{zCwHcM^iel# z1=_bn2sB=HzuEE_i!eg8ot$PYjyH^5os7P`=2^vm2L*)f&mMm9w)I3+De-_kKmAt* zW%e2g-UdIi7I1IIyP|K03bq|z$)r0iw|veX@jY|ej$cc!WG!i$-4Jl}nv>FmiRsk| z2wgOSQRPT%2szKeRg^G^`;H(8qC!LptIQ+nn=iN>C1Mvw+S?j0RY%&}%)UC!P2IJtauGK0ara~5Sj z{YBB)W*=i@=L|1r`ek}=X@W@V#$3hmR6@m0QOOz%n6*0TG2dT}*YUJ8s)POgG``5S zLs>*h=_oXxHI6Tf9@shP^*m{a_lO++=VTnkNg`=IzHNpxPjGo9&2cMz&MAum7JL=>*IrTWm5 z`kd14Pkd9H>L;VA7FSsjzxifF_w(0>J~a%{rC1#K-YBPT_>P2iQuw%9GqzN!Oy5s> zyhe^Yp=J}*R@U)QyqHz9*ivtL+-Xdg#*?R;u?E|bZ?!LJkx+BYY8>RXuEUlNxbKa( z3(EFFg?Ytj5nDH5OM9C@Y*NfFq4Qvu8=on$48+d-sl>Eq(qh{%hC?f|qph zQxSN2l?^=E(FfM*=wO2gFDoO3yef(=P2!3kbcfEDTTf0s)hQQjeOT06dY5(BX18RE z?Xu)~GB4QXu~^kOQ=h&KkvNKzP!jPcd^4}8XPd=zZE2vaZE0{-*(OZW1ZxCuW$lqp zA?N$55z+(I*WTQGAIXxfO27is`j?)c8VThS8&BaJndnm-Co{;A?MZ@>?L3!nQ{S(Z z$~xy$tfLx)hwcTuXJ5yp0YF{Lg!WFgKdCeON?qiyUepcgBi|O`x_af!3cpu*Dg~AB zT)b~r)47-b$`lppK9uclliZiz^kq`sLxT zN3YDqs_?OxSt4_9(aTBv9KU}01uWe@JQ_fu>r9H3xl5p5ZcuBft|=!KK#Lw$738Fr&`A}T5?YcvUKEn*Xh8&oFGzzt6Qn^3Irl)O zo$ljV5<apxX6qDhk!=yho(&ttt{HM{e&l2}C!eQS zwViyt*!tK~ty1X6u4VVpKroGWg|hdQp|j;gSz4z{<&_ZLAJ|<^nMd4*bx~0;VfyzL zBfH61KAATWg1Fn?!;o(fI(7-arGvWd3dM~54$J9@9z#J1OZCAtcs!P4WnaTc>sX#Z z3L~yIxvi!O1Ex1!+XHgyElZW9ueTvXB+o7LgJfgN#-g%&jdmy|!#7l0X5ZyZJ>h1U zg}q*5{VYpJrzTTIQttS;mSmS$Fw;FOpBr)#z#bu~lATRZtJMG!HwK2##Vdx>&pGSX)2$ipWr^-}gJA;A#DP}i-TRu37O9ND`r%RZ3 zL0zxieqdJQsI4_!F-)z|Xz5morF{R+tN3kfgzlPrhp%^i81^PKA_1b zHrodyC)EZ07FL%7*5(swULW+pN*=(S(r?A&Y=qi`CJ5c($>=(gPcmE~K&tLf;fiKO zYgkl8X3L*7GB0)=NKLrzp_{w5QeDjinWp5d0#J#+E4nPi8YwjUpzesPU<^{UqhDsc zq)308A&|Ra@d0>O0!>MRE)3Q3n45qiFfuW>Uq9aB=VN;3IN_&EP9mxH2-Oq%z3~o? z(4L!kPzLjvSj#$Me!R{m5>O!<8gqfAL{y(fdh9e!2cOM1hY0vQoO)E%2ogyu%Z zXzRa(AXH=Qvy4N$CuZK4l<7sIQ*sooHNI(x3PZ=`ozNd2X-vSOxFC^C)XYB9kS(w{ z^+x5sw)Fs+;vr-k(VF#{X|hqmxO1Une7>!0M~-0F89wQrm&X$Y?V zYbD3P6&w<=^+W-ge1H7>*g-nAhnW%$Y5oK{vo)(PhF@->+SQcp6^v;gUJzpJowg)Y zi$&pU4w6nZL)*ke?EE-9?Wu^*PdNvdF|IKWMS~32mT!nG>-AOvr%W0XZkH)kUlMgU z>w^np2`~+C*;Pn*BdRGw*kT~D*{TCCbT0@GRa2g*ts)=_$KPI%rCcX?e`>wgGRqYoeiR zM4b(7(!grbrHAp)2o{9xMGh)d|5)tJUs3kvEK}bbbi%M(V|oCH?0X?M8<&@(k_kCb zi=Wa~Z#GZrJ`ogqiga2t@Bv${+vqm*^umc@JXhdjRyI?# zunTl&EL-mN0H+F5CrT~=`G(NnM|d+%jtEyCg->$5xX?t!Q#MLZ<1 z#p#i%M#WpjWI?k~HUmiSM%Z~wOpFn7w7`7iZC~Z(G=wdd!kCW9m^Js#Dxb&rw(n<| z_Q zeT~c`_4qFrknS>O)eN#Gi{hj|6IZ?uTMUE7ykA@>R}Rin3SRDMHRu~(|4VSwM{lD) zZyLQhrEfW7-uL`7ac@adqslgd#0S^g`uLaCll8b$8Td<8U2 z7pB+V+bS-Rw%;Z2rfgj1jBKW>D#VmUOjNy-Y+1*PvCb@33FZ2q9f^ssltyPo2{2_( zUU86U4Lo>cZJy=@!S%T0@wkYQ*=Y&ma1@sIJj$$R!aCXifU$j8A}uI{RhyviYEMdag+Sb##0V^iuHD#z^_viA=+WjtO5 z`-O79=dl{&B2>~!N*XVBwqoE6N;9Q|=Bvy;tvl_4||K`SzaKtRLe5j zD=62MIKenkOB|w}aU|;|)l+(L26!NS5DB1$KQljXz4g#UI*PYLyi-GZLX3tUKjy>7 z-{dyxJw$;vk5Iq8k-$o$s1#+Y4RraWHm@(&V<2n#W7gZFUsu5brLvK~3vZIN?=UGr zt}Y%H-)|w<*;r!mS?W73_Z!d9)%4>S1;%zi+-V`vR}+lE{86Bs&CAl4+fn|CL9UWT zJs-soKTqcLs)P3zMNh5vrtPqF?GNx1eN%X#nK48ymxu>~A@d6}W6sT4(9p#u_C|>dUUBl%l*)xp;&;i}i+t0XoI|iIXj}o&>fsR06;=6UA$d@3|1p z0DWLJ-^I_a|D4A8P3}<(wcBbLvemt#Bkjt_Y#rUB!*rVgxVL6}1Yf>`b<#{tQ4f43t!POJ5<|8O*(magLNhTh82SilWnB+I1Mm)Nqn8N%A7&5a*Gc=`ZLURog zo|bDGxPSYu))7HvwhSh-Ig$~+-lx)e296XGwy3!wDr{@KJ_xkSgs|KoQ&8QfRXaB2ylo8|X6D;K)hKKg zI;aI~Aw0Ijy|U09)uv2}WU?dlk&8F>g&#w_knuOLn7f0PKO39?zqgqb|gL{ z8WfG?O-S2WJxzn1wOe*-C`&&<|4`Z20nKM;PN;@j)R9In%y_45egt9V;*6%T3e`_= zRgCbdq^AN@MCN?4-aUeR(I8q<`To&QS%WrYJKx$74rG}|&lZ}9!z*SWM&-ULs?Zj# z8jNu*aQwSf%9ZX_HHeLZXVrtWKjkjYYz!V)*lO3)3TmgSrZhav0&|KIvZ%6DtL_$2 zt~Q$#@n0f;316P-LBbqlsP%I(=dM={#V1K85>4r_+9&27spx8Ma-el9(H5^I{H}D= zw5cUSs0!Wtrg$xve8p|<6si|9+@z%spCkg{DucozuCv7_%--S@M2HU6BWPa#c-Z~HEpo+;{&Fs(%!GdUw zp&K&hre2cnbrgA46oo2PP$Yn^zG~LPzbU4!7yom<{*QTy_C&9vaV%xiDH~R_{+T}p zdXKPGLS+Y5-Q{Kw<%UM9Qw2hQ(fk(mW<4Lq#N)<| ztM=R>P$-JG;Vh`Ja?dNkacwcbYwBwz_71cCqrKUDWqMRMNsNX4s@G=BYYm)V0aQHa zCtOVR83_+4&SRBdN%uH3NH%S|EeiB=UXJDDpuSsmE^>Qyh+}QJN0JYIAhnNEabC3yxUJdJ+Z+E_> z1ev*#x`NHpS?OmM6l^}zxNmZM>9>-Bo}gI#?69OxM;~LYKVM6H2gerAWXJ&j@{G)K zweA~s@pfWD!<|OH`-*N$Cv?-!O7+}+*JZHR*}VC*_h?Yk(Ho=(Li9fezhlZ2Whd%l z@clh!)okuyj8Dvzku*>VGWmnLFWn{=?Kh#pejGSdVf@~{uboevv=4Z%lnm;r zs^_-vq%GpaW4e>_%o@`UuuF}^2PB*wZE6Z z6W7a^dYT=~o6fMkwX>eeu8ydUmQ$9BPaDr2PhCMWNMpE98XpgohcS9fv6)P2-=WZ_ zh`*go*2>%JsRcKJG`7G;8PX&Kh!_^+9<*o|j zS{kI-UW|o@iYIsX2ThEFyLyw${>bno#KWK7S zeu%+l)mw8CwgTslzW;jQvJ5|3oo%JJ3KS3BFz7#?BR%S8JAHMHoIp9ruQNyaq|?%K z6UN8uZaVc&T2c4s$=|~KKX9xK z?D8@6|~>b$Iu7EbT$ncI`hN6@07a4Ra#sL`r+bnId;Ak1Aau*zzPI&URqGe+ar zj0DGA9SXjuK@gEGYAT|U%!n6W77)22NRF}$NHz^h`7j$rd5&kJ*8mPx59YaJXm9m0brm!4$%cNc~_IP4IS$XtaU}I>gUzouM_j(cs?M`~5r1dNe8`q z_YPN`reJv>#iYWdFMNvl=2r7|w?+6d$!YlsJ}lzcj<6!nt;w-My_@Fh0A{L7 z&1&hlc+Z+8GT#11>x)*Co%&adr#4}H26kbC%+UfnJ~UhVVq6j{omNw&O{@a$RL9w= zG|KvSD8|20hY@uVm1TBff53LX<*@2;zjBG-R5x9*WCM>nLr*j#5#q}{5p;m`#=K2& zoX@}UPN|rBzV@M)-HG{xw?n>=P*}&!w4ap$wi(Ea@9XhRX32p@|3CB5SGV z2r;~!f*p@7go@oF&+u{HmyO}_@-2ik$(dcxEaIS~IXLb(xGh{%Y0GR>1Uq&#%+4aW zFl+t$Ml(PvMvBSF$0hR?P5spME{xw#w{6k}7& z+;%I*wNE%&Gq-9!`EtNtoih|s_MxeaM%MmroaX8W;R6EKQy`R}*TW5pc91sz6OgAn z0ePMu5hn{Vm*d$BWIC%Jp}?8p>I?^Ui^p7aR8IS>oz`X`1T23Q$53zQ5hz%RF?F$D z!Ev4)QG^Vl8^@Tg^P7&*R4>=}D$!Bo0FYGz{F}M?mOQuDM(j@s)>EHH1gw}h-^TuiNZtoEt1w=*Cz4mF*a;kS>X{a|tWduQ|6SsPM?l+(_~V|Ss> z!eb@?8G!AC!HCClcMPT>x%vA1-G@GnhVROc`jYrMCASR?4T%X=lzgK;ov!*|Pald| zo$alpPYlyaTN^Mp?tXw!9pqI2h5$)W_Gp2sm6h9)k=v2a(RJu{N@Bv5dL;>J*c1F4 z%S8x@>P$qitN(>GR1fUu+KISln=peHM*^K@s-88!c5*$(W$TnI*Sm1dX0|JQ^Ep_d zi~SU31pyhFSGS}S)N=Y$x3^9?bWM zY%P;sG9%X+qxBm5c~DFb_|Dn9I-+7Ry+4uLPfpcZTEZILX$iYN4|Ik*DC>-3tr_1r zT8G1ujhWO;Vb#YwEx*{!H62s}R2Yli`rKw(dvO#Kp{d`){rAt#wM+pDmd`xqYu#}i z46%RFx%h>zGtJLWN&DF@6*^`J_ijk9a~1DY#osM=SP&2J#r)aK0Y@}nPU$l=pY%U$ zx(xrZA@T=4as`Va__4GLa5aFm7Jf!~NcXd7$)lO!SxJ5ulU37&Mi7Ak>i0Ukso_r3 zUU}4~c?llpos6BB&CWAs3)3JIh4?Ofm=i{Fm{(=p)DTDh3*3AY@gN zS7>)I4LB6{BaG9SVdt65trWJb*+om3GOLu+B7OME{x{yKL7ih&Fd_XhQ#=1%8pyqDRhCr{s``ic;DCgcf00Sjy|+BTO5Z%mk#F(CD~-U? z?Vvq}u_iT2^zH72wqFvDveq33zFuv&cad`eHsiyR_Zc;+EB=W9jFr;DNO`-J({P>i zIwWaP6vh^%&6hLA4aJ_-T25t;xAG^8Rz~bPr+ea`5nx7(Dm_05&Q`B71EsR$M8Frx z3i;Fr5F#f+IZeg6cFOTD0gsg?Is%Rv!(bmJGxS2_Sx4(+Nb3kR07MyP_BUA(Chbb@ zV7`22!^Lqcwzv_2E1qSj1TD#eU18)vD>vDb*GZf74Y~WSp1U}jeze2qtzq1=w0vBd#gkR`$@c&yTh*EX^;OFY74*1e~3F)f&Ro?4-!DOk(=)e?G5G*EV&kz9JUu473M*oUi083EeJqLw5I3Q3 zh!SlG2B6s&@18_5`+>SXoOE>OE5cE2PxA#ABCR<*84Ht5r^`IYrGx4Kuf`~N*iw(G z^_G&mn1Q&H!0sovJ$De3gtA6}lN4f5Fn#MdYjSQTLNNp_`J?U-?}-~s8Sd{*_|6J< z6+d?rWyRAHVrsbC)ap$h))p#o1VBDmRA8P~4niTR?I0s3-DWy3I*S4UPp&Sdm43*} zM<@fqSEXV+2;Q&f@!}?tJ$IzoW`4(2)X|&DQo~n6n7L@(^HuE!>y5`TdDNi-%j& z>-h4_+CfW})T$sGqH_W3NHVHqL1k!e&|J}9iB+Utw-sulX_xAjqjL6;ampH2JFYIt ziy}!aC#JB_YT%1Cb(>!%-O6Rs+N`9BU@4xEJ?Zqf*Qm5`IVRU`Cw^j7$K<)bHvpsRCa6wNXRfLb9yFSd#9pEw9D)>OD@7%yMt8-$?!_UGMUb0_-3-2C%9 zGxzf7ofOk^G;03cIHVdks)>}dY0$2FEt>_|_WeTe*TNg(=5j)BQ(Q@90YD2|PenqY z^ZW}uiem7nBnLgZm)Y{hFxhCaK>oukw5|;*9DUlT_TxI8jf%QqSo+j?(}N zpC_yWbhUP4v&`+z4y+@Uw%&ptCR^**^wGV_xJ*Nu5ne4 z8;HMu;ko74V{MGHYG@?-SQyA;j44cqd|aiMw18#hvIukKV0R_zDksXd4Tfv=Fa@AW z7*zDv>DzOOMSyqthR;zpouNqVdpP1&NW9uKPt{ftQJRCt@h-=>9XSD>&~Xc$!Ei^E zx$mB8v9>`Q2x1g94<49mPk^SRtzaa^ z{!sa?*j^?-_}K}Le8pWD&J7)pcIIcN>&57pI++h6zm`_d*ho>~J-<7h>q$K!YTKns z(wEFTOB?n4IUD?2+m6-}u2ep5y8AZb)CE;C>(vwgx0XLC>3O#`FMwrCcpdf=95*ho zC-d3OwNU0h>E&{>WX7QEzPdt|duZ+aPS9gKewb%NKLyfi{AWk{*$<~f0ECIR6fWF= zPZKNUl23k)k-2oKW1DR@x|d_ZFN-^uZ5G#E9BsBQu|sV4mBQF_W`OT7W9X{Pgv=w37%_fOUgX0cI57#;%RS^1s>{k}9z+QAO_1k4K z(c_EDdBDg>>s;qC&@<15+liD3Q8@`e@*gK7~X zqWQQR0+jIdOB&f)jQdhewosJ_FHQHPA1Cm@ag{u!E@jwgzQQyXf3DAOR)$zu{qJuQ zsapWVGK={lTM=hIMvMS?9%~N23rcTs;ZPg(Eo84GweoA}d(9!CT#RQF(NZ687Ziff zto@AU@qGT<#^j-a0hFrI+e3lSdi^DOPCARG6H5!nkcqrbiiLi2zh#v?{bXJ%0BW(D zPkk?=3aUQRcH_a?1qnS0E>0hWYo5!2CmfPN9%pu;?tm4FkdH~>4U*+e{!VKb6gk#miC;jLQ|*tb5z?< zV`*F64Eb9( zi5U3aTMjBJ>CT$$OR>2eU1goaENR8wT2G;Vobb1X@mf++aTxkoFSlKh2a013levQ` z)6bfJA;7Be%~S7A6)8N^ws{`N^Q2_D+eSI3yzlZkZnkkXbyN{O`lBFVki@wAC#QjT ziu|kj!#K#zE+EkwMHh%}Sk<3AY<03Ooa$FCQa2(;k^j`EjBhyIyXW7QkgjEEkZJPL z6OOB+NhDh8VYE89iV~}Wi?2Mn zGROsNIL?z4w*(cr+!wlqe^Cmbn$y*2*jF7HJ3804L{WJ}#cN?__t z5_IZ`6B&F8HLH?aWrgvgh=T~dh$AEKtq)efqd1t)O}{4E2#^rkSJIRoZFPB#+!q7< za8OIh07?q?q+dk=+B5$5@tV)EG}q??t<29U?Z)1#N0gl&Kvbv9pd(g|_^R%RpLBQ7 zhsht8!oxH|;8$*xZZLtMxVS&=Y~@?F=@LQl@-Df~64_NCU zwti+<_73UGtQ_*WkHB%WTJS#zn9BN|N2aOpI{%NGAk}6M1vNz8+;QYcDOZJof{v9(i^2v`1Wn z?n^`~Z9=ym5~^rlwXdXSb4(5?L{+*^946q$)MQcLgn!%VNvSzAQSskYbK{e`f?w@3XGaHEc!-?5k9PWi1>_%ng%QJ@!QpXYT?QqY(>ykfZPUtYkqKY48ghM@x3(ND5s8g=v4 z(Z(5jqLkYhyN#ymIkr=sd4B=V1oL%DpRF}gEj?Dw4CN1%(T-zK& ztE6n0^NLo=uO55OCpZ892%{xM#gYY@XilO`uN-H6G8N4rz-S;kB6t&3 zNn$;FQ9K`HPTvqnj@o}fs`(i&hy+ZlFFwW#3tajy>ecT=5=!O0+vC3-37m`vQcXn; zNV&S!DTMh=j+?jpRlRZ}xdvbm)ImAAN#gQc!y90+k4zi~7x#6eR)JE7E0XrkO0-Z{}N=CY%Qb#dm?^_^x5Epl=uK+Zxk8C4_e>P_@ z)MI0QP;u6mDnn2#qXu1et2{yZ@9K8#Gv^O|YNfAs_UV}h(GFKqwRZh^fqv+%P5Ntj z`@m=3;*A64&)rnY_XY$^7;KD ztJf#@2b^TW*i>*NLY=SZ*pQ@rc%^_Q_K^b=GikPMuY=c%l+i`nE?I1JgiWY|ekkX- zx}f4}f%&-@3@r-d=^SBFNG0^r;Gk>(Wb zN|}O!4PEkzG-+nEu)oo|gfeVRcv}7wra8ZAC6dz;V;0|wj^OQO=52Pf}B@Jp^prsSF9k2gKNl5Y$+Gs&+)u9Z-#CMFzrsx-(?6>GzPL0686s=E_ z%kPTD37#ag7rG*SEQUJM#Ye4rQ{TOhrTkN%rl{YNZ1@IJ=XnMSAN~|XM+TzHu2$w0 zX*CaZoHsndZB^}kCi~u@2i!A(HFv)^5UIKkEmxr)NL0*kfpB_Zp=7owEw=xPy# z(qkqgsXdu@+akf-vHCI~|9R&~3UBQI)rymzF;wR)v-I*-*omIJ8yc=IDa}S4xV22i zAYuMp7`nFybMtr4bavG8#gP`=>NU_}ZxAXEi@L7;d$RMIy}}SGj4z*g+Xr+JDEl#< zzQi4u&3NNCT{w$TRp>s^yw&Aoj6&k8j7KXcz4C;2W)E;t2}Xy2I!eRT^6NT7E(`^| zK%nY5-e^W>?$E=o@xAAdNu(>95am?%2z`$Wiea!{iBB}Mf9yxs$F11+ey zt`JFZS!cFREl-U;&OdgFFz*{aENxHsT#nW;C~2O6>EraUgCQPt)EQiqVM!Z)Gmjjh zkw;@wX(>RbG{5`LG4}NC4efJ}#T#xwEP~f{v>KrAPoLfP;dNs#b(CRvBvJPFjJiJG zo1u2_q7M!EdO$agjAQg}WTD=HEccAEJ2}_xjq7c9gM{qGu28|ulHfw()2VqX*)bAA ztwP>0ojujEm(ZdHwUzth=7#?~7ss0zkN9_wYf2N#53UYt;#yEh8c2S_W@(h{L}l+yKgP>7zpPG7OXr1B;7ZPK9EeHC}^?)?aG`*qxL+Ji5JBq4&o@CWZh zLjqj^40`su#I_9a$!MSpWDrl;)``XmML(Lto(x4hhAu0@y`@H2I-D^ z+3WwhksAD-Cc`(OHRBJu4_RVE%2Jmejx)ze*E2U12SfE46T)ixmx^bEmI(6UuocT> zj9yx@5A^;7;euvritSaKKcWtgbi5`kmwM`@5@e(+;9gc0M(JTMCR5*B4^s%tT5L^U z@~7+R1rvAM8>50V5zfZ@TJirn{@(G{Y;A8(b_5sQYUAjY6 z4uth9=eNf`8yRt?<_KtRhf7ozm^mF;>S}2y+aUiDBm(8fRpINFcldq>j`C!sq)ai@ zZ!L$PeRMA*-`{TzbIch8Eo(sYp>bMT<2)%yiS^&Yix2o}W!nSj>E11WI%Ykk_x4FU zaZ;$y2i8Rp4?4myEcLr&g|h$Crht~%zaN-@H(s*dVZ>`uw->YxSu#pc|Cc|XdH=6* zbfZBjY6RDf!U6@m@Q&R!;#aYsMt~{7fXuXStum(4KbMPUl%kvMG`i{4e+N=r8IkCQ zOlW(rbk7MnJIqak!Kno5eZhf`7Rm1W@8bnu;SL)9w<91En*@%>&+a#0G(eYpx_tw% zxnBjmzl!S^Ceh0NBS0Jb*z1M{#eUI{K1kJ9b`Zs z9a>E_v>7_@LQxvoUqyj=173fDsBxjte=qsJe|Z)6{x*@f8r1r1%y;QOwYiLuyQ`{? zq#rcih93Cx#zEk7(6!4ls9d5JAAIdFFap})={JSYI=}w-QG-YRO;F7BLgiZZO5Psr zG_K=XBLpb^fva1!M#%m=_)Wlj=krz7AOC~mm-oCVeqACewF-x z(F38U2BH&GK$D*MZ0@u2zi&bf1km;w!lc~H*~?n+A{+01qP%hO$^di5Ugj-28v6Y6 zxvF;uW*RJ;Mk)3^R})S|`{fiR>nq_OrCo!>Ol4rdd3HBEh8a00&sDX{!wOVyHI?)H zG0#u$dpCG~E@SaD{L?)QlpBEu1vw3XBTzI(aEi8}Jf5kFBtk$xnagBlfa)1RajI9#VmsL zc5ZKLg9+hpSI?rfN5!T&p^Z%+H_Gt2yq`9r5w&BlK__K=%65n5+;h_4?=PT15f${P$qYjQ@=U}O_ zK*5(dx^B#~5zc{PtkBhh)VN^u@!zM1DWS>0Nq}bE#zli5_HHb(H;`f5($0GmB%^sV zNH7a`@Q7w;J8@nHj+X4vfRo#Jw7rKHJP++EFm^p??0C0}=YY2XZL5ue{e7<1Y-=Fi za2IxdMB^z>|DU|n^>1A7Mi?E0P{B+Zf9(GgK!6_yGD3^W;-vc;u5M$4{`GP;btFWF zjR^`b;$GT?{Sin6LzWkYhD?jD-g^B$a~N$6ok9cWyvqirPzZ7ixk2YMq+5C-~u0_Xk9V(L;mCn&EC|L>r<4x3uT# z4uQ5uEb)y85RPUIz5a)a=+Dp(VWiO2<Ko4c5ug&zc$1F?WKSJ(j}Mi z6{t%Ve9bld@2w(Dub01LCqW5xL(vGqbs#3NpV$?17lschK(1MuG5xo}d2zs(xcy#) zh&qx7{e>d#UFb{orv&_dNj)%MpLhF^*OZp)W%d6$V)8`6_>}5U$pz4Z0HplBU-3P( z@NQX_f4Lq{Ep+!~G;uRrNnA9Hl-=&{-Pc@7+bv$6iH)Bk7Q?+sHg+hLVlDH8DVKz6 zv7gRt2?<*Morv0ngpFLTczj$Is_XB25jqNA7oX)#Tt7??I!W(|NF%5L-x}{ZkT_kd z^wM6~?wDlFhX444;b=a~@3=_ok(Iu0n@w=Jfbf=^j3#8z>^x*tABQ+S_;n6;$o3Q6 zZBD+!8VA$S;l#6d5|8yvbgsO_KNN9!j;f+Z2MPM|b*n9ZX^wgKD*5_LgP#58BUp&> z-Kz=Rxqse;L8q0AWt>!Xp*<*v`ra_3wi--k5*W8ndZPvfQ%i93LqMB3s4Sak{zPJ@QNu8JCcSTspI zxC#s^r7{MXD8++*b4b`_2;_1+9Vg(`qF6d*A%dECKes)ML^k@EU#(ZX)I}MqYeN5 zY|PbJdX;Im;5ff?k`<9>G zb^|tD1wm}a*6#?r<0>AfV%bV%omQFire3WM=6_~4AN)%DLoz?p%V|*+VZZds+R@Eb7DLXQYW=K$h#w^Tb4ui}l$q{A#Z{_A^iJ-v z7^M`n4`LR@%^pE}S{@xbOsEAFiGaEKtBWbE#JRt_9kx9JzFnR|1Bs(yM+EjSJf~d# z=le!SX@4V56dISRX@(E?H>_%cORyLBd*)v^es6+V zz3;&D=Awf9rWn#p7RXb{Ah`B{Zq)g|rbkOkxpW**cqzB1*$&Oitt7*j<4~DRLRIro zIvrWFS%AGAbIgjC3XKl2enaJBlEa^W)qDJmDc)*~(wRt!xI~#L7c8F=9Qt6=&vZF- zup=kFHM)}StP6vN3iIhigZMX*t1d|mUhnw2I0cMt46^G&oC4+q(HP#QVi`} z@7yaVwi+v}@5#W)^yTlFH}xK?)R3_3|Ni;(^^uyYQMz;8KrrpCG0k#5^Z!wojguUR z0~3tKBE{^P7rEv49h@&VT}+cWcyCstbV*NtvTNY)`85CJmMW-CcQIxMZ6X*+3P*S))i?fCA7to)K{mzwLRJY;haf~K5Y;t7+hlG2 zRrBjmekb^F@pBwZKZ89|*gT`1zkcGY^!_w5-Q%=EfFJpOJ7inh0x1|{W-PbfeB;zj z4)dasZqn<^KV#+^2dOQh6$p2 z21=W&FA3SbY@vE?H7EovJo{{V9q%pOHEp4mS3;}zy^I$r-xz4_AsF>L{SzX1{|; zYBPOmm=e7VG%@7`(WIhx>3>=n{b!ElUD4?sSQzx)O=QJvDW84Cqo|hu{dXg8RunX@ zj75ytU9UUcoSV3_R}XeyzPgl!?FM3Mp<{|eMK6Hr4<9-(B&IBgNqL`hi6M~@H4 z{guadB|fgH|DS2&VVptHKfbvmfSfQ&QjHZv+k@H7H;*2W29MX9P6HmId}X-QRLJ5q zFiq6`yO5DGGD&@RPQ7&08Zhzxn@Di1Y3QovtLrla1R%T9BTI*4NIUq!&fCKgUPVT` zAMx%6?Z#gj1SUo!BA$n9uAuVX-)4JwVJyO*m{!1U<9YHUS`#%2w3=RIff=92S36kX zO1bJsA=auL4u5++t|%hqT;_dd(J^aN;NiLa4wOl_^AZl~UNqEl80=U;uq2MN`IxgOdp8< zYCqbO02<#vXE(Ut*erH-=9D~{f2o!vDqwglK>>``pmB*0Hmm;K(@z4IL${1PT>TGf zj)zm1ABV;?9ZLiT)w+-a;*boPCL7kp1-G5p`P86WW&{$S?Q$=`Iy~I&e1V3GbFv0& z{Z8Wn0^*%V^=3^+wC?%2?Ua!+HN00eAItEDjC<-GfxrM8+~u!)Fwht_%vr;Ig$z}_ zMy!R_BklqJJ?qIdA-I#{t|qzdmk_5_a?2^_|962 zw8cxvZv)E~C+TZldd}p`7FMajjOPITSbWHbPRF-eFp_eCLMZ{?bfzuL}Pld+~|&$8)W+k5P}4Y2uGjMh2uj*Hegv&W9FkCpipymnm) z>f8*{E1A?I=T$Gkm$Po+?jqLDaD6xKb93`c=VNTTsl*>nCHEiDxqUD=e`yjv?I0## z5WXr!9Hm>Azfx=Ya{s}`SQ#*vgBD6<4wGJ9DQ8_0Dh2B7IXzo91*|*{T@tArOP5wZ zi}b{c55ocVPdrXD#rUkQOgC62E|`$9x|Z+aloWEpm}J8>k9ap;^Hz$u<*e}kq={yE zyztz!|IkG`emtXon+4BsH-j9j5O=c1`0oy1PKn|ny6D$6Cs!7eH;+)gIkzVp5_HPj zcP^3f5$D5WzLz`5(qQF5P4H~ah1uF!o=fzwl1{f3_nuTnr}C((k(X)ARd*S9-7zOTyL892x31pcnv z`NDLs%a*PZRO`_>-{D|UIZ72X6c}$X4KOxY!CD8#%inn zgYoPiLWeWoOEd`|itAvJG>AM8qlZPSL(Ia8Ij+3~67e^8+?LzOK+)j0+B0sv*&BT+ z^+!I+yq{7E5-|_WhiVjdpCf;(XZjLj6#BRwAbwoBgf#-ozUgZ@?O003t3PG`Acgra7GO*2G`b{iySZ}XXrfTw)}kb-dAxyB zrjY%?LbISskEFnU+h3*HB#D~onl1C4JOgoM5=cK>rpN2trXXG#R z+^==#l^5{y&^V5F{bpxIb3FA6Ek2q?53MDIZSHpuE0YwPg*1JC%jf9+{@dZg4$;Kq z0V({byk7m2qlInEgSTFj6A~}%Se0Z^)4306o;d&+9$D2AW7-5!8VpE>C&z8~S204u zw6Ir3<3YU$YOLyFUG)KNL%iRLqQL;BrTOxF+}cS;x`xFtWSc-o`wX;#-#4*Q<%6^D zPAEQ|lOL3>q&daC?M?8-?aVOryJUJj`p_fliOEM23&)iZk*gR1zWt-AX?mqn<3?ud zx(}Wfn)PGqI-Z|HGBsCgO{WJG;r*ov@aB6=j&<~XhjvmgqIDn0a!ss-J1?;t6gCTc z>2wyaaq|^*H=Rw+NAR-Z*A^uyr7RJ72^qwt-WBx|jOLXZ_f^%~z84h1EP_Fo z|1Ecn8bf4BMbDjJEPfG+*PwPJ$EA%CpJWCH(m;*KD|-5HoXs_&N-EiTvGHOei+gE` z?c#WOVhbI`ze0lZblvp&rGyp8#VDdpl)6|sSSgj#4~3rE#Kd5GVA4l*^xwS*$ji8p z{!wi4H0b*=`R$h$XS41p?9KCL=EG;g=GN7081=uSH4|2*XdL zU1bLqCKJi>wuQqb(iWFqv*k(5SY>$tloFV>%L!||Oy|fS5IgLnC7Mh$FGlP+O&cpd z_82$Da_Lt#VLS<494B62rX-P07*F50y1UeYsm(pM*js@y(N(UXgOYQa&V+}D`}2!Z zR$)}a)u(Tm$`?d$m7NtM)&`m_($@*Y+Hf9_sTTGu+TWQdxlCi46LlaEeok-7VlZTd z;5Pt@%q0cVntWGt%*E9AJqDSo?GS=^_*-o?h7NUbtagM`oNaEQ!$PB$ZOHLh(d;7q zH%m{)LPnjGxyg&pNqJPA{P0#&=0fTNWgS?TI>%H`E)R_(%;dbO`A=FD6Wz>_e_V#G zXqz7uBbLHBk#@#jZR@txs73J#$LPFn1Q=*~PRn^bStSW7ZiY`|11aVy$fAnYH1bV6 zRRMv^T(=0K(~R_~eumK!t2R+yR!g;-uL%D%qHZIimrCZSen$qKZ|$}3rqxvs4n|M- zwvGm`BELi=^fg}?uSq{dRVQezTjZX0-|q`B11?X+N2WLb3~q?3RN8*3Eg)fXv;ftj zlTb_l439+VH1aQMy%yd=djInC7!m9XKdIXaBx2E_hJ>EUkDWfqr1J5;)SJv3(SM1+ zIo+~@1>r>hJ&R3Y413>_Dooe*=JOk|->7xTebHCJpg}aQFg5+i=y-pZTIkwJVA`uM zzhz&3>q`%&x39%mP!totm2%|6_#1}d7Y?J+U>RZq%eAeoVFFRhPt!_VZL6MKus;9z zCvSCa6qiA9LqdolWs}bbs}4b8{AKL>E;CJ@doCeL6NHIbr^Mg36{Z!doImwqAAG-MRU_{LE^Wl8?nd7MY!*SJ`ZG^$R@tg9z?^GskUxH-v=|p@>)1IA5&&1)GPDTlKdcHskSp8-ej?W`4V% zyd>uP7_^;_tJ1fdT069lD$RuOMPzmB+=Ple_=Pm;ZQ8h4Mj(NimCPFq&)W>@B1`Dh<$}Gp%@dwLG8EQXpFu{3779`comWw0{92W&Thg=}q81KNR4=!wnQc z!<0hLlGPop*-4e7Pm_GQRVTrH#=92mtWYwe zD21oJUC0lamU*@Jx;}yLaI%Mf)C#9~ms)!sF(^P*+s}9z8FUGdZjc$*%37pjnv%cx zovvZ|oIW7;b-qqL!tKsO*BL!OEFn34zs24ecZS{z0D2%WOXtQdr7$`@aZ4sq0 zYb@7fq#oHvozEp7y@|4ahHT0X32?HE0@~=E(i?fvj}E{7Marj_rT61(S!1hD;tD=K zD8Q^IO=G}Hv`NfR1CMd?CLgMe0WLP*Q~tFDd=U%gLQwpvQuT(`q~o~ZDEI^i3F4Uh zSH-Nm-x_*ipBXrqkJ+n-D>JBKTG92vEgP7(lJ8Jp4V+|d6tg1&ZRKp6D${%k?M|#m zsyj4;?ojC=Y`4^DSUZPq;T-So4d0QyOcQ-Xij{`JB>9G#d!Sa97-Kr2JX7>xS6O%F z)v*dTZRhX?d+8UGR0Bq)E%UA7<{_-Q70->+V?o|?zl5#bl>l{Q=cU1&8ISa;`t7U2 zc3O{?t}=BktQl5rgr1Ml`Dwug{{m40nZ{}R3NNhl$&_7=ht~x=6F0~9@>NAS;jm+i zc)9ebHm_ckF#4O|zX)ZS-MOa2H--l%IScfP9&N#oj8sK$%nu?QxHc&(63#7=0^F0z zGJ4X&1s!QuZac<3^3e>d0frB^%F|Pi#6jl_pU|BnKBBj$nTovB;C=Kplz`5=+^eMW*e{h zYDh6qKN*{R7V~ks`hux_MAF|zm>pSe)r7cBmqwO`?U5%=qY$rpzMlX9Qh+*x5_C!7 z6Qa_X(hNKday70u2SHUqqIFsY1;=IDU^scx z2Lo0P{?(qBksnty=?XTrKL_tI-MGwBV2k~k@3px%x2T4aN3#f6RC_A49Ai*fb)R-H z`J)SBly{knhT9+e@{Q2ac>KOSDmsn?msG06WrDd%N8qF+kmJo0;v0Sp0Mq#DD3t&4 zabumv@#kN#c%Zi?G`Bt|KYWi*ioqV%vFh_cnAA~7NP<+DqL_Qx*lvI}RirC#tBW0> zIx}A2Q!w?*gE)mT`gmb{WpV7`2=2ha3c`s$kchWqA1Y*4_s6pxw+qX-*YVj70th5& z(b#z@EIJ!5(#ZFIS<^_hPq|PK@%Yj|wTFEitQf^5_d0NIYCWBL;7h~T`2-N_G zG60fj-~&hxqW!jCL$IJd1}vKs?=3&K_=gZt)gKl8tav)GuUHLd{JgMw+s*f+R$cf) zC+qBm+jD${`W0HgD;59iSnz*{`Y*ikhdvZ^dZ5_Q7n;wQUpL$x7gjw;07wZSi$Z=QKRMSkxH?Fo zN=%tZP|9Z<&B2py+~7PF@oGil3(@ z3a_F;CYe@B!yrg77$@;^KgRM4tWI6WrRoGmge(TqQb0viiqa!1$UhN~qfO~JzP~^6LtroHYD1*)&;^$+sJL$04ff_ZSQ-6Q>U*@OE@%y$^Qkr|(w@5iB(p&B z4RO)fXWYDJq}lN$16Wc_SxDjPR=T5MPnpzgT zD&kQMlSjw+TR8z& zjN&Yd%V8abfOb{)p%u`dDBvJeM&L9c&O7E{jbv!=$eEd2^_{y2^WBm1(LwSO6&f#YBOYRNZYaBGpirni=5L)1;5nj2M#vx6| zy$=~REPZBN>%ZQI@Rp_&uCCRH2F_m$DHYCT58QO$j}|ERZ0~qgK_yz^`et3hgb2r8 z(=*Rw7E%^xN#}aQu`e=*z}%M4Hc=SJtZx}yQsBXrf)050&`~emeSn~m*0JgphmX~U zK@aLS!pWBxFF+d09zaeY{}5K1^4zFD*iAVUF20IKtM-wkA@@2c zg&akg%A$MC08S>Y3CS!Fr^6opcH@kVrNP2;IR5p4oa0jcxXkn-m1n}7_Z|K+$?&+dS%WY{P8r4^_vq@9L+8Ufuc2+3zTZ}L4 zo?uo-Dw+&9qMqzY_z~?{GyO+5{~q)H(f=|y-XuadQ-5K(I8()L(-FE`U+9cf+2wz8 zAbNnWftNQ1(p5q=NsLhq_coyP4(iTC2eD-@f94gdR{dm~j&aRac+KaPLAnPRX1;Skz25i#J@VjT z=EOd0?Y-98tA4+I0pTF8gyt}_z?2ztG{M9#I>z0>!3Uu8?_&VkG7QQ`n3>NTD5P+K z4FZ&5RvnubGlZGBPlll;T?u@P;2#Jz{#mDTDm80w$EX-U0o9h46Sv6#&r2BbIuHn( z%l`oEONE5nxP;3&AO71M#}z2gGy{C3H8FiC_ZFIaT&Dc>@^o1ZKBlqX|ETTYwOR zVD#Qx)Lk1r6B>VN{@8ae>7 zS?4dQ`md;eiqS5L7XArrUl)Fo@x~ma=*16%6cE)@EvbDRhtbS&#hVcl0z{Mt^nj!# z`bR+Y8vpd@p5yO`8qj0|j4wZzH@jenGZ^^JcIbP&um~iAsq9TD_K05u=BbC#7XRen zTtu=(s0+aS)sYIZ27jS>T)8h7H)ux_90C$D_BItaz`t?rz+hSyU6e51T`M3M-hy5A zY@>cpRp6=IsnnsvZqj`9B4J_D`?lXxMW9+yoERLzWonc)n2u2J<_B=UkE>`e5y*;f ze9jS5r|KD6=dRTvw@R%W(+Pp(fr@*-1V7)55 zl9UMo5i*A*;YIC8eyK@U1s2^6Y2EMdS-EBtrGfrXX;S4iuq(O%I(yy@yuPL6BqFYMQf%knhhnO~O8n|s?( zn*hotlJ0?9$8co)SH=q*7yuQ>^ReQ}@^sc6JaG=lpGe>nix%vs2T$(L5eh`_0aV0g zLaU2A%A}amlQmJz)l3li0tOT36YCEcJX&1*0xr{;Q_wK;lJLba3-J>weDHycrXAN7 zn>&3PPGc=n8bBJ1eWB7l?aI5@cz*nlQ}^45=x+ES`vgp0y@9{|ve_pvJE&7J?y6n} zT)xxPgO{gsI?ss)=YAMTUR(1H)E=h&2xN|&--Y+c^qp!<5Z?@uiUzn#1Ib_CCOws2 z^&q^+Ir9^`jrrqEg~jb}9Z?wcH~T zPeeu;+J*44O$#f#f=~QdgqE?cLP|GX)7{O&KljBZ@27Tfp8ZrbmtkjV(G&rSWbvTX ziqib0`jgu?i6sO}JWj0pQl5u`)WA^kZ)alLo)_9gF!vN;8I523;&3d-g6VKK z)n@3-o$7`87O8y2n9o69G=5x603WnoA;Tw5ZVbd|P@#afExr?GlrgdG}=QnF|94~3D z5L)ktBWM*N)7SnWTqIxpvBSs!JWKhj&kK}O9tfZp9sROFIfBBWajcw1fR1!4F_h*n zghpZ<2C>%XJlw*dZ4K&LVENlZtYUT`ccWNd^w!j469Yv%dwZ7a2S2F!AK6CUBJ#bh zithfc!eds?WbzF1c`t0+`=T^SGFa%wt;uR+!Mz7vX|h(yZa-B=3KC7%z8i{0{fO)s z+!)b`(TL~bhv_`5)OxVOB|hFig(?&a&iBCJqy1dnX<@8cX<3};;12zqvk}ZaZKpwD zELN0C^{nj9F4`H@_f1D(V&5Aaqh1w|3mnrGz+3;pf@ zrsJ5gI}iY$(M4RKoXz_RVF8>&Ox}7<3wchD8-rO$+xvV9q+Uuh#v5vY4;M*3-6;3+ zw^hlxA#wZ;bv}wjaxAoRApR|3aJ=)Tv=HeZRYlL(zp!I&ZcpWg z@Rq-~gfA9_jNzf3YpN-h;a#e4+v)B`Q{O=7(_5wu^fW}1 z*iJt?+G+&hvmem(dh151ti_GINX$khen|%Erx^4iu|NuGvzfxLAUv@0i|HR|0a-Qd z)8l0(q!U+kJQgTa*6e#CDTP1k?0HnkHRG!)@m*f| zd07kqS+6jsTw^Zvxie7maZYJ%aI(QSXFIL$J?l4Z$oY@Nay1I0CicXah`V<*h^b0I zx0ZlsBTCX>!{9px@OW=fanCt+T;O1V1Et2qlpXQ9O0U%sw_` zCXL9IiHK0zl=nabVZZvZ&s&zw34p?zWJ;!-*G}k@TICfJi?8Tf9|f=$gE#$Zac3;( z7TW`N@Os@ma#}0hk=}aRhVCICENg97(Dh%Ki{W3{>((-fMpo$40Udd<_WiH{YbUKq zUi*?=0ir_Rz?yDgus7|=9FOm8&+3~{YoOU2ksfq^Zso6alo!~BIj_!nZe2TSqjV9G z8FVdld++y!MB~=j5%*r#aS*G&DW-_!-U5zJwaD$26HI08TAM)0vSn7jDl>2LPoc-c z3PH>)(^JnUkY^PSm!%=MP`oSR@3)6kt`pw%(baKX*!EsQzvJXWe*qI%e38|IrQ$^B z84j8|65oeaY2~OP55Yc1Kvt(QXu7mFyS;CRMJDDum^RQEi;(%VlrL%R_qpq#)7ydF zP20G?-}y0s!;knlt53ciF9-nEpE;7*iuVs!T(g7w(ND7re$=H}Zh7pySf+wrqvJ~d za{K=1Oq(bf*MXj(CSZmF(xHfbpTYb9tXhVHIf6ljXQ@Dm>RmK^x;f>@PuUk9fj^tB zUk5_^cygs+vmr^x=a^q6|9nl)=D7xpyZM__005x=?p=oJ9@?;UqQ;)rTM^&^S%B4c zKZj>Y`nP_i#=DrGjN@h(aM!fBir{O<&PUnc-NgfL9)|O#kN~#}yYvP2wb80NIvb>b~*I%KR%;62aN)@R~Ll zG(o>S1g%Hrne)X@E;)T7KyIeal{lbv@gJ-E;@~nBUWD2kO!t1qt)Y0CMGdgbPg=Jc z$liQNYz(Avf(e|s{$MsIKH-`ObuoCh>ao3HbwPvk^|c<;CcS$85tKOlcxr`2@$CR zT1_+JwtP^z*C*2S#{l;e!UJDE5JTodgHTI=f9=;4zXgzcz#=!q4BhiTNwF97r^Vey zyM0k8=O?5IX6_c4`@a99Z0}-01`qECb2PAJf}{SJb&*EvymGq&0}%?a31yIanU@3* z>hYD6P({dK41zE-?A~R9pXYaQv@BRZNfpu3Kp6Gay-03g#x&xOz8u9axvZ$TT-v{W zd)5M5kSOlt@5Vz}z!R;cbbAZQHp?uA$KQ>_jBg4PN$7X~)4Sv2GRYQj7p)Zm;MfFX zISQ}ml-8*XuH^S0^R~{cOm4BRCmr@3Q{T|Z#CO9hFrW@m z-DVuv7!%a2h_7s2wQqr69hKz!>z4e82(uR)$;Te5M#(bpCeo3q!PA7&!YO-q6My0_ zKS*__z9LU3cid)O!tRz>j%t)I_TatzEOV8AXI^mle0YL^{mm4UauVuCf5zR+WuntS)K zpG=|>lMK-(E%S$XvOaHWS1BlTOGN;D$NiTC^07!j)dB5x*kzwxxKe=J%n8#w`IGu$ z4rZSMy?-z~{RcUTz9l9RM6}Kk+m@#6F`hm3v&}h+{;<%Pf$mBjjw?Idy?Gxo`Z_|; zC8_Mm$qz?t&NTzCi-Ol%5r&j&HY*xm_j@IL*4|*#;u1n!so>f>=Y9dgyb|{SF_ive za~Ld-33#U4I4Ny1lBO+R9c`L?n>ZLi)e!f|mWhqmOH_X;>yr&$t_%SmCo*IV2yV*# z&qL56o5*jSTJji{K&kWdH8KmgSg|Q{-&PIP05QNN`OnPFI;wO^KZrOF@F<2!R>gX6 z8fp!oq|@{HeQx!paTn31DA(H&V9~I=G`H1&+d$5X;V^Vb!q>jAio$i{wos&FSIvpSt0=bfn-^n6$9h+LugZj=GgBQ)y}V42I% zUN#*kM=Rk%r90R!=_;tlfbbRa>I-5vlX)*Z))KV_(3o{3gx@(h@1FauV48Kb|KP8g zU%b96!@eb!@If)}K`i<<8*?;)-5pN~O!^i=@P#BeNM59gfQ6-Lm5*cpE53jVzW#Pl zN&tRduntD4YA`PI-%~_rsT2SGi=GCUv*9}#ZzV6U_>Hl7DU&XK16~2l!sa@U0BtnM zfOoBbWYWL#!V@TA{-6?O5%vG`rT`9a=#`!ac%VG-b9lk=uZ+AD{X)k0{iEQ+aaX`z zg!^P)b}j|(;AoiLe$ZxNvrdJvhHfQ%S07KEpraNW;q0-oX?%nnZ7DZD&rmpNdg|30@DA}u zfBaEP&2eCj&Pzm+A-jiQTQsU}ERH+d@Z5#+Em89D7zJ+(d7LZDvt!lcLyzO3!sF%O z_0fPCRH;f`0u|rumeW4S2|EJ>Xv+7>j7+(_5C(65Ui5nC#`<(>@!-I6r2J_mfP`Q_ zZ7i_SwhgmGp2x1n4~_Q(cfq|W%y$%VYU*Ip@f#tUM8jea$3qkArnxM}qJtC%69C%i zi|hXK*N*~K2{rOyox&NVN7}FZF80=JWC^;N9enWbg&NQQZWDOx!m2*Vo+6-;(!wQ2a<)J^&FyhK}76O)NN8O*eP zIa*vDG*y%*qa=I$EVi1B@SmZuTIA1BlV<1A}$~J?;g^g*O&M(MI6K zGvinlI7I+RpXy;;nL|lf+8eaG*aKfIM+1Mihe+bx2BT$$i916hROvbvKhOEBnD)cblJYbca3`3khu>(-NRAzvFFApYf z;SgW5n`?a{^WMr0YM(d9kF!q^XI^s>{cLQos>q-}vj}oiwNUL_wYW)o(L#s25rjld zZTKu&!wf7AXeaVf1v=8%bA>bNVX!!D)5Tj(ZI;9H@zRsv)f?6E`3hdkGGLSYmj&7c zr`yk>Q&(yq$jvyT=)m#)LUNe;KO=mRbQCybI^9d@VJ}`uLiymv&&FV<1hj=y^GU}h z2lHj#L+(i@oz#srol%ZeMODvhc`Ai!8ehs3Mt5JaLs~AH-4qY1sGV5XOk|6dvToJ> z&_N6Lsm~VMurOZg8ap0*$=H+dD6AcAt8VEn8LH`x3!-YoUfllV21`y+RuZkmb3$V0 zCnxLTW~DC10O%NqSVhWh{2GgtyR1_=ry6Zy&roV3?9AdZBFy&lji6#&+d3E(x0`6u??>4){Xpx6B>=Zk){eJ)Aqgn79s5v4lG;4-T|vqNl4=ujeartDWJwt!A_*Q^yFqQRuxCH~MC7y_$? z)dLwapY#EQ7W)O%(?kQ`;Fc=_69nAI)&A^L;~}i$ zT3n>pSr?0jJed_fnrfEd+Gw}}#PqVd({9{!0(zh?=hpXgeu0Jj5Uq*16HKOYK3sz%&iG!HCAzUo$G|lq{XHH)~jdmZ;YCMoZlY4st$fqbI8_qW<%T?wK zVSP5kc}nyw{Cg>qgel$|SySZP%@Yp*iLvb-I@|-C}-~0pf~J zM5W4xOLoPumEYS(6@jR(0Fo&TfM*(T{;Tyfx&!~iA9TkaXWDk3L8Oq+0qb6cjB0H% zzM~6XJ{kZJ^(N4qva-BJSfT<%94S_eson))M(BL>U@N^7THU7BZFv#HREoZPDbO2Z2kBwXcFukHr2RekV^03e838Pb?P;48@B zouf}h6$bY%pv0D^1;gdH4ksa1DHb%)9(8(Vjg_{*;b**@Em%o`3rVwpw+JZ z92=GPx#!GK)0Td@HtEKFeX(u3Q3EntAFOhqoz$DQ-gQ4;6NJsdrl!qwjvZO9QdNoJ zD^nEb(S(KQm^_VWDNQ$-j^DP|mR}S0O@VF|hxa=A>F4>l$wbK>(`l=s`~*2=qYB2cRxu zD@CS#4#{q4`7iUWe-=mEaz9lgLk_fog-2lWX``Sx8>?Dz<8TOx8JM^UEpv#c#lYoySh&>}DiR4u!rqFM7K(614sK8LJQ`+up{I+$_l^`$6t;11HY-faZuw{sracD ziE^`4^q+)BV`AH?@%9v2OLsb003MDuis^k=Bmojqjp4@MJY4vfziif?j?mYP~nRkxeV{xf#4GBc)l~xg! zWxl3y)H4rb*_-qgH~8iePWvDLSOIh3&ms_Q3*QKAdM!Za{LyVcm^-g6Osw(<#07Jh zvC0HGevMaotVP9WDX@O8r0+S(R_Qwgn|P5O9B>xQ6sG#FxvOZ;9S~`|hDqfuy4pH>UEk7}C)*wHJ%ZDPjyHPWOq6w%jek5iT&P1;@`!(TF=FF0j-c%deHqaEy42AJAiKc9&8$ z$D6KF7WKhM*OsrbFN=6>%R<6U*a00;<`hU&C8(6dK{ejbL8%Nh#x+pNQymz0Se-=- zx?7GEv}&#rbi{C5VZ&A4&H zwd=wI7Qe3;v{mO6{t8cX;lJP{VzKRD1;o8;52;GxP^z*NShFF;*K@u|7c**&eG_iw zym}UVZ%9o__}J~`YR&V#8x%uY%`H_iHn(g@1HoIFrGL&|peXZ|Dli$*G}H$loTB6SHb+FcTGROQ!qTUAaS~*)Okm z`>EA!c|Bq@?>GNk#qdOXMvW=aUKcJvp&&<@33i;t00?P z6i`sj)85L%7+6TFsp7}UrWMrrx+g2|^m$-%tB2-sV(f8a<08S#KBdb8Tx07DtA_aA z5S^JrkB#eTccP@o>wRzVFxNZ`GwNQ|nupR(vu{;$?mWM>f#*9)8#y3$sJ(=gzUH4zUoo1(Bi5}oigWjj#%lP^}MRD zw#Dmh5s%gxCZvX`t;J8F84}bj0Wo}gOY`U1>9|Dc_tw@E2G> z5IDyy-ae#yK#j?xM7V_BEfSBYJ3S*{Ty|IOJ)f0gz}?hvI<9K8@BL~6X#ik$7Fyc~ zLZOuRquFJz`3Y_P_5}E?W*W!T3Ld~B>6;b*B-Vg9>{G6GRAijEknbka&X8TfWRF}UcMobf_y zS2GhWze{9S`@5l~@b(FInFAX)l8*+`lpEYXT0^LGR2b|WWyHP6?0r|`;sd3YOQzz@hT_5_hiK#+d^)H2GjmCFfQiQvh^SLSfR(j?ojsE^7!2KB#}O7wOenC5=cL9!i~PfgX%0(ocDmhj!+LTJK^G?Y#Qviv9Tjhwf4dP2pa*{_9pg0zpd!uAMe7!OoOQ=w7RmMY^`iy~ z&(&T=5!RrZ%1yTRLvL{RUL$;6;~P%h;kX=#WT86sIMSI3nf5)9TnVvhckMNfraL;E z*qB-`oY^i$>-4Nog8Elji;`8;PIi(IYeSZ#bO)n1m*NvXmBrSR^}3nh=Vhg+mpJOV z9!jZX8hl`3&RyK@+yw^IM}p}aAB?!horKgklF{2%jAdb5ce`$ zSh=S^qH)h#O3+!gh9pPkw7VZnS)fH3;mFoNm6O686wgQEri5h+LlX;YNnK?PYRb*# zgCqnNemdNGF>!=WrG!uJCy3UW6EG^aQ+1k*_yzbINc}+v9iHs;H11C(t^?QmpG_aRbNsqh^L>-1`JmtC{CZ zslMUtM|khC`}|*SaF+=P*u0iw^%V%hu@G6+uPqj}qX_z%!@@?@V;Zv}(n{rug1vvs zWtvsr)k-?v=(Moa#f{hZx-W?$B1EvdAOMRh;n-*SyMrU=NFVuxCtgb9jfS68Gj0-c zdWY;IT2$n8uLQW(*J8iwK0Rf8zLnVWt*oSY8;K~ljX2w|7X9$>L(T?L`$dnBi=&>@ z;b?zC(FDw4doc!nz^+*>75RE*_z|RQE0MHtV>FD@{qeXx504M<#8|{3`*OLFSbPYD z^?3`*cU3qtFUQXfEm_ueYeqqtuuZM&Ly%#Y3VmvzdyaqBX=>wI(plyf2Ra^O4LcE6 zmk|_?ppCMl^t?QeHlPawFZ=qzCYh2W4G+f&5(*_&A}F5u$O=&H$t8VF7kBhH?jMVh z{-!VQ?)@c18H!zo=e5F?O)BS9HVp=cuj;4O=W{AQHaxMRl>mE<#m z7*1`gwQIX`vaaP@K1Y6XWnDiq&Sv+%DIV z8EZ9q!B|e?N%-tTQG~)=OS;{`0ujHuaQP=$w2F9lTZ7Ks_cncjxq>JD#2oq}d;Z&> zTggfMbUz8KYgTNf>&71zKPRSlFYk*MjheV-;U+a3Si8Dl{jB`#Er!*i7sl~gE|kD1 z`J_71puE7XCm-5sm{!CfsI0Bz7N&A%!TV%CLbaWkt6NOfKm*o5GJUdMwYq6h2q;a` zAy*8KClTTpz zQc6OL^WUGfRwWdtlUT3K8`(UYKppobhm0iKDrmy1IlUGHGiChk*14N+2FCoDYRXM7 zeyfLkuY+VfuG)DkGwW<1jT~o7%{BA?P~ckUNA%U^8P*X0qvitWq?LvoZ{%2_`_GQu z?HPo94)v*{%&pJ*nDPv{q-m!sd4ri>crh4_YERM}_sROOo9SJ@w7xR8=6&dGIvk2qd?%h3K9vFdQnSeHuhh zufPSF17p~3GLM%SRN&v`hTPCzvKQ?(!hukVr;KA-Cg+jtkBHS-HlJ5i+J@EjYE@t7 ze!miXgiL@dC1Wbt=V~TL`=9Zv2VWslS9bXzYm5a(Gdjp>TDa zKTu1+Pg`lLpVDo9*VbPpSHA`_H zG3$Dl>sCPRI$oIH3Av$Qtgu0l`?v#Y{p=}~d-3ODSt=}z(-PuE8O_%AnKDNRdxtyJY0O`g{p{bIlh?jPXRP z+P2{A<)izJkB(KkqLRc0>%7ID7rhy{RKLFj^w&-belIp1F{R#x&VkVKk3=Q zKv6n-;8XJXhoZ3z8F%uOx?-YA52kAN7yj|rE z89!+`yfJzqk0Tk5k*sD%3b|Xp*fGrL`3(1NDbBURX9PpZprpjdg&#!4peyT$z2M`T z5~@R^!5WZ;WCpF{QApaZ>2oUPzP0&4nlr!%K-4M;7!t}KBW6(PlDUEGPAaw?{%*=x zsX~D@dpRDr*kTsRc*m?YsYj~sL9Vc=EixWOv;_wbh6t2n;MtBTK(1A{n?&UA4j_PZL;>Q7&8Enh^Ytz#yovV2ZR0{W%ub zWe2`@p(z%=_=q|2-HFs+-wHv>UI;bhmiNJO3n?H4O8;K>>m)f&w704|IiGAco-qKo z7XQz+kEQF=M`!^8s6SaRQ2WodS!R#|ZQv83C<3{ioIQe#J#b{dE{_uxy8rt)$5*K3 zhITx+S&$Ul6FO|Fezr1yy6U!WCX_mt(d(tX^xd`l>^OV$?0aWT04g^bg}onX8r%v6AZ;EYPj2c20s@e+bsN>58Ikp&QUrZSNl9oRt00Y; zd3^;g9P^Dq*@q*9)j;W0qQ^4F)6Y^|zjiSssB9GCpOT-g8Ge;4z$o?OW zgOQlKE6{0zAVaHBec8{+yzsQ-1s`jJgjQ#Zx^tYT3-A_i_9Ek{*S45Syq>)jSsudaN4STf}hoycDv9IRo#G0`rM`Gfk3Hc_o$r_&m`!zTU&w>itoZ4*Xr z@LbY6-F)SRGjdo`R#^CV?_QjD3^s()VH z4w7|0>dHP{-`JlfrK&<~|71I#C^58=<~wRm;oE>mWHg7?odw&j(e&Qwb`h*|-Ncz! zIhGDBGkMvv34=>ei%eNBU}dKTfpI8b? z;^mHVsJ2y)_1)nQLIw&6#k5?WZuNQf8!0%W%3uV$vp-xSsO{$_rW2*v-Dboh91nj4 zwSBkBlu2AzyjLhSr03;>E$cRRHReVLxt$~aAnwpA?92fc#1Kf-dXK<2qZPStljOhD z6~hR#;5p83@W7B&(zct{vJ=%g9}WkKo$$O=50+HYtV|=`q1}+09Uah`_cyvjIGjC= z?FYV=l~6>N>G$H#HLW^H+G?-WK82vEC;wA;Q=2M3Sz>|YfdBA=D!jcZn2bl^AtV94 zSF8hahXZ$A^~s^^^Pm(HOeMw(-ZFE#d^LXoA-v~EVGkdre?-F*bz?fEp-I7;C`$8D zbG|~$v`Rt~Hne1s1Ol-@X9-VKfhL7U_pi zi>2i)^dmPHMihy*nmSW|(#%T6J$pBD6&&GAJB_G+UIS}6#$pM7lg(6`52$x?pkxi>M=zsWE}Q$d1|m{6C8!S`HQ zqwEnk+h=-Fdnyj*o|2qL)419LV9$kpHU3VAz z=MZZKYSyLLwDs2ZnlelOA;P+k4T$9+6MUT+3Eg<+!z-T0n=eO!;MX%<^YD9^I=ryDwC)s6VvCva$xdpmIKM6bhdoYr%sBUdlMYz*HJps zXU))wS8|GdSCWpOpqwanobPVkm)RbYPMN$`C^|;xY=x7{+hzZSY_KmDCg`soWVRO>NnQ0{6#o z<2lNM2KH9^lRcypSh$xBgNa8us8EY>!Bv3J50HA-WjcK_E@*YwBD@gSgGyhpW-bAc##)?3nhc^0HRn`~Qm@#r>LzlrGcY!|v z%p*U){i0>L%J=;*RaGT|!yV3U+EGZms6_U$m;^UxoF%8e{p69S_H2`qvqBh3d9!xZ zfp#I6y6(4{^X#P2&A8C`mMLhrYq28}_W#nblCt%Fzs&*h6incRxPK%FE_JRDFTn%u z-|F|`8bE&nHp1SHI0Qcn8&dG>KbjmcD&EElJ-?ytQMuw^1!&%4>yz(y601MSA|Llc z_hT7P#%hfXG_B`Io^G(*J#*!!+XNL_o=YezZ}@$#?zTS4>rUu!(4<>AM3}{BGKH)r z)!E_eoY;4N4i%O0h%Yk&tWa@Lt2f;ao1#f5-Kajxv9z`_$Hh4ane;MvA#Yh zKL3~Sj`a}q%;|W2hqZCmkT@P*AgDsLzuYU)s9c_O?6Ei)hWVKPZ*aT? zO1Lu^N%_tB+Pbl+SsuFQMq|ZhXT-^g`>x)7Mo+JRHO-mdulj@dO3eLBlP~Kvgz&y2 zjncOjFDe}7cx_GiGdW)GH;3LH$TvBaX=yjCu5*e8oiJi7B6{1G@r547mtGO0Y$;O`#L^+hd%kSj3SO*Li8uhU^yY zalhb(TGwD9o7{i8IkXj8oJmlR4#6Tn{$$;I^^nj+$!kalM-caVVdcU5cN(wz$kFO6 z66{zw)+7Ce z6(5gSY@M0lSH>!02A9eCx1Vq-wGFL}hUNtwPye!@Y6GEZcE=vt;-rNO1mp}~E9 zepuo6K~`Y*4Ssj~LQ^u_6mhl!tx{3eJa?k3NT{#I(g=@O(blBG=~1V25G;LwJlo>k zJxqp@pq$o&;7E&Pe;pY&8p_yWme;*HWL?9i{+s;T5;m?Mg}N|!H2%Bjg_iKLR2y=6 zv-Vpk**LyPGVAOZhZMSMFn4j8P0Ee5qzBre*Ej%+jtIl528XPcg@erGXLLZZijLc3 z+?3b|oyRX>X+gV)Xv`1bRDT@Qs@r z(`5#7ZFVPQIHCKpl$4o4RJdZdHu2n?ZG-H{D%vI)$n;M+@6Q+b61B>1Ik6R-wcD->svTiJ_@DdW-&Jep1pKK&J>1%3q_iAh*Me7%7f*U ziNZzyXV+;T&-$cNcBy?&j_oJ9C&^OZ`RY6BV z@4UX^)90F}`(`}mSu_d*S#oFnZBVjp=l!*(cH>L}>3Qp(T)~~* zY9!)N)B400vKv~pc!O@)u&Te$weCf@;BbR;Q99t~;eEZXhrM2XH>TdY z!6|3gLV@pcn+IxpaJNi@Dv*5=ZaF?3$zQjhN~C`}*kynWKP=YU??Nvv8W472EMAX? zInw4G@lk(}3I8?GQXXQ&re7%QNE*+8+qJlcTOOv`)eU_5xW^S~ICZ8~u-m~diUVT8 z@U+nAL%P#$z;to*qB+BwDov_-ghr)5pKL&S2EXuMp~kGZaL7(cGK-VPETun(wh9I( z7;i*SRn7adwjB%>WzY0EmL~<_ag`z}n?Hy8P$342Qh~GFN)dkz8({7rG^(Cg}wrQ?=({{tLJnQf|tAjZ=194cv+VZiIvjCUN z^U3p_o4XMx7sk5XeHbIn2GsrZO%CqaX6ec9PM+Z|Vm@X*{l$cV4l411{RokO+?<_? zj8F|5lWQ@okT>LFODh>Tu&rZhvuO_}crs91(R+H~|wj=R& zlWbaBpx@R<;jS_mH9t4S+G=JjU)9bv3mfdkx^v`tRJ?KNf4Vy#5WGgIdn~Pk3;kwf z@48>!<|Rc3+lvsJM~5YnI3SF*rCr_{cdM1M4V#Zf|6`@Lu&SA0`_aX5tHthV*@%wy zd$p~85Bw|vNP+5qq~2W;z?Tu8%JmVgGhV2ochg`oh0+QUcD|4hfep+c9xlPd-i~bA z4T5bRMz$VIJqtc#5mG9SA{SepN;4LJ)P6-wcT(0 zNdXl6$A}DZ=34jTSwyuIc~`vO*xx=2&@d26$Cuk-F61=g!oSN8(c+^Z_gzLS0DR`7 zWU}N*%=Lmv1of4V@Skq%aR&$Q%aD9xZWYDa!W(7R_{#gbSA$|dreMZIV4chSCY#lP z&yh>&FR#yU!!QeQ@st8EmbC%5RFCnqUP2Q2N~L0+P}+#V0QT#xCF!CZ2L{%@11+5w zM;*7T79jDh!?I;WF+74LKSx`oH)8&krCIkNY%jqv&tWJ<&9-n8$dC^Rlvs#A!Ikb_ zipPuRjNx@Pp_w?{kBd`$?VG&ZT1Yi$fLY4ud4ygO+~8Kz`h)3CTeKNJ=?iZcS1YPw z4uK@IAFaRSk5CT*?w?1Yi=^bB6<($N$ZZU|gDjG(9u{WW*5Jr|<94R_&Xm!kdi!K1 zTDAIC5WQz3-fZi^`%X2EeuTLOy}IooRxFU?xh^DeP%nBon9r{>PV7j|8pk z=Y+S13YI5p#LG7MVnZsX8C8e zQ;?o;6MFCH{CcT@DB8p|_U`nW0{Jy>fldCI%8Y{K(NRuV0cVAm0Ehp1H9;uGMhRl~ zX;-qBfZHR4Ub4*s6f^SL6*lGClcceGL^+kXm2A8w()d*ct`lGAt5Dto|}KK*ruq{0C;Vd-ihRDj_v!reg?Vh;Sp|BiXb+0q0P254U?}2M=YE zy@#~No3{^=k33-KhTZY}OyDMu>HkMhX5O|<}U`aw@e3hu^0Z~`Rxm!!!b&* zGY^p1aG`SN5)I*df;;1Li~mszqu8*NX>&d(&tHY2zV^A8xGwI&Zq9Sf*NN6>*_+^1 zuFnxc!fsmK&zb9#Ih=L%9moVy+yKJ-Ewf_mf;j)A{LJ(EVosc>9j1MOsFFoNdW8j((9BSYS1x zNd9nN-=N(UQItFr&8|agGLjd-op3h%!)-5eBwdvp*FaQ;dSZql(9Z%izG?v@HWYC3 zwfbi!P_`vMV%S1XvD4KY>^4?>Ba5+GI}?nX7kIo-Y`fuwrXK&#kfvr%`%++jRM#i5 zvQQUF4Kj9?iE;IN#43M_o_w+TcKo}LEAFuQV;1)r=J#jsYx2Y!a?@`=obzX9ag`T% z{+Q_Okn9-&0FS3QaK!X-C>e;Q)5|$SuDX=@clE6jLn~d>q}_fpuv@U4-+PdDNw1U* zjM$GChvdXp*bEkrnSVWj?}2Q3qB7vS)bkB}aXg-u<1fjw)ze>ssV>z#F0<67{Udcp) z=&sm5PHm`3ws$;^Kdr1$i-r%>gRUw4K{9h@#(xeRd?tGvqCapdw z?p@oWU@Jz96lxN|>Nc4gUMwX$9?Vrskvhb0I0$b|$_BYa6btIx=y2?1@NEJczHsxr zP>(I>Q9c=pNo%}Hr;SuRr4?f*##buCHi#YeRZ0_8XU;|PXpVPw|H~bn`(qjgtOZk# z&*F}GXKm_N>ZkjzK#7iC&+mUJ|Fm;<8>X^*rhIgo^}!5S+3%LVKLc#>DOaHS_*((4 zSEbmUM!7mHx0$Lnm9^{pqYqkNtbLN|TBdshSVRjaZ3lP}Ri?Y2PG~6?Oi7HoI9LOv zsE)b~1np0P1-G92=!!x$LUx}TB)F5-t$qfaLI)G{02Vag|2ny^U@8%N1`yV&M|>^* zAK3#s!n52V1x41FcO8|tsyJ=_XQ!UdsQU!`i0~;8WuTRbkdVCG0E2=nlKl!N;N!ed zPAQWZcO9c9NI0l2Zy@v-*AnZG)c{=c<~iV^$j6xJ2Q-cuo@LqJ-+^HdIa1*J^f}W= zAI#BTyH4=q-!TWFLxQ^}HwbGbLnhz-@1>XLw9u!_Doakk{$ovx>m~yy9Qn6B#ZzHyQ%5%f(ui#b+96OKJ+PNe=tV3PtwwL^h$@BDI!d=b?<|Hud?OTn5U!V^6 z!}9G>JeZqqMhtg!$6(hZb>DwtIFM||e51&{BgeK+bZHKuA^8!0@_#=cq-F{uhSDHy zm*fs)qh334;rmEvK*P83wy*m!t#%biUU%EGCaWn{kCKiQ#-yvz+H-c}#J*mwJ6B!o z{9w?toyl@ecAWVCFh))tA_~k>=18G2Ol`|A;Y+oOK5fb#+=9r3^R|{1HUU25!qqmw zM@i7<>G920L48G;C^&FmmgJ6-;9fQ8UA~-eo_o}~cYYr-$wj|cIWgcA{@P$ z$-?=yKz1=!{~H5*!w8tuuOWj#Wb!ULWchz)J2-<9fFUdZrHjp`e?8>)vbYc|65yix zAVK%%slV=WR|T@n05V(7G6qqo8&8n`PkbZ-9brPQp%tm2TGbKVSWtw(&xWXoa2Hsbl{Qt6mC$ z+mLbbl(zsKm3y*nX)cHKn?rM$tMV75XAT3Xz>xIV=HGY7m#828b9PEMFoviT%!dLO z&(FFAd42Ccqm0A>Xt2ji^qSW5E#0j1Z~FZ!|MH&!()Y_K*59(>_b?6#EWPu=7;stN zyWRic!Vk!m5l|qcd;qv0i7_GfXH3A$w=>Pw{az_8Gys*!3!HybcD5daN9yl+#R{-t zNdE)hg{U@#oc^_uxEQAAGOpatTWnUdF5h>b{${{0UdWxJVOx6N&_>7THU4`LmlqFH z>q=va>!LvU+9!VVH=A-I>*In8zeG`dAENTd$DpE@t3m_9cSrbHVzVdd{0e=mo@Rr?=$8yLgP0_1ef)>qx_PJc5D|Ay?Z(WLzH z%KxPfgv#*bcE-oc~}XFC~(nBF)bF=?M+ zfdA!q0rSuzau^j7FZ~&34JW+ym@XlL6y`+#oQ@qqRX0> zU3CTdaAgU;io-=;99X5~P+WE~S|L45Q?0LjNr+``{kCY<;?oY2-S7L}XRh@lZXT!t zIIKd8*v8`|u#s{49rrMh3ZZzl$sV}UP!gES(y~Fuz%ij%{RalQ2arjP8+sYK*ft@j zhd;@woYtFHzwge%O=NcdA8T(J7UlNE4NEAXA_yW#gCI%@NXvjCptJ&_q%tB6(m8}d zr%I~+jdYPxacJ2CC|kvlb|kP`)&?SnEH;7tnr%* zcytV;M6klfDFa4+q7PhrGA?l;>?lV*dC!aE$N8>DDm)K`HY2FM*2cYL_lBs(my)vL zPdI+dB&WRXXk}EzLQ-sDxV^q&#-!26Xv>qKZa|;J9tT+d8Y3v%U*r>Zn2eHmGC zQ2oH-N1;2n)zF;v)3Mau@5*{){LPZ5bO#Fbzr`;n@LmYK(w6asEsqpQcSOhdNbwR6_C2+J{2Ken zvTN?-pnteRi~8Pe`e6Qxx}W`lqfL5=T=w!3UyPuccw~+vqB{RvsHwrNX|B@GzN%W@ zvFL+H$x>Nx=>lwhGi$`I!n@OK=MJ^w;pr@+I#TzUyoD;mJ7B{0;$Xt2J1ZND9v@)4 zidp-kc(XET!&%7M_MSKzQnAD2-u##HV9J#@zSTt>9dsHDe3G!H5oX_ItV=2==h$ta zB@T-{Sf=WHJ`BGmeMG7thuS65=t&(4L2bJbPgKCyr}jrhw+hm=l=WpxIVC*0wXQ{^ zdGp;g94O8s>^D>2*R2-fZ2-C?m(u<9Ic+=|Yo*e?x$kND^f*9(I+3_A;^LoR0VA9C z?RW4N`<_ZkFZSEX_ANLUtG3z#KV7jYLf%wPgaWK}ey+1Ed z{p3|uBwYCYuKJh9#v-NHUFuA&Vn@=FA$fXRGDKK~3yhPL=Ww&y6Ex8vR)Tp#~v?#%0@ zxOv$Wh%lSc{K>a@wlN~#ZWmGq+x#Nz9_TB#H8pusNKd;{iF(vL?P@FgDuU_dj($|$ z;(RxsK}kh)&aN3Vy}KyEiwc~4<6%UbJ2nlO=E~f!WMz{Er<@x~vBf|)T?jgrA*~&T zCs;Pq*>5`&tXo=&fcNi#*c5{}AGxYlYMAUiaq-&s;u&)pCyU8C;X7V3DRN0hjhZHr zJaH(!ho+KUUF%%w-+htJ>W}Z(wl;WPvsl>4!g-%rOOd9iU@dWLAa_#l!gjb~t2LvK z=|DNJqunw^nUl3l*Rz^07wMw*($osJ{QG%4i~J`wht=3Uq7pWJq~K`3Nx}G8F-e@jlIEY_-@v;wbgC$xc)&VM@W7xYgq5!mb-s0T(=}-|yK? zyyX>c_`tz2AbkmLhSDxe7|0yG!_Hc2P!o9O(0DjU|8suxb50k^#v&{`+XwMe1f`{> zaNVE-h42e!ca&aLeOFr|gqtcbb3F?v1|mXT+^y$oM$_fO>vO*@{PtiuiqcUJ&xhTo zPvECe`duqisAL!Z|GT+Ipc-EL?9mutxs#!3fu_Aqb{)(^F zYZ}C$^7#Rns7RtumWjMyR|ac!KzE5#bv~*oqKF@DDj2}9SR>9vJIiSCU81Ed{t|S! z@3FEmLw=+)*FWsc88uO^F{tR9!j_Vg?ZZP&Z)vKP$EphdZ8s|duV0eo%8_Nusyg(l zsJQ)m{G1roW%hUxM?YGcc}p<1qVvJ`o6BHKv zw`V3?J0o`?JH*KSRF{_BfsK>;9P>9Dxd9JaPGPvU!hWBPBKwwMFN2XD$|U(k!o#i_ zkUg3icpXGx0u{rkc!mtw4ae*A;x9Tz*C=b)iHwvvQZr16z z4rY!BM#c5{i*5v2E8a9A?wgUuM>Udt`%+{gX0ghh%;Q(otLGoSk)=n=-4L?7p>J!F z)eso6$=!yrc~RFY53+f%d=ZqqGfi84j~ebptz-rAQg1x3#!Bez-PXD2rQm1vfMJ05 z2=%RsBELDYqm6oRZ5rMr=+AAE?NKpau`{WHRksu0sY6Arzs@V7tLIWEMsYB4Yf@xb z(Ml1uv^^f)=FAVW`^a^kUcH7eiIA4F)Po&F&}f*f!SxW$Ilv%6T?;z65PqOgXg)s` zcv*9WQpoZ}bDUVk%=W6$;iDJ^_x`Ju2PF`N*?YR|=I!Ki4F?$b)1&C3x?9+KZnel_ zpJonHwJVzq->RDQJ8hHBdt&P{De2*ADr{aHu@iH_o3N`gJFq3YbICE}G_XbRJvf`NN1VzZ;RjJCp?LSn6?_INkA8D=gt2Tj7Xg=m!?XfGfN&Ya;oK z1#`h5Y2digWk_fTrV*DvWkC5A-xRi{p^nmV`xq(fPGR36OI*&;X4@NIGJJoDb|_$Uu6eH70Mz zQ`}l@6ihNj*&9x#ju2)>XY-O1T#BrmBh0uKR#rmuF<;WD9bl_yJmD>FeyZ~g$@yOe z(mz7P!SEVZ^eH`~Dya@p--Ey|(r`fIg`Ln_{va(=V`4OsCMdomj`J_>MsA5gDX zx@3Q_D$K4r7+N>md(ci1?X6uTPCviXWiYft?nNr)lUk_D=h_-7;ey-U;ps$4DzOMC*FjGh&XY{UEmcbO4Zp(XV z_6x(8SBvh+%>>C1?M2R0)CKSJ-#PoMA;{hjf*?Qj$$zigi6*#=R^E;!r z;gaeKqXVlPXYouzZ=nTzzwKyW@%G-#=PdixG3(^374dL508b(n-Fk?^IM*D%Xxp`HKcVGS+LbP!GVZY37?>|Fv)C;atJ!BU^AU30eSCX;`Uqr@jm6>geQ%l0 z$ZGsJ;(9LSyVR*)r{j%p#7*;7l(e|+e;u(aS{R@}(H?G%*onGKHw|(5 z6mb-fNUjY^%=q!PdY$7eq%XFazic(t>|L^4E@D z&{&GIsHWOhX_Vq#tD0m%?}1KYM)m&ZggDRG3QIxP{iV9)SNjkVKerV!?D;v$-g7jQ zd~nF-Jra`kR;Q+@c`n+j)mxUci0N>Y?vP>V=x}@B#fa6&saTv+7kipjH2K(^(o~!B z7pt@qq!kt@qe;u8V)ybW5L2$T!RGr#@@L47sOz;q!{XScHy4W5G|AS5ZcY)3`O26| zPgxcmu%E3P@fXq|8$8_FecU%MnEkrVy$ChP7B+pVqGVNsx}s@SL<&s3HJ2x!qa0RG2uzJSI1&l*UAD$T$oO70Xw zt5EXC)%@@uN~=TQw6A_ye$T?(e=r(ofFw(7Q^pkMcbtlUV=kD%+`Wij;~IIok!j~y zrE4Ys>%~fOZbdum2(5s}bDje_$q1H}G05G49Sf8n|(PM10fZ-){m1 zD5b13FCtQK8-3mS{!1x0T#)hhT8*L2xBPD!9rFqr=v#n5u7evqcmB)(^X;T!<#}ky zA4vh$U@1&B^%oMVB#*1f?3w8e|oK!RX_Ky&>ecTc4zEXFveMNFvclj zn-c^DdM}93esC*4-nABpoo{Qa|2Zvx!NTYf{lO%RF8SkN)?kR+{GYK-0XoM&`yjqa z@=6c=o4+UO9(h&i&1UgV#MiqTuJOa9RY zVgKwrQ~OOUrxTt_f&o75vQy9s#!xms5jXmT$EdjzmP|2{)A0xU{?~ep6(TqchV`k<0 zTd;RrZ#hje)9#+X32KNt&cbLD*Lg4#oz)xp>5eWsy@55j=zcPopSq1X^( zy9Zk3x%V@@YUqQHapi>8PP^``X#Y^VX3Y(H59K{*P(?E~APvRQXkgtK>WElbY{R|m zZHC;8?s+Pc?J}4&w3th=;Lwjo%weJtOiI$vHiBaBB$Cs3se5TGboPD=M3`ji4MUC; z{XKej9+p@2g{$CuLq;P8vn)k;9BeDADii^Q&D zztOaIUt?_l*qHXU_hU~=Gbr3ix(rDa%WGPm;1K)bYFfP`j<@tZn_l-*?|f8U-Qsbe z;Nw*+)`FxFBX)thOhc1Fw2nWC*qWWj_CS~4#o}(MR^?@6t!es0+11eY**Wf12BxL*6xp-Gop}A8V$eFFXKkYrQf+uW?-=w$tZheJ?&$gyRpy4?&U+ZRWK^+;ickJ zz@bXUrPW#yciuj1Lnvw)oIF2gQl9tYla&n<93!L@V?3N2?~GDNU=^CSG_>iV`+T-l^Jc>&`&N zr!yqi>Hp_Jx98TX?$BITzC>i>GPU2vmeo-YN+~+rPS()vs?+O8gJ)3iuJqcsTqLg% zA|0k8r$u$x&s(VV1H-&>T3l5O&;B18Oo#?=OR>JT&oCJ(`QCjot;x-*u4tUvl2hng zv_>oUbfneNbX43j4) zdTQCGbb56N=|sQ$u9RhMs9vYmA!9-`$AtWut`qTyrbVlu-M>h+9X{a>F)%Arr4!Av zL+SP^ZM_r2vNyOQh_=Crc{xJ0&^j!a4lpDNYa$bz(<0oRXQ(cnBev^Su-xSx``|XHp z{UJ7owuv%jo;*pw!uhRsMXBAlnUt)tKXMKq_RxJ1{K&5e+c?iwL_!GQ2`qwz2W}O9 z>(Jb;mfEe+HkPODI<-aE}$a@Bk zmB&yeV0LbT{o&MsIo6WvrkpR?5s7uf#Suu`GWo4ZFDAb{GsG>rQi2@FV!ko)m`mQ!m)Ln7WWWntOoY(&_7LZj3c>5W&}av)1X z=@8c*(DQ>_&i11%(=I5wTillM{eyVRMd*Bw7<=uWyCCpARp?Y#*)s^@`RI0PIi}Qe zO}o*_h~O?|0s*cdc{FJ&;dBy#@-2*>xs8ok#Es=-2Z^P}Hz4&cn#KA3F@9m|9`00n zbV%`;ZORaUr$Ulfe4XIM@~1v0qjgci&+WL`R>3h*cTqj)@@{qdpfb;w%gK3}Q|j9A zO#Hlx-f}9Q`7N@AMzesN2dO#P+wBXYdAqdH4-K&U zit1bCKY><&j`)Qn;9CpqJ|}~%*#5%At6h)*PuZGa3??v#Z_-!Jy;vYtXoZ1(0C2{F znR?5%(qrBf#)t*Y7t>!^FHbT;T=_9ve2X@+>|3Z{>8Okxcu!3wA)xK0u*&bNMrSJp zvcAyJrM#|8Wr+)DxEkPZfm65KF#~7Um}s128@KtOQ{gGk@99#6`c(Cid^`htpf%dyYc1EdBh;AKEWa2Rq65A@h5+TP_rXO;h z8^hcpRGHkSVDDttct>COvDLBKQg)J{o^=MMSqvSh^WSiI++Jv`L z?I%7-!}oG$-yX+_vG8|xy9I0Z;>`Uu@=a>m3PM+{@y9|Txn0#K1_<(pk zS>1f(+_bEJ6bIXHvr!$|wVHQL9}_{bx7O_pPFjU$Z4^yUQ{6Y^yBALxmNKI1uiwQ7 z3CE<1*yK7c7Avwp`uU9EwQC_LEt?DpN6Csq#CEm>uEhCjav1NM&NgAuA?J?W)y-uD zN1Z%Hl#PI~Q;UB3gS*OyUlg53KFs2w_RCFNyL?nFR-=ZBq!asS3J$(1#5XR)#%!9j z3){|7c&R@X({QnmgDnOdn3VOl?@W&U6wrx$3kxu;8N+jVD-f{z30RPZUaxmmZM2GQ zy&0;{;O&oi`C|-qbt$(tM%agaXAwi9Ak?1o=l&fcyQ5F7v0_XNMyn1|?*v>|lBt>( zHI|t6;zYBO>3o$Csm)&a zI-NXrzmwS7P<+DqZze^}uZFDF-}lyX=d0wd%C@Bm9I~q#J(q2Aepx=#L^(!R!NhQn z;XcTOO|$#`o;54U`tz87?|w+MO;2Dg(SGMQdBp_kF4I2hwT6_TY#+{6ticr=Bs(4` z85HLuc_p97Sn2dV&b$Q1wjR*~_L{BJw-bLAQ7)uO^bMF_Z4U zE6$>SdSPFW5;ZPG6USOJcl6BV*Oygp-Cv&wi|^PoUaMeV#c`65nz)JRzxn`Ii zM9AH3tOqeD`=lcYZdc}Oo(mZ*Dv=K@>hvz7Ugh)<7hQd^$oI;PM#beJR({6p=L>Xs z8PAcBsJI`0A9sg+jMuyE8tilSO_y>#rmdkL=gc;x`OSx-%Ok>3Z_4HoVPBlw9|>s< zx5B{#dr*qku#k=9q;#bpU^T{z;V@Ij6qkOrJZ0s($~WE-aZZ+O)@jUtrI-0_F<}1M zh;3{B%-kYq`LQ40BEZm1rQoS~*K4JAlEnH|#OZ-O4$=tX`ph?_lB2lc)kd9wm; z846>6PbDETvGIKt~{!W*YK=Oy3m)ifD^b0*QPRObW>Ai5uR#nKPaz4E4_*+pjn z#G)56e)nmDBDN3gk3+>=cc*?zn*C~HX577Z z-t9?s%Lo<8=eZK(7Se9NaOseC6S-QY7o^GgUQ8;HOd!2&xoPX7Gi>vnB3}6{e5v9P zS&56amJHv&cxF$j(Dn;``poGP*{(JMlp zU*Y`LEDNmMTjFTo=rG}uSd{0hw{A7|-PznzuAJ+>*J6$X3L zAmoZz=#(DkCA=7f*n7$~|2eCz-_WJ}PSrI~Lk8@RzUj6F3@YPDkZYjVml%cGo44`B zUWmPIT$eiXGsuI}V$gI9pbco__$ClTWakqDn3OhwdPq6ER3c8GLW%_%*$~_cIpw-K zb*Q0fRBYMKe@rv3O6CMCql)tJv_WKwRS(aO1`5`76Iu$r!B7Xwk1k8?u&( z|H(tfz#d7ufK^nTEw$9w@eTeLJA{yc-mno>8a8XPlL62R!Hl)w>joNv2t%*1lhMM| zmz;dvZi(%PvEF5%y=l;I8~n3I@f!pb|D=Vg-~%7xtkA=+7_fQrL1ikM1d#uLHviWk zyv1d@#*ay97PndF_+!ihpkK77fHlgz9;W~Qn4*dpmf90e@O|T?9Gam2JVx(s#558K z4V8-xX7Jd}>^=t3K@Zx0ZRAgg4EeCHo3VQ38KnMnWaJb3Rc zSVd&3%yAvU@wf4%_~HJTsl?v_8$qa>g~7j5+$-xQHFS4V^SI6DvhAk|&W(A;wffH< zZJONPSkxbD>uL2%)^TE`-I@Eqy*aN~8&@I3+jC@nd++n~SVLT!Z=W59A33ciE!)cs z-8C!)fcf#6Bo(5`N(tBbMq zVBd{1L=@EH0ar+oaY8o~N><(q&TA=;$%nic?0|1$s(SZBmHWcS#}UnUl`sfn z#R-rg0j&no<*~c&J=<;0Tb(dnA771{uG>n?sQGl&qc(y(<~?bqg5PIF8ahu!iYsrV zwgR;eiluvKV?HB1-+Y!Ic(Wzmp071vFr19Ap6_FJBwDy4ae;~$e$C&f;hUmYPx<5* z!Gj;9@R^{hEQisi)!x9kKo-}-yX;N1{@90V9P`)dNwki!M>NljxZ&pt#Xs#5YEVUg zEdK7IIKATro|SHwGL`PH8Hw>t@;FLT1!0~u=a;GQ3e45oyR7TIpL@Ohpg=>eU$M@| zZMm`XWOHKAe&Ophyq0%f{nB^!?hZzLg%e2t80G4-FO>A{U(a;UkYEI1BmTnRp-l7P zV#1TX-eRU#F|WEiI$WQ`y-&BST!C4P^&xA5r2_-V>C|c3ewP*Yt$x>|Y0)mkL@?ve z6NE#N8}~T~Y1W$V?1G4#M1ZY$wV$)Ps4j5{c0fy&ec?(o6WB+zgG4VlsYcP+ph+P$gim;&l~(4yM+lsXJ1p|&v8Sa;Sfh5nW@6OdBhr(IF0Mz#HUHh3oGxXg!7(2Jww5|L% zyfcymWd}y@J(9`8OrM9|t$i}A^w#pdOY2QWkzwB5B-ib*C!^v(T52p6rnt#{5BEZU zSC-3SotB8+R_{bYQWCa1Dif^QW0l)^S1?CkG3|5ZAa?`Hv z2Tq)3O+-t}_o6LHIRsJ9WMmKR14kf_R4!qCFr~o;)J3xHz7I2cB0YcmoFA)uKNOZP znH~y3BU(?exy+q}UcxpspH3>mCc2`(Wa?{MO#Lp7Mep*NVGcn=4rKimhv=2Y=i>eh`z%MH-5)RS$6s8ev-arpya6L+ z3t%&BC}==0r}ZFy1lJnu#;yDDzpG5gXF8E}E8K6!lMUPIHcH~WCeNwy8EB*`j9m=d znYaTj#lDv}aCfJQ&LwL6b7FqYRZh$_5?x|2LCLiH@y!*~iE1Iv4YD74I(!sQOUTBD z5~xIvs5Y@QVd8$3vA7<7w8Y=aHs4a*G7>LcX&x)w?lu&{GO=aX$mG!;Du*kMU#11+ zafXS&Ae<_dHH_0WGeEEgXb0wv zXiPnt-knobGjDTP>KdkLNwu78{UxnJ>*!n#pUIUW9B;2ycCHSR0x4HInPpqM*Sqs` zs<(V`(q2!0scqPy&q_{gy!UzA?Wu7**PU813oq%x+vFTWwRE6QB=reFBwZ#RluyLa zg~Ru*ATKyganp(IRC67AOWYotLcPN1&&E*k91^x3E(Sw0?Up?A%U>>2gc)mdJbACo ztfUGQP0E~gF*+oXCm(NBpunv=7$mr~NjkP*r|A7axlSss%Cn}GMx>KDuuaaDllI_1 znR`>Mb38Uq!N0_;RBUO#gRV0a8b3hxAROtuKLUD9u_$mq7>^S?*&pBi`VYJ;+8Tjj z#wpz5?IlXIbPk}oA{__gRDju8MjzU-)Xa0jn1^1?8Uh8Q7-%)L-k&=e?oyXk9qZ7H zR5a!cL9-8rT4g=Lmh*6Gk%b5_+1x2%Ka4Sy$Xpo~MBliQ$mA^a!0_|(6@TbmzVKkh14gqLf2>`Zi!3^pSJb=H28 z__U@y#O5n9Ih_%Cdyx~$u|3hpU{pSAd@1Lr4Gqlu@54xkZ{G0IxZk~Fyeq6NucJF2 zR(ffzc}5m{5`x%ETRs1kqY-a|Pm%W$U!^JRvr3b&(UkY}U1lZ7L}Wf=CU`PCR#2LQ zUfpSI=VSsMvmas6)Qe%IxVMc)OG~5(R78NWWF%Id4ke+0>W!IyBCTj0E#3!}J)o3d zygo_kK!5zt{I9cM*Iq1bG<^4WiU-E-0xv}8+WcMI@z>Hrfa!?Qdz(;L{ZQy84tmlC ztnO-2Mo*?}fdb$`nmFo7R=zm4$XXe*uiBz4EEJ-sTzKGI# z^NPbx;gn}ID6OP3e~m_BKm#a~R{c8(yhH+b;e$9S{s!GJZ>-bYyK7HEhu)UfXWR(R zsxH5gpP27(*l_xeOJaaAx(xQ@Eth(>VNo;HqJb4k+fw!>y2P8n*70WOd1_8cb z*|R4m2tBL)UAw@#NWFO(;;dET8Wy6pE2ydG{#5)oUyH7X2gaR|lj)XpUZbbm31!c8 zKPHSn=_?Ta530lhyCwY0S^-_B>fX)Ro{xy4uo`No5G>Naz1z;a8sI&?KA=tY(?+4S(K@7wJYEC47(c|pBM zzzy#pd$Qtf9oz;an&rFA6Su8hH^Z)!eu`Eu$_IyQ|B$i ziA=_YLo!)spVZU7#?*Ux&uaDfKR^}zD#-AHf55y~l;D`pw{D9-ZA1T#rz^=|=UqPc z>E#c}t79{=Qj#cl)3JBE=f=wizSiQ{-KGvaa(LP7V5!X>QEPzz7VC&Y zjM@rOj6d3mtqKeJI4)D~3s`{4wpXAw{}=y+!jHQ2FeO*LQ=+ zclnrk{;(YR`AkjlG-2R_WlA#S zQT0WOpPqq!${Cb&$}k6&=&fV|!=^t3iZiG^^7?BoA=v?s?->|CC!Ftr8yCZ9)BT-C zB<0ax_!x-M0y*--IjGFtC6c$GJs)CwhK#!?u2?Zq+4wE|+~hLm65SJ-hB^CF1iwz5 zDu4UJ50cTmK&D;7{j9*G^av7wwWc@!{8*_Rw^M3Ec=(E5Bm zqhX~g)5g`Rssca^&cQ0uq3AkDT9T^l!yH!k6b83vHDMtuiOkLp1UkAu!Xj1@uQ?YH z=ve*;i&;rzao#1+dHo~o(*(}QA8kc+xZ>+NCg+T*r#CAP3|$(}_%%;YYoQTY0y2j{ zh7~yXG{#pS#LM6p0#}hf?*FatH&tSD>~(|dydgss$kBut0~*=*Iv_jA#t7muwZBPM zur+#21Ig($$ar0M?~-K3maW%5Ioj`@Da=tmPkU3aj)wPIfSA>zXxmW&i|&%IHBp>m zqOA^X#^dr~F|--3$YhKDC`GS^Z!~1mepEC#jzoU88p1XGPos3ZGcMQV&g<8v4v^Qp zN2(#a!}5F1E0be@yfoQko2#R^?ab%pFt;;g5$%00`t;GQ2H622VmS?hjJ66P+wI#>1A6qFqM4xvlF*dP`R(?H7=3$PZ$!455=_ z9!JLxU}LyjGW6$lX>8L*?i*Ie`K2GvVZBY4cD>2+STv78${%)1+jTXWd-Tgyc;!mD z#VeHUY55!ph4T`(qO9i%1yL~>-oc+ektr-aNVtF~9b4>Ewe}~%AgkI(Mtu5259IQ= z^QTb{uN<>lHdCe*ij5mp;Hx_epMPIz^W&+s^wD#KOys=VC}ON8C(!$9AT13SELS+T z#0kVE$oa#Ejuim<6-p4elL$9uf_UGFWF;w;;5GvJW2XJ9RkY($mPNf=Y+9u8f1SVY zQ@XGI92Uq!f$$X>L0P_!pJgGBM(G0X<=cg~g>&bLrDN>vg}x|f&> zuAf^DI2V5$3N#Op5!>_WHpvboIAx^k6@_afN|0aqG#YsX&`&AZV`KCU76E9HaykQN ze2N{t2GAY}Z&AH>FOSM@@navnXiD{Ec!~%Ox*`I4FPLMd4xkcB@(Ywb0ph>4&}u-7pdyw))=0Y?HfSG!A)Hbf3XaMT8xM)eSPI5MZtw=spvC=( z?*9$yJc%~`Zoc830%a?bed~|}I{|K<)dv|Q|F_=FWyoPbdpE7I!J^0fcf8S{M5gL{ z@|Ro`71lOJ!vu(L4{5>cpX2B@XI+In+uDX{g@DeZrMLFKYroY{af)&bS?|tTv^r@= z<^N+wBwNk>tMx~tyZGtp_OGw41OVo7OiRdqMDzIr0t!euod-M_ViDW?2t#Vj*OM}p zrHvme157h&nt>qACmhiDIO@LlbZuwuGSbX!jH|5h#>;(Y?I0@&EB)c3o0*0Nsh?oI zRm85wQ|abwgOiFG{tn7vd=o!mfG6&O-IKxXzlo7HSOmXFuzDbY9PfyqJKuh-PDk*z zu(k2GHhPgGEg@;_hEI#E-v|dVSqtQuXm+DNs`I#JZd&K8 zu6ZN4DTaeZ=M+-5kVt6Zh=cLs0H9uva~6zqn7>;q;dnt?Y!zjPMPH;4uFvNLZk&YvbES8LVU8HqkHfYa_*Rs)#!uD>(4B|;3g=^JH zZZ`EP)yH7pS7e6e{^biF|2Bx0fb0l02-3xGP8(rO} z3;V+N<*l8Vk~`nVj!Q}4*3c`%v4F)4sj%{xgz0uptT2DJf~mFMD;X1(&k5P$wCE6l zGqKn7sf!60q61Jg9lU#>4YYKu0DFGtu z(O<;0?h#&#`(2#)BIm#ay`aI;X3|Jm%Pht5BCfP%1t*Y zyZO4g`aa&v^i!?-#2}WbQSb_08L*?0ZK|Hf5z!sznVD~e3b-{@t&O_?vMnK#h%BS^ zKREua5b@DPTeIN^*7qx=jzbnKxnLq^Az4fH>F+ak0etHMXH<>6o}<<~vYb#e5u^>zq@C5_LiDM5ro(2g zoHXc^H6W<)m2Cb2E0~n%v7wS;b-9t*VNSWOhxiYxEq`1L>rc}tys;_jT;16qrYdVV zeTtU&Yhl6VE(Mim7P}0LSGwc_hi?n8gy;G9*>TFOP7NOD`efz=r?9MDRdg2JY}&P$ zxp6Sw{sF~u(H};ss;`+4PR+N9#qeDV_2%bfnB1rLzD=yjk0dG+8q<2b%+7IrZ-ZJJq2>%CE~9dW~j8^Ph4W)Z{7pz+8_!>3Ef(yt1C2DGn+v~v3;Yy5_bZHZRTw^*1>^10^b-RS@?ixlZk_ zc5YhcPXz(!;Zk#b>A94odR9)tm~WHxMWhdn>=Agf8VFm;sYj>FI5BS21+U6G7h@c) zy*7;nXxle*6&bnK*lei}j&*BFaAmla2j3py$9I}sW0Yh%?>3UT2yQ7<273Sr#CNY2 z2ymb-k_@pAklTLKv5ueEuK))wM9=aX)K3R|b)6i03KX#Xx(%$MNY}IY7(FvbO5tL8 z1o|)KhwrXL?@6RN%E1QgOwO$tcilS=l&uOA)`w#Q1gHaWC5Z4qI#e75@{;AAEu5~_ zoo9<0Z>w;+)y=NaGvQEslL^=@ZM_0SY zxqNtzplC)C9nJ;v+4?Oe%(78ZaoURst)=-pD>DvX>T5qJ$~0UO-U_+S{c^bImCiLA z_`3kD>0V$?KC1xq+w#4SMuhV2Rp!0K?awzSg||{s;;?M{vCuhQJG5*Y&vG73F)3uy zIKeVh-;ZEw%G@Jaf{qB^$-m1ooDA*FwKP_4(Ji*399!Hm;e%X!hxip;iU_*4RTA$Qu%lqKx7zamsVRZpO0S$!RaY#xdm$Xk>so4;V zzlwa-;U&2D{$Yp9vnc!Lhd&CJ^9@%kX^4>HuCZ6FVx!4rr#J%h*t(H*Vf@G~Q!XT% zh|X{L>OZy-fc?=BiyaXQE!qGAoF@Kkpl*@-g=L(|?yqA-+Pn9&Y1V2XS5e;U0ePSg zp{xM1buoj<9NtEC`NObiaR+M>uuAIce}|}O;aLaR`104zIZ)|t%;SBd_~v5bOwE&^ zLA}*~M+hrLe}(X!NWr2XX$D199Wt8t)?-)+tkzL)fb9-nH_+69wnFYcDK}a3>eq8M z%}g4?%2OTZ>or|+mL0jhA2&QG!+V>?;tF%kkl|cg7%Y*+Wk15rPu+v1o>lgUo=eyv5Kbr6v~5;I@`{bL*)n6>+_??#hwHYs5wsx>y*^S)2Us2QPL*~Yx> zdb2D<_0|mc7g>77hyEKiX;xMVEv>f{R4;Ws6tTQ}iA>%v=XaLI`ReP#bU!X2-UPy! zro=MYRe2s}NlKC*Je`MqF>Wq2HKU9}Oeds@2yZThv!)Zj7PdAG+q?L-{sT@;_H&kv zbN581f{Av`p5}0LbMz9|Byf?2dHMG8jV@mONx9nbq)EqE;&33(oK&O7Tv137V;TtA zmha0fG!wYcloTkMHGFWsNZc?xO7)TjCF%O~A}|DH`LBEiy?}eO4!R};3g$sF@xRSk z(8Uf7Qdxnf2%hU#+o)(i4DqoB9C=ZcvNrp4+_>>-gq;d{S#Ht2#4 znf62tb7=1`bNm+(&mOxrL+NXRxc!%qY1+q@3!g&e!9nHAsNHxr#~r@M68tJf4{hFYSgmP*9knr{ zn!ju&Ge}!LI{PA^GQ?bD*Q5^Jh&#px*~uHJwDGv z4X0<>I`t|&eaDr*lVI@_7AE(zS6};3P^i5}&+fJ*(-2UqiA}`blBdbbdo4loFN_kF z?a-_j2dm4j15jn>io0Mf;>QdN-d=Xr-eCjRJe=mHZd!B0-l)vb&NFu3mQcP85U@n{^R$$YaU#o$6rxTi2ci{%ek)xEcTttEt7;2y)-LS^dXW z2Wo^&0k9GQixTK|g0Xx4nj=*IkxC0((k?4bt>df5PJZNmm$V&S-QWXd#0P(C`SRDh z)dSsvPyP!M1i%G~E*{Otwc6m)xaQ-^I}9!LpgLwh1&?GffiB6)E>J%J-1YI%W!MKw z6dsA`Zl^JP=$Bqgq)aMvV1`+D^j(S%^T4UUHRBUaF_5(wat28?S3_s4-wN4Dz^x7x zYBqhP2}Ehd1#?@?09G z#9w?Dn>uczfuWKuslldvrz?-yri%PA-9SSz-6q}&EsCZ%ZE_WU3%R@=932GD;eJcR z>QCRO_Y8M0B5o}_(jUSJk?Fm8lofY7Y)ZEY7NY29C0Bjqkr<_Wh|{Foq4Rh`H$&dS z3n@s5s&noLBwB#Zs8|ITS6QyFJkksg36AgA@jW8H<=#Uh>)C-Ds^7C{E%gceB4?cr z17+bSwiAWM+C*gh@~#+XLzy600LOVR`{(^@Pk7DYsj=&i zqU~$Let;j+*D~d7r2Pw)~iXk;$*VTa7t>tI?vR!lkQFG zZu`Ux*iP*S%8%ne0uFebroaT7n6#zj%>*J>a7**c;Z5Cz`pMwv1(#=OZvvLePzK9{ zWconV;ir^11}#r3&0*<~Z%{UFy$tHcl~1cL` z76c1DB4Hb8u>}l$7Uo>6x&67uRDih2B|VpP6$%b%-ppNNP3g@C1dZ(Ml?Y_e8v7OK zuSll&RP}NB;`(-ZIO~@c>33>}PYw{4gT*ClXBmvQw0sjJdbVaCNltYpF;ZE|4POcz zKZHAV+2ypsYl_WqM8Hk+-BQ4;PWyFY!~2$Ybr^&j%w z#%B#ldh(;Z2%BWPQkRqP2pDj<0?B86P`V7P4-swBB2GEubDj-t$_VL=bw_3caEPa zU_wH5V9S4CJJRD7vVWU06LU-DvCP%G8!sO*(Xn=G10xyvB;Dq|b5Ea_HJ*a0sYvUo zKJeqNOnkG^QP@L{*d@d|Pc_9GtI9q~PPf4w=Ha>!Ud_F3u+W)AQC%*(957q1yNTc? zr|Zqeq%%9eSYKOz5m&gQFIF-7rir;|8gz4#*Sow0B|3AhXCVW)*YwX*)nzi^S$+!E znEpNAFvXB+k&!L8Umf;OZo~hn?7IW0eBZds$Vx~e>yTAsW+n$&WrS>!O+{w*F;4cb zWRIg{Z;Fsn$;rsfo|)NOj``jXX?)-J_s9Dm=YHD~7nk{#mp^6fmWVU&SM`j@#ki%lGE|C8y z*k+l+Sb@qqewsy*8SJ66g~sUi8i-PY@>nbQ#seOwhGV$&4=;kpOek$UNF8N3U8Ec> z&G@NM<%L;}g4_NaF$JMLN}e9O%X;F9=&v4I9bE=z5JJxP6y}9&`e|eIUHVQ=4u0GC zIKJ<`XrmcFfvaWAZ!wlsQ*kw36z`xJK=&f~b@kNRUVDQx)2YuY$SSq?TXIM@-L+JUZ1IKzWP>JbwQaJ5Zx@5v6%EhGNpPJ5;C0s-d{b zzIs!kr{1Dwm3s!WJud1=D})&L_wKsq?*yh$0E%@?SN8f8QU-ZtfmCO`rAF?{3rPM= z5@xMxBRg<Y)`cOPZgLr8lV%Izjvz1Q9F=x?4 zTi+)qk$N_KYwJU@|JOXBVQS~W&4jIQ6sTYg4y}dHPcgk8c*f-ox&?DbsFrJC)j`kl zp24ZtW)6OsY1PM}-*bH)rGia-5#DJwp$jcC;|lYrFR2%<)}{=5zZ^Ldp*#h&U@k%0 zPGP(fi?3&rrAFRO_19fU$o0TfaHD4|l9JEtOiDLe4^^h`%_HyXa2GXjuYFx1k7&7R zu47%7iW=u11>A6<5(3;>A9CVb^DQBq7x)1{=?~44r(jJ7$8f|OcGA_9PB04L@0ZF@BC0vy*F;d((bo0d=$E9 zp?F8iwtP`MUs+@f_I)NUuB21jgk){Hnlv@y`gPvd9cOBwa6FJ8d~G~O2qCbzg`r!C zziJR!zvQ{)6}c=+yPLEV?J!n^iKu@^U?P%h8A8cyDqMLgJ$laRC`rT{S zUS3&~#P_i!;sz6UN4&b_yt+wOA&q57h|y`LlaVHSz*@JQrT?w?rM?NYIZD?0O|v*i zKuq5quRs^h! z@%(VEYjdyEvt(qH_AUfcM_gaOPb)OLmBROMA^vTVeHq9t9Las2m-3~rLf^9Mc3sb@ zkK}MFrtU!1fZ|Fyspi>^w+cM|N`k_+%e7QOc6ZN8lY0C>YndxsZG%vp8Z(Ero%jlI z^Fgx9l)DBf^5@~S;^rtdvK)|GjY^%iy|^*M>MTZep^i#1xiKh|Bf3EG69`a?DlN1~ zB+Tl{(UgnE_5muL=BUZMstI;GiRhL&7OBP^bp@l!g6<3h6a)SASy-KP%4_*xlU2A~ z<1J+7#)JVX)OmML^m*mZVucXt$H1Qfl?P9giF!vsK-wLUW_A3DI`K^yhRHlFs61eM z-$8$7*fvmqq;k^Vo6Z9h=FqryQg+Dx75h#iQL;^M(n0B(8t#RaFG*A&Zv^laXKYJ% ziyo6dJiCEc8UqfU>u})@@luOW&!JlFt6bF?(zk}^+ z!MtJJ7$#m|Te*-km8&4~jAl*r=OXEvJr?a6%|rYCWIPr9 zl&c*wTZt5$YNpD`nlfflk-;HlBULUYWb-|RgXbHv-6WlPLn5W9Rrn&7hNSwG$_77( zlSGo#7f0hfj|^AR(Ck%JE#W?yx#aE1YuM&p5e%DBS$To6yszNblOC3EhlOme!HTr$ z9p$;0Z=5=Bm>nZY1l8%x_SVxrV2P22AZ!CJt!M1)*IDaJA;wPF*{I$AVT^isxm#Z5 z!lsS@g~^26Tk>&!s%}cRVvPFDjm1vURfm*DLRg|!9XCSddlx6$sqZyKAf`$r;|{TC z3+`H0N8|RF0C+tqUD3|C*iZDAcBbC2X>)Pd^&a6ei}&9{E2eJ<2D~+eb5-xseMl>P zV7$oab^1X4huwpjuQixkDoU|e9{}HR`ileUgAUlKR$|RQ2ng`@WR8}`!Mp}%hrveEXId!QGYgn*!6IV+(! zS}bK}u!#2M=?Zkci@DKKgpdbxJoDNVp#Ed;hicP z?-ft{xpUJzJ#$e_ml^_g?P4^}7ey>b zuI!JA8<2xw>M%rj^QYv8?9w?QBkL_Ldm;rS(mcEEq%3npAzS$)!?t5IYF$)&xisw| zgDWRFwuNkV_58RK#p6Y5CZx=0u~bL+4$XZ&S6wrluy{enYp;BrS7;8sG(jF7WW}{#)UaC=;jsrXtxp*%8pFtBiugzkESBs4SDNzwaNE#fqu5L4j7yqR+Ia+IiY|NtW=LUuuTIAq zxL)YhfY|kCOz64~(bs3#B61@+{{2A3byXPwV4-=80z&&#Q4r=0MM-!4Xe!CA z&a8Uf^&hjfUYdJ#CyZ$ZwuyNIw^?0yOq1Bxqa-qgoEbrcVXZ_v(aVt;#gN;WJ zoN?DJ?rTG-?Sn7kQ91yadcj@_*`E1qdaA$DmjjU?u)(zDNYh)M|1o6iu5R3^HTq1} zak%kL4!kL04A6!Ax>`!r`pJXjguq9G%PaK|{g${&z)&A5Vxb-1rp& zsR4+0^;hC+aZk|=@}%B5&3H~#=`#Wl2Aff6_1YtPE%9Wly{uL~2oHSOi&+ZsoN6!p zN-XCI+v~ete?ONuhsPvQb0l41 zvS*qbrb<4C!%IUu-x-O;coo|mD$n6KnT-wEevKE!BM(7RFKIF{gHa2xw{RN&L{|S*j zE_(x0G##lR+hLa_EXhV$7f8H*#-$W_F}N;=1q}LRi#P;b~k zRH|i55DnZ5tnUBt-j4s|w_7NlB5|B{^UYK5!w|~(2mm1OH+uIxS$(0a;=f4e9Rr23391+Y;thU7{Juu(k=sS+$S39T1) z<+bpt{!(6jkQ=d|yFwfa$t}8yx2+YMfP?>kD7u};imr7!hXADKDzoWz2XbQF1-Ov$ zOlXHj)ibHJ`0iso0bHBj%BLvhaSb{`TR!s^7De@aoRnkqvb}(YiU@X>ERu_!BE#e1 zjdki_&ht|^mvw>rkQ$8Q4mZ4%oz$IL&n;o|(H#m3Pf`SFb9c*#%5=;>kfqF$s@SXP z(Ut~ryUH^^Xw|*G*HjlM<&4z=V>@}tptbK5d z1(Xq2KjZa6Y`iCKx0Eic$iE%%SW#qywm(hsK)s&Mqk&7aPq%@)X|j%^zCL_GG*Z0o ziuTj1BGv%iBk(xANk7DNd#`uvg)*8RZ+Z$O6q2n6`u-p51aNtH)W|}%2Ce{PJU}P_ zhAUu{q|)DZk|+Anks-y7j*lp3m-Cl^W|iN48OE8E7z#6H)RsxGFD6Ixkd8Xs-JLq`h?j6r6} z1NE?HLQ&ciF)+yc?N*(!n7URPL)dkOL|)&*uBTo&B>+E;H--abGg0O3)zYdrOe+rt z;OH!=1ZHYh*jyR9!jA^KC@fKXJY8deicmNsX^3P%&^jV%zMb>TG1CDJ{<4oOIu?oFxkhUaf$CWL=fp9UInBm*ySbDWrEaAzqjux)G>Lqh%EiR^;Z-X=8X;6Xk<>HC2BYR}=I3 zo!qcTYSbU~rS{31x2Jq1j8(q-k@mint7JxeAQ-{7u@ZH%fr$k;Dn;4!c2bYa3eY{d zu=aapKzbSSXo)Uhb;|BnoBIe!5OtxGTsE{$_BBBIvcCopVqGEmcAEbeX_$t(6zk7h zKg%DUA@;pI)v4L6O9R=Rl`^O;+f#F}4|F!jvBL=N1WE!&SA>!1R&4dN5jk<=b0#__ zk1&FL{l#{rRdP;5TvevxHC?C#Oa`CjW>UxBkmTzcJBN_+U8L#Leb zbCy5i#b~8}c#oIe@GgR?%hGjd@h~{b7AyV8l=nB46tjvH4yj3f;?*m8!j%&dgs&Tx z=;`pe4<$k*U^2_pe1)oqOX16?>+qsD9UiFP<@M)82UIy3PVAu1!$M2K2EBSaBe=KY zZy4A!h%TZxPlXqI^1`L#bvV9U9!9#C-r8dEMpsSVwM_1)n2U8EK zcHROoSZMTu|jS3L&A(465M>G|p~KY!phk7&YcWTS z|M^nd&H{b)l=@^$3j`ie)U5FPz!J{eLr#)&5rQq_}9hwA!Kw?N_L{u zZ^F6zo1fs%%lg6VWz=^>oIHO2_NS}TUs51vu&%ZSr>y@JZ!(WU+`hCMc`< zP4izC>Eljam5?Fgp_YNGW258bYScVZ4}#TQnsx zq2iv)pjYeOSNX{Iqw=9sitpF7oMj!RP4^oj9U2Aaa(p>+J@-e-1mfApVO8^u=3-?o z^s5M~Z&^n~(7!fC<_RncCKrtp3TnW%lbv|Nxbc4??d<;>X-_;v+97luUlgG0PPJSy zoI3PnLkn>^h3)pl{_ydZc}w{OJIsSUfYdBA^VRb}S=e|FVoIUNv8nsr`q!9x6EBQq zc7<`GJfC!`)v%w%dlrqT@2D-EPb82H#$qS>>IgOec2EcmT!{4T_bW~Iwuxi$;0;%4 zOi+VVulotzsmHerPt%O?^E-dv^;_v`J+RR23E~DGrvECajnS5s6Gl#y;rvTBf4s9Tr?ypFQ$5X zZ3Ri^*^7>Le9ERb)z)iqy~Zdqk7b0rI#$Z(DqKfgaJ^`<9LvNW4Ikm&0Zd*Iu>CPS zyVVD54)5fcxS|X|pm0T9@R|%QVq`gtX4a}94&3b=u@?`wr$iP3)sYWCJVetKR||+` zaF}gVa7Y=F-Hgq%nP1%qoxLPo(&{;B@X913d)a)&j+U37wxh0Y5x`AXI4 z%I?g4LiuO}f_Gi%K(mx3mE$ zB6>%Bnm`r5{gTN22}V(y+>q-vfRgQJv9Ht7q3zt9Uhl!=E_Ve|jr~(<+5fzUE|X)&Gt+k7mGQvh|%3 z)?xbPV|@_tK@#2zE3t#ZU?n68R4zCHDm1%x9m+KMJb!j`K%YkQ{{Vz}o^zjnHVBC2 zjQgIOAzM4yss%}tmp@Py$%bBgF(gK9tQnK9{{-Zc`GEAntA7_X{+z+@hm?3VpSy9V zGRy?SnUFBj^KFlJJp5+)=)rd435SU--kro?Z9fL7F0wpNM{3`waoi4u$! z7jYPpRib$omq>nSxUk3YUv$1C^f6(rJUTfP?l+{Qm%I>@`f-h%}+T z(uQ611(?)D_7VfP}i$;++oW7jWjAM6tguo z%ZMU>dzN8kFI$J*2UFkPoMP`;mPwvyy1U2}K)m;6clvxQZ}^Oj{HOCPMAf||W>xc% zOe@8^YchiMMuH^l2yYBTO-S`J<;-ml*W@ zsT9d9qAF({co7rJL&+C2JI2n!*`JeshAcNv6v$fPOSuy$Pxkk-1!XOgRdl`y_X5@l zyW@w(v;V+}k}NcO)E7!%3J9un|)4gFM^2 zSDfn&G28~#*8pgR=5|KezUY_8VC!KT{z+(GyNyoFrE(0xy- zCk?TRZiw_;{W8T9Aux$Hg~2u)%Kns@#o6qZD8mFnywRafhB?av5<~&6dZ6ny#yK6axcu6F_XE9u zsfK{}8E83vY+Wo690SI>Bw^xN)Q=gW*kr7b1|~o48~;#Cgn(gd{QKo`CQfg7{zu-NsvA zb>PM+8di*#j(kJPy&m&++}{d^p@~P{F&3sCExnY1V8C83BfjTWM}%pByOq$R!586I zlMxDxBU{dMVP*A}EeIv!-M)fdg}XaxmXS#$M-zm!XEwZj_@@dSTtFJ9@5Tam5fw9V zTT0GUxv^{%6evmCxriAOdz|L4ElWYZhpHG;%Zk98N^)(UdqT*bow~YEI4xXa!WEW% z$@fxy;mZiw`DVeA@4}YQ`3D)ESEfQ*S}Tkf6U*(o%PrEz$x_dR0pq+Zt4ezV*tT|| zoDRLM)UJ?Vg)G!&`tDX*R}3ydytlVeTm4^J9r~V6+Ks)4e7*oqsE%mIG&(3Bm^_1F z`0rIt(E!`&EuoVSQ0%)T3}?pD9I?I%<(CU)N;Dosf3hR5uZJj7yKdp%gamPsE%5E1 zx@thk{;~LBW+-l#f|NgTxZKq32uS@H6{pcFbJACl+?brJPRvdO{GsnAevUPbkOi7- zrG31zmY{KS4``ORxQZJuSBE_d|LF+`!~1L#-;UpgbA1sNwZ30zv{x^Y^6>tZxg2hn z50i;yiIK!hZ}rQq4yi^TC{4I^I@>%f)U|GS#o`|L$Xz5m@XQet#P1tlFtn?CeK)|x zeI6y@r#oTL5$F;XbgR!kQ~FhKaB!OFDi@h4;@kJs-ftSW(|iU$rM&ccqJYUX|LB#^ zu_yYlV=LteAu13K;x%pzgb-26Wwaz=t%R6tXU)FKp4}~5$o|&Io{@Q8;f&t0gn$;r zvOB}5Hv$h^J!KRSE2jligI9VHi^RbXmyKve6aGF-!Sq8RxB{~2Q9Hz-gX(akCEP)Y zp^d>`^?Ov!t6VJhgXgmKWF@3;-@W0f`53}WZA=9Rq!ayB5qt2=lvfW!OxRx+e#Qen z%E}v*8X&vm`z+!W_9h3Q(n@~*C-aT{@xW<9PtN;M879 zSUL``L}s&$9->BXVu3;uLN*J80Z8~AD6FoaDj*}EQ99DQU6fHA?6d>(q=@tI6pvXk z+uJgOxi1!chL_fjQJoCb#yIsFf1&~&FufT|2Np|&|X+C9J*z=xWL|uz7-gT zell|`twc?)xg@umxN@NrJ?3B9z=cWB_KMaPtquFDh?cKCeGV%9vJDgqL z9^ooL`u?ZdH0~f9cr}s>_w@S;tDIL*15lkEBH_qi?`W&=MrzH`z43x-3Hy6n{(1!J zOxsgXCiS&g1Iv-1m-w&ug2GI-+FHwoYVDB!^yP@`wM?D^u#yWM6?TNF_;lDiT-q{4dpUUMn9DQ3|!~uRVEb+GZ zQvm&-dJ?-pMufBUg}3AJ2oL8HG`El2o!wSz$*D7U_OD3>1w*)rlHRpgphY4E-6wwr zffT^aRXUtkCvu+U{;y5#$FI7EGynU!(L10?g$LIqp(pbQKw!Y1NyoWpI7)#lgtbSQ z8T#kF&;zI0ct&e2-_di#{^$2gDHG4Y63H%B_y09ADg}wRpm8c`Fw;+EIgVFMV$}F# z#IMop5m32o^r%rB?jWQTZldz9Ex{m>@&I%(eZXQ32cK5CHtL^=NudB`n%6YO;R`|k zc7^nZLCOb0;`h{6@tov;&sgmP+`CC9kQJbtvx^Z!f41y>832<(>m(rnCbR3!N7a<)#Z&i&YVgn9G`xc3=!^n)!0S8nyM1`j;`i#O znEr^G=5zc>ky!LbVn$}>YsZ`(^rUovg+MG}6 z>_N3(unMZTgL zZHnkltt?o_umA%jX08w|Jdf#?4Gr)sPgCJ94R624 zc#cD)lMGm&V4UYs4b7$&e2;7LL@Xa6*IC8n3pz^P7Fc+Io%M$6X|AFRYqf+I#Urs)nAZ6XwFTz1i(Rd(IW-+y3n-Au7Z?#9dU@#)0n zL6tH6d2r~>jV&D|>s(^Z9zxKrb-BmT7aOXmIn@rD=QpSfV>BEUkVR?SN?}KuJer!C zTSUR_M=a72LTn>lnD*3>P1#NKM`t=LoOseNG+V4Rk>VlNu<2`JE^=f1ixA2#wMM!|$FVY9!_H&7S-A-lCJX0HR8qA*k;#?8w+EBggCRfZ8r#;-stP|IqYuV%E5GUal zl{~q3zvI(7pO}(O=lJ?Hv6^{A1X!vRN~EbiA9r8OX~pxbxdSU9EUzSMkK?fZ!^+pZfB7pfZH z-kCyeyY=aFqr4{VnpkLM!A3F1UFD!_c#kL(y%2jvk(HGufirtLY)M7WE%sh|#>^wP z#CsXjqF2UMz|!(xewwQ=#j^7p+3?ZSxVWg{iIp2KXzueRPHArm4y>Mnq1a)s@l>_y z@ihzy!g0F;+aRHOx3&iwsQNzj^7MLPqWrJ1ni_GGKSB)p*hwrgqRg;=vM)Ur$N= z1PeV>rj3LuEstS^AIpF6iWD|57s(Y_=dlzB|Et9CvxDP7YMfU7?Or*-a>kDFls z_Ak98{M`-kM`5Y=1~^LEUVxFSEfgQ=FOWoc!>+>T>@yilzYaBfl(X&!ATBudzd|r+6VFHSUi(^0`s6{-tQ0K(64Z zD|*-R0!1tQhv`3HjPVCK^r8rG9IS2oH%iU`j)h$`tgDtc3S)-1z9h{ rGDtFT;^2V5r&k#KaSD10`^3+3ynTDBhO6M06Srh;OXuG(eDZ$)$oXs) literal 0 HcmV?d00001 diff --git a/docs/using-the-nodejs-wrapper/ClusterId.md b/docs/using-the-nodejs-wrapper/ClusterId.md new file mode 100644 index 00000000..8e310f00 --- /dev/null +++ b/docs/using-the-nodejs-wrapper/ClusterId.md @@ -0,0 +1,239 @@ +# Understanding the clusterId Parameter + +## Overview + +The `clusterId` parameter is a critical configuration setting when using the AWS Advanced NodeJS Wrapper to **connect to multiple database clusters within a single application**. This parameter serves as a unique identifier that enables the driver to maintain separate caches and state for each distinct database cluster your application connects to. + +## What is a Cluster? + +Understanding what constitutes a cluster is crucial for correctly setting the `clusterId` parameter. In the context of the AWS Advanced NodeJS Wrapper, a **cluster** is a logical grouping of database instances that should share the same topology cache and monitoring services. + +A cluster represents one writer instance (primary) and zero or more reader instances (replicas). These make up shared topology that the driver needs to track, and are the group of instances the driver can reconnect to when a failover is detected. + +### Examples of Clusters + +- Aurora DB Cluster (one writer + multiple readers) +- RDS Multi-AZ DB Cluster (one writer + two readers) +- Aurora Global Database (when supplying a global db endpoint, the driver considers them as a single cluster) + +> **Rule of thumb:** If the driver should track separate topology information and perform independent failover operations, use different `clusterId` values. If instances share the same topology and failover domain, use the same `clusterId`. + +## Why clusterId is Important + +The AWS Advanced NodeJS Wrapper uses the `clusterId` as a **key for internal caching mechanisms** to optimize performance and maintain cluster-specific state. Without proper `clusterId` configuration, your application may experience: + +- Cache collisions between different clusters +- Incorrect topology information +- Degraded performance due to cache invalidation + +## Why Not Use AWS DB Cluster Identifiers? + +Host information can take many forms: + +- **IP Address Connections:** `10.0.1.50` ← No cluster info! +- **Custom Domain Names:** `db.mycompany.com` ← Custom domain +- **Custom Endpoints:** `my-custom-endpoint.cluster-custom-abc.us-east-1.rds.amazonaws.com` ← Custom endpoint +- **Proxy Connections:** `my-proxy.proxy-abc.us-east-1.rds.amazonaws.com` ← Proxy, not actual cluster + +In fact, all of these could reference the exact same cluster. Therefore, because the driver cannot reliably parse cluster information from all connection types, **it is up to the user to explicitly provide the `clusterId`**. + +## How clusterId is Used Internally + +The driver uses `clusterId` as a cache key for topology information and monitoring services. This enables multiple connections to the same cluster to share cached data and avoid redundant db meta-data. + +### Example: Single Cluster with Multiple Connections + +The following diagram shows how connections with the same `clusterId` share cached resources: + +![Single Cluster Example](../images/cluster_id_one_cluster_example.png) + +**Key Points:** + +- Three connections use different connection strings (custom endpoint, IP address, cluster endpoint) but all specify **`clusterId: "foo"`** +- All three connections share the same Topology Cache and Monitor Threads in the driver +- The Topology Cache stores a key-value mapping where `"foo"` maps to `["instance-1", "instance-2", "instance-3"]` +- Despite different connection URLs, all connections monitor and query the same physical database cluster + +**The Impact:** +Shared resources eliminate redundant topology queries and reduce monitoring overhead. + +### Example: Multiple Clusters with Separate Cache Isolation + +The following diagram shows how different `clusterId` values maintain separate caches for different clusters. + +![Two Cluster Example](../images/cluster_id_two_cluster_example.png) + +**Key Points:** + +- Connection 1 and 3 use **`clusterId: "foo"`** and share the same cache entries +- Connection 2 uses **`clusterId: "bar"`** and has completely separate cache entries +- Each `clusterId` acts as a key in the cache Map structure: `Map` +- Topology Cache maintains separate entries: `"foo"` → `[instance-1, instance-2, instance-3]` and `"bar"` → `[instance-4, instance-5]` +- Monitor Cache maintains separate monitor threads for each cluster +- Monitors poll their respective database clusters and update the corresponding topology cache entries + +**The Impact:** +This isolation prevents cache collisions and ensures correct failover behavior for each cluster. + +## When to Specify clusterId + +### Required: Multiple Clusters in One Application + +You **must** specify a unique `clusterId` for every DB cluster when your application connects to multiple database clusters: + +```typescript +// Sample data migration app +const sourceParams = { + host: "source-db.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "source-cluster" +}; +const sourceClient = new AwsPGClient(sourceParams); +await sourceClient.connect(); + +const destParams = { + host: "dest-db.us-west-2.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "destination-cluster" // Different clusterId! +}; +const destClient = new AwsPGClient(destParams); +await destClient.connect(); + +// Read from source, write to destination +const result = await sourceClient.query("SELECT * FROM users"); +// ... migration logic + +// If you are connecting `source-db` with a different host later on, use the same clusterId +const sourceIpParams = { + host: "10.0.0.1", + port: 5432, + user: "admin", + password: "***", + database: "mydb", + clusterId: "source-cluster" // Same ID as sourceClient +}; +const sourceIpClient = new AwsPGClient(sourceIpParams); +await sourceIpClient.connect(); +``` + +### Optional: Single Cluster Applications + +If your application only connects to one cluster, you can omit `clusterId` (defaults to `"1"`): + +```typescript +// Single cluster - clusterId defaults to "1" +const params = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb" +}; +const client = new AwsPGClient(params); +await client.connect(); +``` + +This also includes if you have multiple connections using different host information: + +```typescript +const params = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + port: 5432, + user: "admin", + password: "***", + database: "mydb" + // clusterId defaults to "1" +}; +const urlClient = new AwsPGClient(params); +await urlClient.connect(); + +// "10.0.0.1" -> IP address of source-db. So it is the same cluster. +const ipParams = { + host: "10.0.0.1", + port: 5432, + user: "admin", + password: "***", + database: "mydb" + // clusterId defaults to "1" +}; +const ipClient = new AwsPGClient(ipParams); +await ipClient.connect(); +``` + +## Critical Warnings + +### NEVER Share clusterId Between Different Clusters + +Using the same `clusterId` for different database clusters will cause serious issues: + +```typescript +// ❌ WRONG - Same clusterId for different clusters +const sourceParams = { + host: "source-db.us-east-1.rds.amazonaws.com", + clusterId: "shared-id" // ← BAD! + // ... +}; + +const destParams = { + host: "dest-db.us-west-2.rds.amazonaws.com", + clusterId: "shared-id" // ← BAD! Same ID for different cluster + // ... +}; +``` + +**Problems this causes:** + +- Topology cache collision (dest-db's topology could overwrite source-db's) +- Incorrect failover behavior (driver may try to failover to wrong cluster) +- Monitor conflicts (Only one monitor instance for both clusters will lead to undefined results) + +**Correct approach:** + +```typescript +// ✅ CORRECT - Unique clusterId for each cluster +const sourceParams = { clusterId: "source-cluster" /* ... */ }; +const destParams = { clusterId: "destination-cluster" /* ... */ }; +``` + +### Always Use Same clusterId for Same Cluster + +Using different `clusterId` values for the same cluster reduces efficiency: + +```typescript +// ⚠️ SUBOPTIMAL - Different clusterIds for same cluster +const params1 = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + clusterId: "my-cluster-1" + // ... +}; + +const params2 = { + host: "my-cluster.us-east-1.rds.amazonaws.com", + clusterId: "my-cluster-2" // Different ID for same cluster + // ... +}; +``` + +**Problems this causes:** + +- Duplication of caches +- Multiple monitoring threads for the same cluster + +**Best practice:** + +```typescript +// ✅ BEST - Same clusterId for same cluster +const CLUSTER_ID = "my-cluster"; +const params1 = { clusterId: CLUSTER_ID /* ... */ }; +const params2 = { clusterId: CLUSTER_ID /* ... */ }; // Shared cache and resources +``` + +## Summary + +The `clusterId` parameter is essential for applications connecting to multiple database clusters. It serves as a cache key for topology information and monitoring services. Always use unique `clusterId` values for different clusters, and consistent values for the same cluster to maximize performance and avoid conflicts. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md index ba213062..f34de548 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheFederatedAuthPlugin.md @@ -48,3 +48,22 @@ Note: AWS IAM database authentication is needed to use the Federated Authenticat | `iamDefaultPort` | `Number` | No | This property overrides the default port that is used to generate the IAM token. The default port is determined based on the underlying driver protocol. Target drivers with different protocols will require users to provide a default port. | `null` | `1234` | | `iamTokenExpiration` | `Number` | No | Overrides the default IAM token cache expiration in seconds. | `900` | `123` | | `httpsAgentOptions` | `Object` | No | This property adds parameters to the httpsAgent that connects to the hosting URL.
For more information on the parameters, see this [documentation](https://nodejs.org/api/https.html#class-httpsagent). | `null` | `{ timeout: 5000 }` | + +## Using Federated Authentication with Global Databases + +When using Federated authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md new file mode 100644 index 00000000..91e1b1f1 --- /dev/null +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbFailoverPlugin.md @@ -0,0 +1,122 @@ +# Global Database (GDB) Failover Plugin + +The AWS Advanced NodeJS Wrapper uses the GDB Failover Plugin to provide minimal downtime in the event of a DB instance failure. The plugin is based on [Failover Plugin v2](./UsingTheFailover2Plugin.md) and unless explicitly stated otherwise, most of the information and suggestions for the [Failover Plugin](./UsingTheFailoverPlugin.md) and [Failover Plugin v2](./UsingTheFailover2Plugin.md) are applicable to the GDB Failover Plugin. + +## Plugin Availability + +The plugin is available since version 3.0.0. + +## Differences between the GDB Failover Plugin and the Failover Plugin v2 + +The GDB Failover Plugin introduces the notion of home region and extends configuration parameters to allow setting different failover logic for **in-home** and **out-of-home** cases. + +Driver configuration for the GDB failover plugin should include a home region defined with an AWS region name. This introduces two cases: + +- **in-home**: when the primary region of the Global Database is the same as the specified home region +- **out-of-home**: when GDB switches over to another region and the primary region is not the same as the specified home region. + +Users are allowed to specify different failover logic individually for **in-home** and **out-of-home** cases. The following are two examples where in-home and out-of-home cases impact the failover logic. + +**Example 1** + +When a user application needs a writer connection, it makes sense to define the failover mode to follow the writer (`strict-writer`). However, some applications may choose not to follow a writer node when cross-region failover occurs (see [Configuration Example 3](#configuration-example-3) below). + +- **in-home**: when in-region failover occurs, the driver reconnects to a new writer node and continues serving the user application with a write connection. +- **out-of-home**: application turns to inactive mode and stops performing writes, this prioritizes performance due to reduced connection latency over being connected to a writer node in another region. + +**Example 2** + +When a user application needs a reader connection, prioritize reader connections in the home region to reduce connection latency (see [Configuration Example 2](#configuration-example-2) below). + +- **in-home**: the primary region of the Global Database is the same as the specified home region, connect to any such reader. +- **out-of-home**: connect to readers that are in the specified home region, these readers are no longer part of the primary region of the Global Database. + +The mentioned scenarios can be configured with the GDB Failover plugin. [Failover2 Plugin](./UsingTheFailover2Plugin.md) and [Failover Plugin](./UsingTheFailoverPlugin.md) don't support the home region and can't be used for the cases mentioned above. + +## Using the GDB Failover Plugin + +The GDB Failover Plugin will not be enabled by default if the [`plugins`](../UsingTheNodejsWrapper.md#connection-plugin-manager-parameters) value is not specified. If you would like to override the default plugins, you can explicitly include the GDB Failover Plugin in your list of plugins by adding the plugin code `gdbFailover` to the [`plugins`](../UsingTheNodejsWrapper.md#aws-advanced-nodejs-wrapper-parameters) value, or by adding it to the current [driver profile](../UsingTheNodejsWrapper.md#connection-plugin-manager-parameters). After you load the plugin, the failover feature will be enabled. + +Please refer to the [failover configuration guide](../FailoverConfigurationGuide.md) for tips to keep in mind when using the failover plugin. + +> [!WARNING] +> Do not use the `gdbFailover`, `failover` and/or `failover2` plugins in any combination at the same time for the same connection! + +Verify plugin compatibility within your driver configuration using the [compatibility guide](../Compatibility.md). + +### GDB Failover Plugin Configuration Parameters + +In addition to the parameters that you can configure for the underlying driver, you can pass the following parameters for the AWS Advanced NodeJS Wrapper through the connection configuration to specify additional failover behavior. + +| Parameter | Value | Required | Description | Default Value | +| ---------------------------------------- | :-------: | :-----------------------------------------------------------------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `failoverHomeRegion` | `String` | If connecting using an IP address, a custom domain URL, Global Database endpoint or other endpoint with no region: Yes

Otherwise: No | Defines a home region.

Examples: `us-west-2`, `us-east-1`.

If this parameter is omitted, the value is parsed from connection url. For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint. If the provided endpoint has no region (for example, a Global Database endpoint or IP address) the configuration parameter is mandatory. | | +| `activeHomeFailoverMode` | `String` | No | Defines a mode for failover process when GDB primary region **is home region**. Failover process may prioritize nodes with different roles and connect to them. Possible values:

- `strict-writer` - Failover process follows writer node and connects to a new writer (in any region) when it changes.
- `home-reader-or-writer` - During failover, the driver tries to connect to any available/accessible reader node in home region. If no reader is available, the driver will connect to a writer node (in any region).
- `strict-home-reader` - During failover, the driver tries to connect to any available reader node in home region. If no reader is available, the driver raises an error.
- `strict-out-of-home-reader`, `strict-any-reader`, `out-of-home-reader-or-writer`, `any-reader-or-writer` - other possible values with prioritizing different writer or reader nodes in home, out of home, or in any region.

If this parameter is omitted, default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | Default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | +| `inactiveHomeFailoverMode` | `String` | No | Defines a mode for failover process when GDB primary region **is not a home region**. Possible values are the same as for `activeHomeFailoverMode` parameter: `strict-writer`, `home-reader-or-writer`, `strict-home-reader`, `strict-out-of-home-reader`, `strict-any-reader`, `out-of-home-reader-or-writer`, `any-reader-or-writer`.
If this parameter is omitted, default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | Default value depends on connection url. For Aurora writer cluster endpoint or Global Database endpoint, it's set to `strict-writer`. Otherwise, it's `home-reader-or-writer`. | +| `clusterInstanceHostPattern` | `String` | If connecting using an IP address or custom domain URL: Yes

Otherwise: No | This parameter is not required unless connecting to an AWS RDS cluster via an IP address or custom domain URL. In those cases, this parameter specifies the cluster instance DNS pattern that will be used to build a complete instance endpoint. A "?" character in this pattern should be used as a placeholder for the DB instance identifiers of the instances in the cluster. See [here](#host-pattern) for more information.

Example: `?.my-domain.com`, `any-subdomain.?.my-domain.com:9999`

Use case Example: If your cluster instance endpoints follow this pattern:`instanceIdentifier1.customHost`, `instanceIdentifier2.customHost`, etc. and you want your initial connection to be to `customHost:1234`, then your configuration should include `clusterInstanceHostPattern: "?.customHost"` with host `customHost` and port `1234`. | If the provided connection string is not an IP address or custom domain, the NodeJS Wrapper will automatically acquire the cluster instance host pattern from the customer-provided connection string. | +| `globalClusterInstanceHostPatterns` | `String` | For Global Databases: Yes

Otherwise: No | This parameter is similar to the `clusterInstanceHostPattern` parameter but it provides a comma-separated list of instance host patterns. This parameter is required for Aurora Global Databases. The list should contain host patterns for each region of the global database. Each host pattern can be based on an RDS instance endpoint or a custom user domain name. If a custom domain name is used, the instance template pattern should be prefixed with the AWS region name in square brackets (`[]`).

The parameter is ignored for other types of databases (Aurora Clusters, RDS Clusters, plain RDS databases, etc.).

Example: for an Aurora Global Database with two AWS regions `us-east-2` and `us-west-2`, the parameter value should be set to `?.XYZ1.us-east-2.rds.amazonaws.com,?.XYZ2.us-west-2.rds.amazonaws.com`. Please note that user identifiers are different for different AWS regions (`XYZ1` and `XYZ2` in the example above).

Example: if using custom domain names, the parameter value should be similar to `[us-east-2]?.customHost,[us-west-2]?.anotherCustomHost`. The port can also be provided: `[us-east-2]?.customHost:8888,[us-west-2]?.anotherCustomHost:9999`

For complete Aurora Global Database configuration, see [Aurora Global Databases](../GlobalDatabases.md). | | +| `clusterTopologyRefreshRateMs` | `Number` | No | Cluster topology refresh rate in milliseconds when a cluster is not in failover. It refers to the regular slow monitoring rate described in [Failover Plugin v2](./UsingTheFailover2Plugin.md). | `30000` | +| `failoverTimeoutMs` | `Number` | No | Maximum allowed time in milliseconds to attempt reconnecting to a new writer or reader instance after a cluster failover is initiated. | `300000` | +| `clusterTopologyHighRefreshRateMs` | `Number` | No | Interval of time in milliseconds to wait between attempts to update cluster topology after the writer has come back online following a failover event. It corresponds to the increased monitoring rate described in [Failover Plugin v2](./UsingTheFailover2Plugin.md). Usually, the topology monitoring component uses this increased monitoring rate for 30s after a new writer was detected. | `100` | +| `failoverReaderHostSelectorStrategy` | `String` | No | Strategy used to select a reader host during failover. For more information on the available reader selection strategies, see this [table](../ReaderSelectionStrategies.md). | `random` | +| `clusterId` | `String` | If connecting to multiple database clusters within a single application:: Yes

Otherwise: No

| A unique identifier for the cluster. Connections with the same cluster id share a cluster topology cache. This parameter is optional and defaults to `1`. When supporting multiple database clusters, this parameter becomes mandatory. Each connection string must include the `clusterId` parameter with a value that can be any number or string. However, all connection strings associated with the same database cluster must use identical `clusterId` values, while connection strings belonging to different database clusters must specify distinct values. Examples of value: `1`, `2`, `1234`, `abc-1`, `abc-2`. | `1` | +| `telemetryFailoverAdditionalTopTrace` | `Boolean` | No | Allows the driver to produce an additional telemetry span associated with failover. Such span helps to facilitate telemetry analysis in AWS CloudWatch. | `false` | +| `skipFailoverOnInterruptedThread` | `Boolean` | No | Enable to skip failover if the current thread is interrupted. This may leave the Connection in an invalid state so the Connection should be disposed. | `false` | +| `skipInactiveWriterClusterEndpointCheck` | `Boolean` | No | Enable to skip a connection role verification when connection is opened with inactive cluster writer endpoint. This parameter is applicable to Aurora Global Databases. Region-bound cluster writer endpoints are available in Global Databases. However, depending on the GDB primary region, they may be inactive and may act differently. The parameter is aimed at supporting such inactive cluster endpoints and helps configure the desired behavior for them. | `false` | + +Please refer to the original [Failover Plugin](./UsingTheFailoverPlugin.md) and [Failover2 Plugin](./UsingTheFailover2Plugin.md) for more details about error codes, configurations, connection pooling and sample codes. + +### Sample Code + +[PostgreSQL Failover Sample Code](./../../../examples/aws_driver_example/aws_failover_postgresql_example.ts) + +This sample code uses the original `failover` plugin, but it can also be used with the `gdbFailover` plugin. Configuration parameters should be adjusted in accordance with the table above. + +### Failover Configuration Examples + +#### Configuration Example 1 + +**Goal:** Provide a user application with a writer connection. The application is deployed in `us-west-1` region and connects to Global Database with `us-east-1`, `us-east-2` and `us-west-1` regions. + +**Solution:** Use the following configuration parameters to configure the driver: + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-writer` +- `inactiveHomeFailoverMode=strict-writer` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use Global Database endpoint in your connection string + +#### Configuration Example 2 + +**Goal:** Provide a user application with a reader connection in `us-west-1`. The application is deployed in the `us-west-1` region and connects to a Global Database with `us-east-1`, `us-east-2` and `us-west-1` regions. If the GDB primary region switches over, prioritize reader connections from the `us-west-1` region. + +**Solution:** Use the following configuration parameters to configure the driver: + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-home-reader` +- `inactiveHomeFailoverMode=strict-home-reader` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use cluster reader endpoint in region `us-west-1` in your connection string (`.cluster-ro-XYZ3.us-west-1.rds.amazonaws.com`) + +#### Configuration Example 3 + +**Goal:** Provide a user application with a writer connection when the primary region is the closest to the application deployed region. When the primary region switches over to another region and network latency becomes unacceptable, the application deployment in region `us-west-1` becomes inactive and lets applications in other regions act and process business transactions. There are application deployments in other regions. The Global Database has `us-east-1`, `us-east-2` and `us-west-1` regions. + +**Solution:** Use the following configuration parameters to configure the driver for application deployment in the `us-west-1` region. Other application deployments need similar driver configuration with different home regions. + +- `failoverHomeRegion=us-west-1` +- `activeHomeFailoverMode=strict-writer` +- `inactiveHomeFailoverMode=strict-any-reader` +- `globalClusterInstanceHostPatterns=?.XYZ1.us-east-1.rds.amazonaws.com,?.XYZ2.us-east-2.rds.amazonaws.com,?.XYZ3.us-west-1.rds.amazonaws.com` (make sure you replace `XYZ1`, `XYZ2`, `XYZ3` with actual values that correspond to your database) +- `wrapperDialect=global-aurora-mysql` (or `global-aurora-pg`) +- `plugins=initialConnection,gdbFailover,efm2` +- use cluster writer endpoint in region `us-west-1` in your connection string (`.cluster-XYZ3.us-west-1.rds.amazonaws.com`) + +**Explanation:** While the primary region of GDB is `us-west-1`, the application deployed in this region needs writable connections. By using the cluster writer endpoint, the driver opens connections to a writer node. If in-region failover occurs, the driver reconnects to a new writer node in the same `us-west-1` region. Since after GDB failover the writer node is in the same region, the driver considers this case as in-home and effectively uses the `activeHomeFailoverMode=strict-writer` setting. Following the writer node for in-region failovers helps the application deployment in the `us-west-1` region to process business transactions with minimal latency. When a cross-region failover event occurs, the driver detects a new writer node, let's assume it's in the us-east-1 region, and uses the `inactiveHomeFailoverMode=strict-any-reader` setting. The driver serves a reader connection from any available region `us-east-1`, `us-east-2` and `us-west-1`. + +> [!WARNING] +> This example covers failover and failover-related settings only. Complete driver configuration may require settings for other plugins. For instance, `initialConnection` may require `endpointSubstitutionRole` and `verifyOpenedConnectionType` parameters to be set properly. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md new file mode 100644 index 00000000..22932ddd --- /dev/null +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheGdbReadWriteSplittingPlugin.md @@ -0,0 +1,57 @@ +# Global Database (GDB) Read/Write Splitting Plugin + +The GDB Read/Write Splitting plugin extends the functionality of the [read/write splitting plugin](./UsingTheReadWriteSplittingPlugin.md) and adopts some additional settings to improve support for Global Databases. + +The GDB Read/Write Splitting plugin adds the notion of a home region and allows users to constrain new connections to this region. Such restrictions may be helpful to prevent opening new connections in environments where remote AWS regions add substantial latency that cannot be tolerated. + +Unless otherwise stated, all recommendations, configurations and code examples made for the [read/write splitting plugin](./UsingTheReadWriteSplittingPlugin.md) are applicable to the current GDB Read/Write Splitting plugin. + +## Plugin Availability + +The plugin is available since version 3.0.0. + +## Loading the Global Database Read/Write Splitting Plugin + +The GDB Read/Write Splitting plugin is not loaded by default. To load the plugin, include it in the `plugins` connection parameter. See the example below to properly load the GDB read/write splitting plugin with these plugins. + +```typescript +const params = { + plugins: "gdbReadWriteSplitting,failover2,efm2" + // Add other connection properties below... +}; + +// If using MySQL: +const client = new AwsMySQLClient(params); +await client.connect(); + +// If using Postgres: +const client = new AwsPGClient(params); +await client.connect(); +``` + +If you would like to use the GDB read/write splitting plugin without the failover plugin, make sure you have the `gdbReadWriteSplitting` plugin in the `plugins` property, and that the failover plugin is not part of it. + +```typescript +const params = { + plugins: "gdbReadWriteSplitting" + // Add other connection properties below... +}; +``` + +> [!WARNING] +> Do not use the `readWriteSplitting` and `gdbReadWriteSplitting` plugins at the same time for the same connection! + +## Using the GDB Read/Write Splitting Plugin against non-GDB clusters + +The GDB Read/Write Splitting plugin can be used against Aurora clusters and RDS clusters. However, since these cluster types are single-region clusters, setting a home region does not make much sense. In these cases, use the original [Read/Write Splitting](./UsingTheReadWriteSplittingPlugin.md) plugin instead. + +## Configuration Parameters + +| Parameter | Value | Required | Description | Default Value | +| --------------------------------- | :----: | :-----------------------------------------------------------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| `readerHostSelectorStrategy` | `str` | No | The name of the strategy that should be used to select a new reader host. For more information on the available reader selection strategies, see this [table](../ReaderSelectionStrategies.md). | `random` | +| `gdbRwHomeRegion` | `str` | If connecting using an IP address, a custom domain URL, Global Database endpoint or other endpoint with no region: Yes

Otherwise: No | Defines a home region.

Examples: `us-west-2`, `us-east-1`.

If this parameter is omitted, the value is parsed from the connection configuration. For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint. If the provided endpoint has no region (for example, a Global Database endpoint or IP address), the configuration parameter is mandatory. | For regional cluster endpoints and instance endpoints, it's set to the region of the provided endpoint.

Otherwise: `null` | +| `gdbRwRestrictWriterToHomeRegion` | `bool` | No | If set to `true`, prevents following and connecting to a writer node outside the defined home region. An exception will be raised when such a connection to a writer outside the home region is requested. | `true` | +| `gdbRwRestrictReaderToHomeRegion` | `bool` | No | If set to `true`, prevents connecting to a reader node outside the defined home region. If no reader nodes in the home region are available, an exception will be raised. | `true` | + +Please refer to the original [Read/Write Splitting plugin](./UsingTheReadWriteSplittingPlugin.md) for more details about error codes, configurations, connection pooling and sample codes. diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md index 055862b2..ee6dafe3 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheIamAuthenticationPlugin.md @@ -76,3 +76,22 @@ You can learn more about `clusterInstanceHostPattern` [here](../UsingTheNodejsWr [IAM Authentication Plugin example for PostgreSQL](../../../examples/aws_driver_example/aws_iam_authentication_postgresql_example.ts)
[IAM Authentication Plugin example for MySQL](../../../examples/aws_driver_example/aws_iam_authentication_mysql_example.ts) + +## Using IAM Authentication with Global Databases + +When using IAM authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` diff --git a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md index 7ece8ba0..9b810ac5 100644 --- a/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md +++ b/docs/using-the-nodejs-wrapper/using-plugins/UsingTheOktaAuthPlugin.md @@ -49,3 +49,22 @@ In the case of AD FS, the user signs into the AD FS sign in page. This generates | `iamDefaultPort` | `Number` | No | This property overrides the default port that is used to generate the IAM token. The default port is determined based on the underlying driver protocol. Target drivers with different protocols will require users to provide a default port. | `null` | `1234` | | `iamTokenExpiration` | `Number` | No | Overrides the default IAM token cache expiration in seconds. | `900` | `123` | | `httpsAgentOptions` | `Object` | No | This property adds parameters to the httpsAgent that connects to the hosting URL.
For more information on the parameters, see this [documentation](https://nodejs.org/api/https.html#new-agentoptions). | `null` | `{ timeout: 5000 }` | + +## Using Okta Authentication with Global Databases + +When using Okta authentication with [Amazon Aurora Global Databases](https://aws.amazon.com/rds/aurora/global-database/), the IAM user or role requires the additional `rds:DescribeGlobalClusters` permission. This permission allows the driver to resolve the Global Database endpoint to the appropriate regional cluster for IAM token generation. + +Example IAM policy: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["rds-db:connect", "rds:DescribeGlobalClusters"], + "Resource": "*" + } + ] +} +``` From d820f400ccb6ec25b910cf2e5fbb67e68d61ce08 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:53:45 -0700 Subject: [PATCH 16/20] fix: add pg_catalog to gdb status queries --- pg/lib/dialect/global_aurora_pg_database_dialect.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pg/lib/dialect/global_aurora_pg_database_dialect.ts b/pg/lib/dialect/global_aurora_pg_database_dialect.ts index d452428e..9bc88611 100644 --- a/pg/lib/dialect/global_aurora_pg_database_dialect.ts +++ b/pg/lib/dialect/global_aurora_pg_database_dialect.ts @@ -24,18 +24,18 @@ import { GlobalAuroraHostListProvider } from "../../../common/lib/host_list_prov import { GlobalTopologyUtils } from "../../../common/lib/host_list_provider/global_topology_utils"; export class GlobalAuroraPgDatabaseDialect extends AuroraPgDatabaseDialect implements GlobalAuroraTopologyDialect { - private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_status'::regproc"; + private static readonly GLOBAL_STATUS_FUNC_EXISTS_QUERY = "select 'pg_catalog.aurora_global_db_status'::regproc"; - private static readonly GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY = "select 'aurora_global_db_instance_status'::regproc"; + private static readonly GLOBAL_INSTANCE_STATUS_FUNC_EXISTS_QUERY = "select 'pg_catalog.aurora_global_db_instance_status'::regproc"; private static readonly GLOBAL_TOPOLOGY_QUERY = "SELECT SERVER_ID, CASE WHEN SESSION_ID = 'MASTER_SESSION_ID' THEN TRUE ELSE FALSE END AS IS_WRITER, " + "VISIBILITY_LAG_IN_MSEC, AWS_REGION " + - "FROM aurora_global_db_instance_status()"; + "FROM pg_catalog.aurora_global_db_instance_status()"; - private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM aurora_global_db_status()"; + private static readonly REGION_COUNT_QUERY = "SELECT count(1) FROM pg_catalog.aurora_global_db_status()"; - private static readonly REGION_BY_INSTANCE_ID_QUERY = "SELECT AWS_REGION FROM aurora_global_db_instance_status() WHERE SERVER_ID = $1"; + private static readonly REGION_BY_INSTANCE_ID_QUERY = "SELECT AWS_REGION FROM pg_catalog.aurora_global_db_instance_status() WHERE SERVER_ID = $1"; async isDialect(targetClient: ClientWrapper): Promise { try { From a5ab0b9f0ccc269895f1db3456caf0145582d835 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 15 Jun 2026 18:47:21 -0700 Subject: [PATCH 17/20] chore: remove aliases --- common/lib/host_info.ts | 39 +---- .../lib/host_list_provider/topology_utils.ts | 1 - common/lib/partial_plugin_service.ts | 42 +----- common/lib/plugin_service.ts | 47 ++---- ...rora_initial_connection_strategy_plugin.ts | 20 +-- .../routing/substitute_connect_routing.ts | 1 - .../aurora_connection_tracker_plugin.ts | 13 +- .../opened_connection_tracker.ts | 25 ++-- common/lib/plugins/default_plugin.ts | 2 +- .../efm/host_monitoring_connection_plugin.ts | 27 ++-- .../lib/plugins/failover/failover_plugin.ts | 13 +- .../failover/reader_failover_handler.ts | 6 +- .../failover/writer_failover_handler.ts | 18 ++- .../lib/plugins/failover2/failover2_plugin.ts | 6 +- .../limitless/limitless_connection_context.ts | 11 ++ .../limitless/limitless_connection_plugin.ts | 3 + .../limitless/limitless_router_service.ts | 16 +- common/lib/utils/full_services_container.ts | 3 + common/lib/utils/host_id_cache_service.ts | 137 ++++++++++++++++++ common/lib/utils/messages.ts | 1 + common/lib/utils/service_utils.ts | 3 + mysql/lib/client.ts | 5 +- pg/lib/client.ts | 5 +- 23 files changed, 261 insertions(+), 183 deletions(-) create mode 100644 common/lib/utils/host_id_cache_service.ts diff --git a/common/lib/host_info.ts b/common/lib/host_info.ts index 3ac57751..4d332a38 100644 --- a/common/lib/host_info.ts +++ b/common/lib/host_info.ts @@ -25,15 +25,13 @@ export class HostInfo { public static readonly NO_PORT: number = -1; public static readonly DEFAULT_WEIGHT: number = 100; - readonly host: string; + readonly host: string; // full domain name readonly port: number; role: HostRole; readonly weight: number; // Greater or equal 0. Lesser the weight, the healthier host. readonly lastUpdateTime: number; availability: HostAvailability; - aliases: Set = new Set(); - allAliases: Set = new Set(); - hostId: string; + hostId: string; // id; could be a host name, host domain name, or a unique string hostAvailabilityStrategy: HostAvailabilityStrategy; constructor( @@ -57,7 +55,6 @@ export class HostInfo { this.weight = weight; this.lastUpdateTime = lastUpdateTime; this.hostAvailabilityStrategy = hostAvailabilityStrategy; - this.allAliases.add(this.asAlias).add(this.host); this.hostId = hostId; } @@ -65,38 +62,6 @@ export class HostInfo { return this.port != HostInfo.NO_PORT; } - addAlias(...alias: string[]) { - if (!alias || alias.length < 1) { - return; - } - - alias.forEach((x) => { - this.aliases.add(x); - this.allAliases.add(x); - }); - } - - removeAlias(aliases: string[]) { - if (!aliases || aliases.length < 1) { - return; - } - - aliases.forEach((x) => { - this.aliases.delete(x); - this.allAliases.delete(x); - }); - } - - resetAliases() { - this.aliases.clear(); - this.allAliases.clear(); - this.allAliases.add(this.asAlias); - } - - get asAlias() { - return this.isPortSpecified() ? `${this.host}:${this.port}` : this.host; - } - get url() { let url = this.hostAndPort; if (!url.endsWith("/")) { diff --git a/common/lib/host_list_provider/topology_utils.ts b/common/lib/host_list_provider/topology_utils.ts index 500f3046..3cddf486 100644 --- a/common/lib/host_list_provider/topology_utils.ts +++ b/common/lib/host_list_provider/topology_utils.ts @@ -130,7 +130,6 @@ export abstract class TopologyUtils { .withLastUpdateTime(lastUpdateTime) .withHostId(finalInstanceId) .build(); - host.addAlias(finalEndpoint); return host; } diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts index b3b8f729..11079e35 100644 --- a/common/lib/partial_plugin_service.ts +++ b/common/lib/partial_plugin_service.ts @@ -221,15 +221,11 @@ export class PartialPluginService implements PluginService, HostListProviderServ return hosts; } - setAvailability(hostAliases: Set, availability: HostAvailability): void { - if (hostAliases.size === 0) { - return; - } - + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void { const hostsToChange = [ ...new Set( this.getAllHosts().filter( - (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) ) ) ]; @@ -430,36 +426,12 @@ export class PartialPluginService implements PluginService, HostListProviderServ return provider.identifyConnection(targetClient); } - async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise { - if (!hostInfo) { - return; - } - - if (hostInfo.aliases.size > 0) { - logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); - return; - } - - hostInfo.addAlias(hostInfo.asAlias); - - try { - const res = await this.dialect.getHostAliasAndParseResults(targetClient); - if (res) { - hostInfo.addAlias(res); - } - } catch (error) { - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + getRoutedHostInfo(): HostInfo | null { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "getRoutedHostInfo")); + } - try { - const host = await this.identifyConnection(targetClient); - if (host && host.allAliases) { - hostInfo.addAlias(...host.allAliases); - } - } catch (error) { - // Ignore errors from identifyConnection - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + throw new AwsWrapperError(Messages.get("PartialPluginService.unexpectedMethodCall", "setRoutedHostInfo")); } getHostInfoBuilder(): HostInfoBuilder { diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 8e8a5a1a..0d235355 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -96,14 +96,20 @@ export interface PluginService extends ErrorHandler { getHosts(): HostInfo[]; - setAvailability(hostAliases: Set, availability: HostAvailability): void; + setAvailability(hostInfo: HostInfo, availability: HostAvailability): void; updateConfigWithProperties(props: Map): void; - fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo): Promise; + getRoutedHostInfo(): HostInfo | null; + + setRoutedHostInfo(routedHostInfo: HostInfo | null); identifyConnection(targetClient: ClientWrapper): Promise; + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo | null): Promise; + + identifyConnection(targetClient: ClientWrapper, connectionHostInfo?: HostInfo | null): Promise; + connect(hostInfo: HostInfo, props: Map): Promise; connect(hostInfo: HostInfo, props: Map, pluginToSkip: ConnectionPlugin | null): Promise; @@ -162,6 +168,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private _isInTransaction: boolean = false; private servicesContainer: FullServicesContainer; protected hosts: HostInfo[] = []; + protected routedHostInfo: HostInfo | null; private dbDialectProvider: DatabaseDialectProvider; private readonly initialHost: string; private dialect: DatabaseDialect; @@ -459,15 +466,11 @@ export class PluginServiceImpl implements PluginService, HostListProviderService return hosts; } - setAvailability(hostAliases: Set, availability: HostAvailability) { - if (hostAliases.size === 0) { - return; - } - + setAvailability(hostInfo: HostInfo, availability: HostAvailability) { const hostsToChange = [ ...new Set( this.getAllHosts().filter( - (host: HostInfo) => hostAliases.has(host.asAlias) || [...host.aliases].some((hostAlias: string) => hostAliases.has(hostAlias)) + (host: HostInfo) => (hostInfo.hostId != null && hostInfo.hostId === host.hostId) || (hostInfo.host != null && hostInfo.host === host.host) ) ) ]; @@ -496,30 +499,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService this._currentClient.config = Object.fromEntries(props.entries()); } - async fillAliases(targetClient: ClientWrapper, hostInfo: HostInfo) { - if (hostInfo == null) { - return; - } - - if (hostInfo.aliases.size > 0) { - logger.debug(Messages.get("PluginService.nonEmptyAliases", [...hostInfo.aliases].join(", "))); - return; - } - - hostInfo.addAlias(hostInfo.asAlias); - - // Add the host name and port, this host name is usually the internal IP address. - try { - const res: string = await this.dialect.getHostAliasAndParseResults(targetClient); - hostInfo.addAlias(res); - } catch (error) { - logger.debug(Messages.get("PluginService.failedToRetrieveHostPort")); - } + getRoutedHostInfo(): HostInfo | null { + return this.routedHostInfo; + } - const host: HostInfo | void | null = await this.identifyConnection(targetClient); - if (host) { - hostInfo.addAlias(...host.allAliases); - } + setRoutedHostInfo(routedHostInfo: HostInfo | null) { + this.routedHostInfo = routedHostInfo; } identifyConnection(targetClient: ClientWrapper): Promise { diff --git a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts index 7d7891a6..cdabc7d1 100644 --- a/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts +++ b/common/lib/plugins/aurora_initial_connection_strategy_plugin.ts @@ -121,9 +121,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu continue; } - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); } return writerCandidateClient; } @@ -139,16 +137,14 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu } // Writer connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(writerCandidate); - } + this.pluginService.setRoutedHostInfo(writerCandidate); return writerCandidateClient; } catch (error: any) { await this.pluginService.abortTargetClient(writerCandidateClient); if (this.pluginService.isLoginError(error) || !writerCandidate) { throw error; } else if (writerCandidate) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); } } } @@ -185,9 +181,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.hasNoReaders()) { // It seems that cluster has no readers. Simulate Aurora reader cluster endpoint logic // and return the current (writer) client. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); return readerCandidateClient; } await this.pluginService.abortTargetClient(readerCandidateClient); @@ -196,9 +190,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu } // Reader connection is valid and verified. - if (isInitialConnection) { - this.hostListProviderService.setInitialConnectionHostInfo(readerCandidate); - } + this.pluginService.setRoutedHostInfo(readerCandidate); } else { logger.debug("Reader candidate not found"); } @@ -233,7 +225,7 @@ export class AuroraInitialConnectionStrategyPlugin extends AbstractConnectionPlu if (this.pluginService.isLoginError(error) || !readerCandidate) { throw error; } else if (readerCandidate) { - this.pluginService.setAvailability(readerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(readerCandidate, HostAvailability.NOT_AVAILABLE); } } } diff --git a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts index 94f35677..812873d7 100644 --- a/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts +++ b/common/lib/plugins/bluegreen/routing/substitute_connect_routing.ts @@ -73,7 +73,6 @@ export class SubstituteConnectRouting extends BaseConnectRouting { .withHostId(iamHost.hostId) .withAvailability(HostAvailability.AVAILABLE) .build(); - reroutedHostInfo.addAlias(iamHost.host); const reroutedProperties: Map = new Map(properties); reroutedProperties.set(WrapperProperties.HOST.name, iamHost.host); diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts index 651bbea8..9b70960e 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts @@ -16,7 +16,6 @@ import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; import { PluginService } from "../../plugin_service"; import { RdsUtils } from "../../utils/rds_utils"; import { HostInfo } from "../../host_info"; @@ -56,14 +55,18 @@ export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin impl connectFunc: () => Promise ): Promise { const targetClient = await connectFunc(); + let connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; if (targetClient) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + connectionHostInfo = identifiedHostInfo; + await this.pluginService.setRoutedHostInfo(connectionHostInfo); + } } - await this.tracker.populateOpenedConnectionQueue(hostInfo, targetClient); + await this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); } return targetClient; } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 6c8b6a58..8514bf97 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -32,7 +32,9 @@ export class OpenedConnectionTracker { } async populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): Promise { - const aliases = hostInfo.aliases; + if (!hostInfo || !client) { + return; + } // Check if the connection was established using an instance endpoint if (OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host)) { @@ -40,21 +42,20 @@ export class OpenedConnectionTracker { return; } - const instanceEndpoint = [...aliases] - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .reduce((max, s) => (s > max ? s : max), ""); - - if (!instanceEndpoint) { - logger.debug(Messages.get("OpenedConnectionTracker.unableToPopulateOpenedConnectionQueue", hostInfo.host)); - return; + // It might be a custom domain name. Let's track by hostId and custom domain name + if (hostInfo.hostId) { + this.trackConnection(hostInfo.hostId, client); + } + if (hostInfo.hostAndPort) { + this.trackConnection(hostInfo.hostAndPort, client); } - - this.trackConnection(instanceEndpoint, client); } async invalidateAllConnections(hostInfo: HostInfo): Promise { - await this.invalidateAllConnectionsMultipleHosts(hostInfo.asAlias); - await this.invalidateAllConnectionsMultipleHosts(...Array.from(hostInfo.aliases)); + if (!hostInfo) { + return; + } + await this.invalidateAllConnectionsMultipleHosts(hostInfo.hostAndPort, hostInfo.host, hostInfo.hostId); } async invalidateAllConnectionsMultipleHosts(...hosts: string[]): Promise { diff --git a/common/lib/plugins/default_plugin.ts b/common/lib/plugins/default_plugin.ts index 2c3b45d8..ecd5ae0a 100644 --- a/common/lib/plugins/default_plugin.ts +++ b/common/lib/plugins/default_plugin.ts @@ -84,7 +84,7 @@ export class DefaultPlugin extends AbstractConnectionPlugin { ); const result: ConnectionInfo = await telemetryContext.start(async () => await connProvider.connect(hostInfo, this.pluginService, props)); - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(hostInfo, HostAvailability.AVAILABLE); this.pluginService.setIsPooledClient(result.isPooled); await this.pluginService.updateDialect(result.client); return result.client; diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts index 753e2f69..59ad986f 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/host_monitoring_connection_plugin.ts @@ -59,10 +59,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp ): Promise { const targetClient = await connectFunc(); if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); + const connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; + const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); + const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); + if (identifiedHostInfo) { + this.pluginService.setRoutedHostInfo(identifiedHostInfo); + } } } return targetClient; @@ -110,10 +113,6 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (monitorContext.isHostUnhealthy) { const monitoringHostInfo = await this.getMonitoringHostInfo(); - if (monitoringHostInfo) { - this.pluginService.setAvailability(monitoringHostInfo.allAliases, HostAvailability.NOT_AVAILABLE); - } - const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; if (targetClient) { @@ -125,7 +124,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp await this.pluginService.abortTargetClient(targetClient); } // eslint-disable-next-line no-unsafe-finally - throw new UnavailableHostError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo.host)); + throw new UnavailableHostError( + Messages.get("HostMonitoringConnectionPlugin.unavailableHost", monitoringHostInfo?.host ?? "Unknown host") + ); } } } @@ -149,7 +150,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp if (this.monitoringHostInfo) { return this.monitoringHostInfo; } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); + this.monitoringHostInfo = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getCurrentHostInfo(); if (this.monitoringHostInfo == null) { this.throwUnableToIdentifyConnection(null); } @@ -158,12 +159,16 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp try { if (rdsUrlType.isRdsCluster) { logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); + this.monitoringHostInfo = await this.pluginService.identifyConnection( + this.pluginService.getCurrentClient().targetClient!, + this.pluginService.getCurrentHostInfo() + ); if (this.monitoringHostInfo == null) { const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); this.throwUnableToIdentifyConnection(host); } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); + // Update identified HostInfo for the current connection + await this.pluginService.setCurrentClient(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); } } catch (error: any) { if (!(error instanceof AwsWrapperError)) { diff --git a/common/lib/plugins/failover/failover_plugin.ts b/common/lib/plugins/failover/failover_plugin.ts index ceefae36..73014d2d 100644 --- a/common/lib/plugins/failover/failover_plugin.ts +++ b/common/lib/plugins/failover/failover_plugin.ts @@ -184,12 +184,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { if (this.isHostStillValid(url, changes)) { return Promise.resolve(); } - - for (const alias of currentHost.allAliases) { - if (this.isHostStillValid(alias + "/", changes)) { - return Promise.resolve(); - } - } } logger.info(Messages.get("Failover.invalidHost", currentHost?.host ?? "empty host")); return Promise.resolve(); @@ -279,7 +273,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.invalidateCurrentClient(); const currentHostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } this._lastError = e; @@ -291,7 +285,7 @@ export class FailoverPlugin extends AbstractConnectionPlugin { } async failover(failedHost: HostInfo) { - this.pluginService.setAvailability(failedHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(failedHost, HostAvailability.NOT_AVAILABLE); if (this.failoverMode === FailoverMode.STRICT_WRITER) { await this.failoverWriter(); @@ -308,8 +302,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { const telemetryContext = telemetryFactory.openTelemetryContext(FailoverPlugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); - let failedHost = null; if (failedHostInfo && failedHostInfo.getRawAvailability() === HostAvailability.AVAILABLE) { failedHost = failedHostInfo; @@ -334,7 +326,6 @@ export class FailoverPlugin extends AbstractConnectionPlugin { await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.updateTopology(true); this.failoverWriterSuccessCounter.inc(); } catch (error: any) { diff --git a/common/lib/plugins/failover/reader_failover_handler.ts b/common/lib/plugins/failover/reader_failover_handler.ts index 21c82b1e..fddf12a6 100644 --- a/common/lib/plugins/failover/reader_failover_handler.ts +++ b/common/lib/plugins/failover/reader_failover_handler.ts @@ -141,7 +141,7 @@ export class ClusterAwareReaderFailoverHandler implements ReaderFailoverHandler async failoverInternal(hosts: HostInfo[], currentHost: HostInfo | null): Promise { if (currentHost) { - this.pluginService.setAvailability(currentHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHost, HostAvailability.NOT_AVAILABLE); } const hostsByPriority = this.getHostsByPriority(hosts); return this.getConnectionFromHostGroup(hostsByPriority); @@ -336,7 +336,7 @@ class ConnectionAttemptTask { try { this.targetClient = await this.pluginService.forceConnect(this.newHost, copy); - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.AVAILABLE); logger.info(Messages.get("ClusterAwareReaderFailoverHandler.successfulReaderConnection", this.newHost.host)); if (this.taskHandler.getSelectedConnectionAttemptTask(this.failoverTaskId) === -1) { this.taskHandler.setSelectedConnectionAttemptTask(this.failoverTaskId, this.taskId); @@ -344,7 +344,7 @@ class ConnectionAttemptTask { } throw new AwsWrapperError(Messages.get("ClusterAwareReaderFailoverHandler.selectedTaskChosen", this.newHost.host)); } catch (error) { - this.pluginService.setAvailability(this.newHost.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(this.newHost, HostAvailability.NOT_AVAILABLE); throw error; } finally { await this.performFinalCleanup(); diff --git a/common/lib/plugins/failover/writer_failover_handler.ts b/common/lib/plugins/failover/writer_failover_handler.ts index e14a73fd..dbc2320d 100644 --- a/common/lib/plugins/failover/writer_failover_handler.ts +++ b/common/lib/plugins/failover/writer_failover_handler.ts @@ -37,12 +37,14 @@ export interface WriterFailoverHandler { function isCurrentHostWriter(topology: HostInfo[], originalWriterHost: HostInfo): boolean { const latestWriter = getWriter(topology); - const latestWriterAllAliases = latestWriter?.allAliases; - const currentAliases = originalWriterHost.allAliases; - if (currentAliases && latestWriterAllAliases) { - return [...currentAliases].filter((alias) => latestWriterAllAliases.has(alias)).length > 0; + if (!latestWriter || !originalWriterHost) { + return false; } - return false; + + return ( + (!!latestWriter.hostId && latestWriter.hostId === originalWriterHost.hostId) || + (!!latestWriter.hostAndPort && latestWriter.hostAndPort.toLowerCase() === originalWriterHost.hostAndPort.toLowerCase()) + ); } export class ClusterAwareWriterFailoverHandler implements WriterFailoverHandler { @@ -273,7 +275,7 @@ class ReconnectToWriterHandlerTask { } success = isCurrentHostWriter(latestTopology, this.originalWriterHost); - this.pluginService.setAvailability(this.originalWriterHost.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(this.originalWriterHost, HostAvailability.AVAILABLE); return new WriterFailoverResult( success, false, @@ -454,13 +456,13 @@ class WaitForNewWriterHandlerTask { let targetClient = null; try { targetClient = await this.pluginService.forceConnect(writerCandidate, props); - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.AVAILABLE); await this.callCloseClient(this.currentReaderTargetClient); await this.callCloseClient(this.currentClient); this.currentClient = targetClient; return true; } catch (error) { - this.pluginService.setAvailability(writerCandidate.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(writerCandidate, HostAvailability.NOT_AVAILABLE); await this.pluginService.abortTargetClient(targetClient); return false; } diff --git a/common/lib/plugins/failover2/failover2_plugin.ts b/common/lib/plugins/failover2/failover2_plugin.ts index f9e04fe6..85288c65 100644 --- a/common/lib/plugins/failover2/failover2_plugin.ts +++ b/common/lib/plugins/failover2/failover2_plugin.ts @@ -150,7 +150,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin { throw error; } - this.pluginService.setAvailability(hostInfo.allAliases, HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(hostInfo, HostAvailability.NOT_AVAILABLE); try { // Unable to directly connect, attempt failover. @@ -209,7 +209,7 @@ export class Failover2Plugin extends AbstractConnectionPlugin { await this.invalidateCurrentClient(); const currentHostInfo: HostInfo = this.pluginService.getCurrentHostInfo(); if (currentHostInfo !== null) { - this.pluginService.setAvailability(currentHostInfo.allAliases ?? new Set(), HostAvailability.NOT_AVAILABLE); + this.pluginService.setAvailability(currentHostInfo, HostAvailability.NOT_AVAILABLE); } await this.failover(); this._lastError = error; @@ -247,7 +247,6 @@ export class Failover2Plugin extends AbstractConnectionPlugin { const telemetryContext = telemetryFactory.openTelemetryContext(Failover2Plugin.TELEMETRY_READER_FAILOVER, TelemetryTraceLevel.NESTED); this.failoverReaderTriggeredCounter.inc(); - const oldAliases = this.pluginService.getCurrentHostInfo()?.allAliases ?? new Set(); const failoverEndTimeMs = Date.now() + this.failoverTimeoutSettingMs; try { @@ -265,7 +264,6 @@ export class Failover2Plugin extends AbstractConnectionPlugin { this.failoverReaderSuccessCounter.inc(); await this.pluginService.abortCurrentClient(); await this.pluginService.setCurrentClient(result.client, result.newHost); - this.pluginService.getCurrentHostInfo()?.removeAlias(Array.from(oldAliases)); await this.pluginService.forceRefreshHostList(); } catch (error) { this.failoverReaderFailedCounter.inc(); diff --git a/common/lib/plugins/limitless/limitless_connection_context.ts b/common/lib/plugins/limitless/limitless_connection_context.ts index ccc244c9..5891c2da 100644 --- a/common/lib/plugins/limitless/limitless_connection_context.ts +++ b/common/lib/plugins/limitless/limitless_connection_context.ts @@ -17,6 +17,7 @@ import { HostInfo } from "../../host_info"; import { ClientWrapper } from "../../client_wrapper"; import { ConnectionPlugin } from "../../connection_plugin"; +import Map from "@arrows/array/src/map"; export class LimitlessConnectionContext { private readonly hostInfo: HostInfo; @@ -25,6 +26,7 @@ export class LimitlessConnectionContext { private readonly connectFunc: () => Promise; private routers: HostInfo[] | null; private plugin: ConnectionPlugin; + private connectionHostInfo: HostInfo | null; constructor( hostInfo: HostInfo, @@ -40,6 +42,7 @@ export class LimitlessConnectionContext { this.connectFunc = connectFunc; this.routers = routers; this.plugin = plugin; + this.connectionHostInfo = null; } public getHostInfo(): HostInfo { @@ -58,6 +61,14 @@ export class LimitlessConnectionContext { return this.connectFunc; } + getConnectionHostInfo(): HostInfo | null { + return this.connectionHostInfo; + } + + setConnectionHostInfo(value: HostInfo | null) { + this.connectionHostInfo = value; + } + public getRouters(): HostInfo[] | null { return this.routers; } diff --git a/common/lib/plugins/limitless/limitless_connection_plugin.ts b/common/lib/plugins/limitless/limitless_connection_plugin.ts index 775c6cbd..768674ae 100644 --- a/common/lib/plugins/limitless/limitless_connection_plugin.ts +++ b/common/lib/plugins/limitless/limitless_connection_plugin.ts @@ -65,6 +65,9 @@ export class LimitlessConnectionPlugin extends AbstractConnectionPlugin { await this.limitlessRouterService.establishConnection(context); if (context.getConnection() != null) { + if (context.getConnectionHostInfo() != null) { + this.pluginService.setRoutedHostInfo(context.getConnectionHostInfo()); + } return context.getConnection(); } throw new AwsWrapperError(Messages.get("LimitlessConnectionPlugin.failedToConnectToHost", hostInfo.host)); diff --git a/common/lib/plugins/limitless/limitless_router_service.ts b/common/lib/plugins/limitless/limitless_router_service.ts index b33569c2..46850838 100644 --- a/common/lib/plugins/limitless/limitless_router_service.ts +++ b/common/lib/plugins/limitless/limitless_router_service.ts @@ -118,6 +118,7 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { try { context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnectionHostInfo(selectedHostSpec); } catch (e) { logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); @@ -161,12 +162,12 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } } - let selectedHostSpec: HostInfo | undefined = undefined; + let selectedHostInfo: HostInfo | undefined = undefined; try { // Select healthiest router for best chance of connection over load-balancing with round-robin - selectedHostSpec = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); - logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostSpec ? selectedHostSpec.host : "undefined")); - if (!selectedHostSpec) { + selectedHostInfo = this.pluginService.getHostInfoByStrategy(HostRole.WRITER, HighestWeightHostSelector.STRATEGY_NAME, context.getRouters()); + logger.debug(Messages.get("LimitlessRouterServiceImpl.selectedHostForRetry", selectedHostInfo ? selectedHostInfo.host : "undefined")); + if (!selectedHostInfo) { continue; } } catch (e) { @@ -179,13 +180,14 @@ export class LimitlessRouterServiceImpl implements LimitlessRouterService { } try { - context.setConnection(await this.pluginService.connect(selectedHostSpec, context.getProperties(), context.getPlugin())); + context.setConnection(await this.pluginService.connect(selectedHostInfo, context.getProperties(), context.getPlugin())); if (context.getConnection()) { + context.setConnectionHostInfo(selectedHostInfo); return; } } catch (error) { - selectedHostSpec.setAvailability(HostAvailability.NOT_AVAILABLE); - logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostSpec.host)); + selectedHostInfo.setAvailability(HostAvailability.NOT_AVAILABLE); + logger.debug(Messages.get("LimitlessRouterServiceImpl.failedToConnectToHost", selectedHostInfo.host)); } } throw new AwsWrapperError(Messages.get("LimitlessRouterServiceImpl.maxRetriesExceeded")); diff --git a/common/lib/utils/full_services_container.ts b/common/lib/utils/full_services_container.ts index 59902f88..c1be9827 100644 --- a/common/lib/utils/full_services_container.ts +++ b/common/lib/utils/full_services_container.ts @@ -23,6 +23,7 @@ import { StorageService } from "./storage/storage_service"; import { MonitorService } from "./monitoring/monitor_service"; import { EventPublisher } from "./events/event"; import { ImportantEventService } from "./important_event_service"; +import { HostIdCacheService } from "./host_id_cache_service"; /** * Container for services used throughout the wrapper. @@ -37,6 +38,7 @@ export interface FullServicesContainer { hostListProviderService: HostListProviderService; pluginService: PluginService; importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; } export class FullServicesContainerImpl implements FullServicesContainer { @@ -49,6 +51,7 @@ export class FullServicesContainerImpl implements FullServicesContainer { hostListProviderService!: HostListProviderService; pluginService!: PluginService; importantEventService: ImportantEventService; + hostIdCacheService: HostIdCacheService; constructor( storageService: StorageService, diff --git a/common/lib/utils/host_id_cache_service.ts b/common/lib/utils/host_id_cache_service.ts new file mode 100644 index 00000000..9ac7e32c --- /dev/null +++ b/common/lib/utils/host_id_cache_service.ts @@ -0,0 +1,137 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../client_wrapper"; +import { HostInfo } from "../host_info"; +import { PluginService } from "../plugin_service"; +import { DatabaseDialect } from "../database_dialect/database_dialect"; +import { TopologyAwareDatabaseDialect } from "../database_dialect/topology_aware_database_dialect"; +import { RdsUtils } from "./rds_utils"; +import { RdsUrlType } from "./rds_url_type"; +import { AwsWrapperError } from "./errors"; +import { Messages } from "./messages"; + +/** + * Tuple of (instanceId, instanceName) identifying a connection's underlying instance, as + * returned by a topology-aware dialect. Either value may be null when identification fails. + */ +export type InstanceIdAndName = [instanceId: string | null, instanceName: string | null]; + +/** + * Identifies the underlying instance a connection is established to. + * + * For static endpoints (IP addresses or custom domain names) the identification result is cached, + * keyed by the connection's host name, to avoid repeatedly querying the database. + */ +export interface HostIdCacheService { + /** + * Identify the connected host, using the cache where possible. + * + * @param targetClient the connection to be identified. + * @param connectionHostInfo the {@link HostInfo} the connection was established with. + * @param pluginService the plugin service instance. + * @returns the identified {@link HostInfo}, or null if it cannot be determined. + */ + identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise; +} + +function isTopologyAwareDialect(dialect: DatabaseDialect): dialect is DatabaseDialect & TopologyAwareDatabaseDialect { + return typeof (dialect as Partial).getInstanceId === "function"; +} + +export class HostIdCacheServiceImpl implements HostIdCacheService { + static readonly PROP_ENABLED = "AWS_NODEJS_HOST_CACHE_ENABLED"; + static readonly PROP_REGEXP = "AWS_NODEJS_HOST_CACHE_REGEXP"; + + private static readonly cache = new Map(); + private static readonly isEnabled = (process.env[HostIdCacheServiceImpl.PROP_ENABLED] ?? "true").toLowerCase() === "true"; + private static readonly hostRegexp = new RegExp(process.env[HostIdCacheServiceImpl.PROP_REGEXP] ?? ".*"); + private static readonly rdsHelper = new RdsUtils(); + + async identifyConnection(targetClient: ClientWrapper, connectionHostInfo: HostInfo, pluginService: PluginService): Promise { + if (!targetClient || !connectionHostInfo || !pluginService) { + return null; + } + + const urlType: RdsUrlType = HostIdCacheServiceImpl.rdsHelper.identifyRdsType(connectionHostInfo.host); + switch (urlType) { + case RdsUrlType.RDS_INSTANCE: + return connectionHostInfo; + case RdsUrlType.IP_ADDRESS: + case RdsUrlType.OTHER: + // It might be a custom domain name. Cache the identification keyed by host name when allowed. + if (HostIdCacheServiceImpl.isEnabled && HostIdCacheServiceImpl.hostRegexp.test(connectionHostInfo.host)) { + return this.getCachedHostInfo(targetClient, connectionHostInfo, pluginService); + } + return pluginService.identifyConnection(targetClient); + default: + // Other hosts are dynamic and may change at any time, so they can't be cached. + return pluginService.identifyConnection(targetClient); + } + } + + protected async getCachedHostInfo( + targetClient: ClientWrapper, + connectionHostInfo: HostInfo, + pluginService: PluginService + ): Promise { + const host = connectionHostInfo.host; + + let instanceIdAndName = HostIdCacheServiceImpl.cache.get(host); + if (!instanceIdAndName) { + instanceIdAndName = await this.queryInstanceIdAndName(targetClient, pluginService); + HostIdCacheServiceImpl.cache.set(host, instanceIdAndName); + } + + const [instanceId, instanceName] = instanceIdAndName; + if (!instanceId && !instanceName) { + // We've already tried to identify the connection, but got nothing. + return null; + } + + let topology = pluginService.getAllHosts(); + if (!topology || topology.length === 0) { + const provider = pluginService.getHostListProvider(); + topology = provider ? await provider.forceRefresh() : null; + if (!topology || topology.length === 0) { + return null; + } + } + + return topology.find((candidate) => instanceId === candidate.hostId || instanceName === candidate.host) ?? null; + } + + private async queryInstanceIdAndName(targetClient: ClientWrapper, pluginService: PluginService): Promise { + const dialect = pluginService.getDialect(); + if (!isTopologyAwareDialect(dialect)) { + return [null, null]; + } + + try { + const [instanceId, instanceName] = await dialect.getInstanceId(targetClient); + return [instanceId ?? null, instanceName ?? null]; + } catch (error: any) { + throw new AwsWrapperError(Messages.get("HostIdCacheService.errorIdentifyConnection"), error); + } + } + + /** + * Clears the static host identification cache. Intended for test cleanup. + */ + static clearCache(): void { + HostIdCacheServiceImpl.cache.clear(); + } +} diff --git a/common/lib/utils/messages.ts b/common/lib/utils/messages.ts index 32d9b9df..94e619b9 100644 --- a/common/lib/utils/messages.ts +++ b/common/lib/utils/messages.ts @@ -59,6 +59,7 @@ const MESSAGES: Record = { "RdsHostListProvider.noClusterId": "No clusterId found. Please ensure clusterId parameter is set to a non-empty string.", "ConnectionStringHostListProvider.parsedListEmpty": "Can't parse connection string: '%s'.", "ConnectionStringHostListProvider.errorIdentifyConnection": "An error occurred while obtaining the connection's host ID.", + "HostIdCacheService.errorIdentifyConnection": "An error occurred while identifying the connection's host ID.", "ExecuteTimePlugin.executeTime": "Executed method '%s' in %s milliseconds.", "ConnectTimePlugin.connectTime": "Connected to '%s' in %s milliseconds.", "ClusterAwareWriterFailoverHandler.failoverCalledWithInvalidTopology": "Failover was called with an invalid (null or empty) topology.", diff --git a/common/lib/utils/service_utils.ts b/common/lib/utils/service_utils.ts index b7bda542..35c98b38 100644 --- a/common/lib/utils/service_utils.ts +++ b/common/lib/utils/service_utils.ts @@ -31,6 +31,7 @@ import { TelemetryFactory } from "./telemetry/telemetry_factory"; import { EventPublisher } from "./events/event"; import { PartialPluginService } from "../partial_plugin_service"; import { ConnectionUrlParser } from "./connection_url_parser"; +import { HostIdCacheServiceImpl } from "./host_id_cache_service"; export class ServiceUtils { private static readonly _instance: ServiceUtils = new ServiceUtils(); @@ -70,6 +71,7 @@ export class ServiceUtils { servicesContainer.pluginService = pluginService; servicesContainer.pluginManager = pluginManager; servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); return servicesContainer; } @@ -104,6 +106,7 @@ export class ServiceUtils { servicesContainer.pluginService = pluginService; servicesContainer.pluginManager = pluginManager; servicesContainer.hostListProviderService = pluginService; + servicesContainer.hostIdCacheService = new HostIdCacheServiceImpl(); return servicesContainer; } diff --git a/mysql/lib/client.ts b/mysql/lib/client.ts index 45b9bed7..2048848c 100644 --- a/mysql/lib/client.ts +++ b/mysql/lib/client.ts @@ -28,6 +28,7 @@ import { AwsWrapperError, ConnectionProvider, FailoverSuccessError, + HostInfo, InternalPooledConnectionProvider, TransactionIsolationLevel, UndefinedClientError, @@ -86,7 +87,9 @@ class BaseAwsMySQLClient extends AwsClient implements MySQLClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } diff --git a/pg/lib/client.ts b/pg/lib/client.ts index b8c55464..bb0b3d9f 100644 --- a/pg/lib/client.ts +++ b/pg/lib/client.ts @@ -215,7 +215,10 @@ class BaseAwsPgClient extends AwsClient implements PGClient { // Ignore } } - await this.pluginService.setCurrentClient(result, result.hostInfo); + + const connectedHostInfo: HostInfo | null = this.pluginService.getRoutedHostInfo() ?? this.pluginService.getInitialConnectionHostInfo(); + await this.pluginService.setCurrentClient(result, connectedHostInfo); + this.pluginService.setRoutedHostInfo(null); await this.internalPostConnect(); }); } From d7fb39579de5876bd9a25cf6d70a4959ce2979f1 Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 15 Jun 2026 19:04:08 -0700 Subject: [PATCH 18/20] chore: update aurora connection tracker plugin --- common/lib/partial_plugin_service.ts | 10 ++ common/lib/plugin_service.ts | 15 ++- .../aurora_connection_tracker_plugin.ts | 72 ++++++++-- .../opened_connection_tracker.ts | 124 ++++++++++-------- 4 files changed, 155 insertions(+), 66 deletions(-) diff --git a/common/lib/partial_plugin_service.ts b/common/lib/partial_plugin_service.ts index 11079e35..4203b230 100644 --- a/common/lib/partial_plugin_service.ts +++ b/common/lib/partial_plugin_service.ts @@ -40,6 +40,7 @@ import { FullServicesContainer } from "./utils/full_services_container"; import { HostListProviderService } from "./host_list_provider_service"; import { StorageService } from "./utils/storage/storage_service"; import { CoreServicesContainer } from "./utils/core_services_container"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; /** * A PluginService containing some methods that are not intended to be called. This class is intended to be used @@ -62,6 +63,7 @@ export class PartialPluginService implements PluginService, HostListProviderServ protected readonly driverDialect: DriverDialect; protected allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; private _isPooledClient: boolean = false; + private _trackedConnectionHost: TrackedConnectionListHost | null = null; private connectionUrlParser: ConnectionUrlParser; constructor( @@ -519,4 +521,12 @@ export class PartialPluginService implements PluginService, HostListProviderServ setIsPooledClient(isPooledClient: boolean): void { this._isPooledClient = isPooledClient; } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; + } } diff --git a/common/lib/plugin_service.ts b/common/lib/plugin_service.ts index 0d235355..1b80affb 100644 --- a/common/lib/plugin_service.ts +++ b/common/lib/plugin_service.ts @@ -47,7 +47,7 @@ import { AllowedAndBlockedHosts } from "./allowed_and_blocked_hosts"; import { ConnectionPlugin } from "./connection_plugin"; import { FullServicesContainer } from "./utils/full_services_container"; import { StorageService } from "./utils/storage/storage_service"; -import { CoreServicesContainer } from "./utils/core_services_container"; +import type { TrackedConnectionListHost } from "./plugins/connection_tracker/tracked_connection_list"; export interface PluginService extends ErrorHandler { isInTransaction(): boolean; @@ -157,6 +157,10 @@ export interface PluginService extends ErrorHandler { isPooledClient(): boolean; setIsPooledClient(isPooledClient: boolean): void; + + getTrackedConnectionHost(): TrackedConnectionListHost | null; + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void; } export class PluginServiceImpl implements PluginService, HostListProviderService { @@ -180,6 +184,7 @@ export class PluginServiceImpl implements PluginService, HostListProviderService private allowedAndBlockedHosts: AllowedAndBlockedHosts | null = null; protected _isPooledClient: boolean = false; + protected _trackedConnectionHost: TrackedConnectionListHost | null = null; constructor( container: FullServicesContainer, @@ -791,4 +796,12 @@ export class PluginServiceImpl implements PluginService, HostListProviderService setIsPooledClient(isPooledClient: boolean): void { this._isPooledClient = isPooledClient; } + + getTrackedConnectionHost(): TrackedConnectionListHost | null { + return this._trackedConnectionHost; + } + + setTrackedConnectionHost(host: TrackedConnectionListHost | null): void { + this._trackedConnectionHost = host; + } } diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts index 9b70960e..4669b0fb 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts @@ -16,6 +16,7 @@ import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; import { CanReleaseResources } from "../../can_release_resources"; +import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; import { PluginService } from "../../plugin_service"; import { RdsUtils } from "../../utils/rds_utils"; import { HostInfo } from "../../host_info"; @@ -27,7 +28,15 @@ import { HostRole } from "../../host_role"; import { OpenedConnectionTracker } from "./opened_connection_tracker"; export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - private static readonly subscribedMethods = new Set(["notifyHostListChanged", "connect", "query", "rollback"]); + private static readonly subscribedMethods = new Set([ + ...SubscribedMethodHelper.NETWORK_BOUND_METHODS, + "end", + "abort", + "notifyHostListChanged" + ]); + private static readonly CLOSING_METHODS = new Set(["end", "abort"]); + private static readonly TOPOLOGY_CHANGES_EXPECTED_TIME_NS = BigInt(3 * 60 * 1_000_000_000); + private static hostListRefreshEndTimeNs: bigint = 0n; private readonly pluginService: PluginService; private readonly rdsUtils: RdsUtils; @@ -57,48 +66,89 @@ export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin impl const targetClient = await connectFunc(); let connectionHostInfo: HostInfo = this.pluginService.getRoutedHostInfo() ?? hostInfo; - if (targetClient) { + if (targetClient && !this.pluginService.isPooledClient()) { const type: RdsUrlType = this.rdsUtils.identifyRdsType(connectionHostInfo.host); - if (type.isRdsCluster) { + if (type.isRdsCluster || type === RdsUrlType.OTHER || type === RdsUrlType.IP_ADDRESS) { const identifiedHostInfo: HostInfo | null = await this.pluginService.identifyConnection(targetClient, connectionHostInfo); if (identifiedHostInfo) { connectionHostInfo = identifiedHostInfo; await this.pluginService.setRoutedHostInfo(connectionHostInfo); } } - await this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); - } + const host = this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); + this.pluginService.setTrackedConnectionHost(host); } return targetClient; } override async execute(methodName: string, methodFunc: () => Promise, methodArgs: any[]): Promise { + const currentHostInfo = this.pluginService.getCurrentHostInfo(); this.rememberWriter(); + const isClosing = AuroraConnectionTrackerPlugin.CLOSING_METHODS.has(methodName); + try { + if (!isClosing) { + let needRefreshHostList = false; + const localRefreshEndTimeNs = AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs; + if (localRefreshEndTimeNs > 0n) { + if (localRefreshEndTimeNs > process.hrtime.bigint()) { + needRefreshHostList = true; + } else { + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; + } + } + if (this.needUpdateCurrentWriter || needRefreshHostList) { + await this.checkWriterChanged(needRefreshHostList); + } + } + const result = await methodFunc(); - if (this.needUpdateCurrentWriter) { - await this.checkWriterChanged(); + + if (isClosing) { + const host = this.pluginService.getTrackedConnectionHost(); + if (host) { + this.tracker.removeConnectionTracking(host); + this.pluginService.setTrackedConnectionHost(null); + } else if (currentHostInfo) { + this.tracker.removeConnectionTrackingByHost(currentHostInfo, this.pluginService.getCurrentClient()?.targetClient); + } } return result; } catch (error) { if (error instanceof FailoverError) { - await this.checkWriterChanged(); + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = + process.hrtime.bigint() + AuroraConnectionTrackerPlugin.TOPOLOGY_CHANGES_EXPECTED_TIME_NS; + // This call may effectively close/abort the current connection. + await this.checkWriterChanged(true); } throw error; } } - private async checkWriterChanged(): Promise { + private async checkWriterChanged(needRefreshHostList: boolean): Promise { + if (needRefreshHostList) { + try { + await this.pluginService.refreshHostList(); + } catch (error) { + // Ignore: continue with whatever topology is currently available. + } + } + const hostInfoAfterFailover = this.getWriter(this.pluginService.getAllHosts()); + if (hostInfoAfterFailover === null) { + return; + } + if (this.currentWriter === null) { this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; - } else if (!this.currentWriter.equals(hostInfoAfterFailover!)) { - // writer changed + } else if (this.currentWriter.hostAndPort !== hostInfoAfterFailover.hostAndPort) { + // The writer changed. await this.tracker.invalidateAllConnections(this.currentWriter); this.tracker.logOpenedConnections(); this.currentWriter = hostInfoAfterFailover; this.needUpdateCurrentWriter = false; + AuroraConnectionTrackerPlugin.hostListRefreshEndTimeNs = 0n; } } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 8514bf97..55c3b6c2 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -21,9 +21,10 @@ import { logger } from "../../../logutils"; import { MapUtils } from "../../utils/map_utils"; import { Messages } from "../../utils/messages"; import { PluginService } from "../../plugin_service"; +import { TrackedConnectionList, TrackedConnectionListHost } from "./tracked_connection_list"; export class OpenedConnectionTracker { - static readonly openedConnections: Map>> = new Map>>(); + static readonly openedConnections: Map = new Map(); readonly pluginService: PluginService; private static readonly rdsUtils = new RdsUtils(); @@ -31,24 +32,28 @@ export class OpenedConnectionTracker { this.pluginService = pluginService; } - async populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): Promise { + populateOpenedConnectionQueue(hostInfo: HostInfo, client: ClientWrapper): TrackedConnectionListHost | null { if (!hostInfo || !client) { - return; + return null; } - // Check if the connection was established using an instance endpoint + // Check if the connection was established using an instance endpoint. if (OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host)) { - this.trackConnection(hostInfo.hostAndPort, client); - return; + const host = this.trackConnection(hostInfo.hostAndPort, client); + this.logOpenedConnections(); + return host; } - // It might be a custom domain name. Let's track by hostId and custom domain name + // It might be a custom domain name. Let's track by hostId and custom domain name. + let lastHost: TrackedConnectionListHost | null = null; if (hostInfo.hostId) { - this.trackConnection(hostInfo.hostId, client); + lastHost = this.trackConnection(hostInfo.hostId, client); } if (hostInfo.hostAndPort) { - this.trackConnection(hostInfo.hostAndPort, client); + lastHost = this.trackConnection(hostInfo.hostAndPort, client); } + this.logOpenedConnections(); + return lastHost; } async invalidateAllConnections(hostInfo: HostInfo): Promise { @@ -58,75 +63,86 @@ export class OpenedConnectionTracker { await this.invalidateAllConnectionsMultipleHosts(hostInfo.hostAndPort, hostInfo.host, hostInfo.hostId); } - async invalidateAllConnectionsMultipleHosts(...hosts: string[]): Promise { - try { - const instanceEndpoint = hosts - .filter((x) => OpenedConnectionTracker.rdsUtils.isRdsInstance(OpenedConnectionTracker.rdsUtils.removePort(x))) - .at(0); - if (!instanceEndpoint) { - return; + async invalidateAllConnectionsMultipleHosts(...keys: string[]): Promise { + for (const key of keys) { + if (!key) { + continue; + } + try { + const connectionList = OpenedConnectionTracker.openedConnections.get(key); + this.logConnectionList(key, connectionList); + await this.invalidateConnections(connectionList); + } catch (error) { + // Ignore and continue with the remaining keys. } - const connectionQueue = OpenedConnectionTracker.openedConnections.get(instanceEndpoint); - this.logConnectionQueue(instanceEndpoint, connectionQueue!); - await this.invalidateConnections(connectionQueue!); - } catch (error) { - // ignore } } - private trackConnection(instanceEndpoint: string, client: ClientWrapper): void { - const connectionQueue = MapUtils.computeIfAbsent( + removeConnectionTracking(host: TrackedConnectionListHost | null): void { + host?.remove(); + } + + removeConnectionTrackingByHost(hostInfo: HostInfo, client: ClientWrapper | undefined | null): void { + const hostAndPort = OpenedConnectionTracker.rdsUtils.isRdsInstance(hostInfo.host) ? hostInfo.hostAndPort : null; + if (!hostAndPort) { + return; + } + + const connectionList = OpenedConnectionTracker.openedConnections.get(hostAndPort); + if (connectionList) { + connectionList.removeIf((ref) => { + const conn = ref.deref(); + return !conn || conn === client; + }); + } + } + + private trackConnection(instanceEndpoint: string, client: ClientWrapper): TrackedConnectionListHost { + const connectionList = MapUtils.computeIfAbsent( OpenedConnectionTracker.openedConnections, instanceEndpoint, - (k) => new Array>() + (_) => new TrackedConnectionList() ); - connectionQueue!.push(new WeakRef(client)); - this.logOpenedConnections(); + return connectionList!.add(client); } - private async invalidateConnections(connectionQueue: Array>): Promise { - let clientRef: WeakRef | undefined; - while ((clientRef = connectionQueue?.shift()) != null) { - const client = clientRef?.deref() ?? null; - if (!client) { - continue; - } + private async invalidateConnections(connectionList: TrackedConnectionList | undefined): Promise { + if (!connectionList || connectionList.isEmpty()) { + return; + } + + const connections = connectionList.drainAll(); + for (const client of connections) { await this.pluginService.abortTargetClient(client); } } logOpenedConnections(): void { - let str = ""; - const hostList = []; - - for (const queue of OpenedConnectionTracker.openedConnections.values()) { - if (queue.length !== 0) { - for (const connRef of queue) { - const conn = connRef?.deref() ?? null; - if (conn) { - hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); - } - } - str = hostList.join("\n\t"); + const hostList: string[] = []; + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + for (const conn of connectionList.getConnections()) { + hostList.push(`${conn.id} - ${conn.hostInfo.toString()}`); } } - logger.debug(`Opened Connections Tracked: \n\t${str}`); + logger.debug(`Opened Connections Tracked: \n\t${hostList.join("\n\t")}`); } - private logConnectionQueue(host: string, queue: Array>): void { - if (!queue || queue.length === 0) { + private logConnectionList(host: string, connectionList: TrackedConnectionList | undefined): void { + if (!connectionList || connectionList.isEmpty()) { return; } - logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${queue.map((x) => x.deref()!.hostInfo).join()}\n]`)); + const connections = connectionList.getConnections().map((conn) => conn.hostInfo); + logger.debug(Messages.get("OpenedConnectionTracker.invalidatingConnections", `${host}\n[${connections.join()}\n]`)); } pruneNullConnections(): void { - for (const [key, queue] of OpenedConnectionTracker.openedConnections) { - OpenedConnectionTracker.openedConnections.set( - key, - queue.filter((connWeakRef: WeakRef) => connWeakRef?.deref() ?? null) - ); + for (const connectionList of OpenedConnectionTracker.openedConnections.values()) { + connectionList.removeIf((ref) => !ref.deref()); } } + + static clearCache(): void { + OpenedConnectionTracker.openedConnections.clear(); + } } From 414b0e535a5483a00903c9a284a501507383985f Mon Sep 17 00:00:00 2001 From: karezche <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 15 Jun 2026 19:04:39 -0700 Subject: [PATCH 19/20] chore: update workflows --- .github/workflows/aurora_performance.yml | 14 ++++---- .github/workflows/dependabot_auto_merge.yml | 2 +- .github/workflows/integration_tests.yml | 32 +++++++++++-------- .github/workflows/main.yml | 6 ++-- .github/workflows/release.yml | 6 ++-- .github/workflows/release_draft.yml | 8 ++--- .github/workflows/remove-old-artifacts.yml | 2 +- .github/workflows/run-autoscaling-tests.yml | 14 ++++---- .../aurora_connection_tracker_plugin.ts | 3 +- .../opened_connection_tracker.ts | 6 +--- 10 files changed, 47 insertions(+), 46 deletions(-) diff --git a/.github/workflows/aurora_performance.yml b/.github/workflows/aurora_performance.yml index 72ebb931..5d0bd082 100644 --- a/.github/workflows/aurora_performance.yml +++ b/.github/workflows/aurora_performance.yml @@ -18,25 +18,25 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_aurora_perf_tests @@ -59,7 +59,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -73,7 +73,7 @@ jobs: - name: Archive Performance results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.db }}-performance-results path: ./tests/integration/container/reports diff --git a/.github/workflows/dependabot_auto_merge.yml b/.github/workflows/dependabot_auto_merge.yml index c7574bab..536f793b 100644 --- a/.github/workflows/dependabot_auto_merge.yml +++ b/.github/workflows/dependabot_auto_merge.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Dependabot metadata id: metadata - uses: dependabot/fetch-metadata@v1 + uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v2 with: github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Approve PR diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 4b0e28e9..2218d9ea 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - chore/remove-aliases paths-ignore: - '**/*.md' - '**/*.jpg' @@ -63,24 +64,24 @@ jobs: steps: - name: 'Clone repository' - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_int_default_tests @@ -102,7 +103,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -116,7 +117,7 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: integration-report-default-${{ matrix.dbEngine }} path: ./tests/integration/container/reports @@ -126,6 +127,9 @@ jobs: name: Run Integration Tests (Latest) runs-on: ubuntu-latest needs: run-integration-tests-default + if: | + always() && + needs.run-integration-tests-default.result == 'success' strategy: fail-fast: false matrix: @@ -133,22 +137,22 @@ jobs: steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Configure AWS Credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_int_latest_tests @@ -170,7 +174,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() @@ -184,7 +188,7 @@ jobs: - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: integration-report-latest-${{ matrix.dbEngine }} path: ./tests/integration/container/reports diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed517c71..7ba7b23a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,11 +16,11 @@ jobs: run-checks-and-unit-tests: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Install dependencies run: npm install --no-save - name: Run eslint - linting diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aedac085..dcd54a9a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,11 +11,11 @@ jobs: packages: write steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" registry-url: "https://registry.npmjs.org" - name: "Install dependencies" run: npm install --no-save diff --git a/.github/workflows/release_draft.yml b/.github/workflows/release_draft.yml index fb2c2207..85973b38 100644 --- a/.github/workflows/release_draft.yml +++ b/.github/workflows/release_draft.yml @@ -19,11 +19,11 @@ jobs: runs-on: ubuntu-latest steps: - name: "Clone Repository" - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - name: "Set up Node.js" - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: "Install dependencies" run: npm install --no-save - name: "Run eslint - linting" @@ -46,7 +46,7 @@ jobs: touch RELEASE_DETAILS.md echo "$RELEASE_DETAILS" > RELEASE_DETAILS.md - name: "Upload to Draft Release" - uses: ncipollo/release-action@v1 + uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1 with: draft: true name: "AWS Advanced NodeJS Wrapper - v${{ env.RELEASE_VERSION }}" diff --git a/.github/workflows/remove-old-artifacts.yml b/.github/workflows/remove-old-artifacts.yml index 5899e867..71d497c8 100644 --- a/.github/workflows/remove-old-artifacts.yml +++ b/.github/workflows/remove-old-artifacts.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Remove Old Artifacts - uses: c-hive/gha-remove-artifacts@v1 + uses: c-hive/gha-remove-artifacts@44fc7acaf1b3d0987da0e8d4707a989d80e9554b # v1 with: age: "1 week" skip-tags: true diff --git a/.github/workflows/run-autoscaling-tests.yml b/.github/workflows/run-autoscaling-tests.yml index 32d243db..0c65d3ae 100644 --- a/.github/workflows/run-autoscaling-tests.yml +++ b/.github/workflows/run-autoscaling-tests.yml @@ -21,21 +21,21 @@ jobs: dbEngine: [ "mysql", "postgres" ] steps: - name: Clone repository - uses: actions/checkout@v4 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: fetch-depth: 50 - name: "Set up JDK 8" - uses: actions/setup-java@v3 + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 with: distribution: "corretto" java-version: 8 - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: - node-version: "20.x" + node-version: "22.x" - name: Configure AWS credentials id: creds - uses: aws-actions/configure-aws-credentials@v4 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_DEPLOY_ROLE }} role-session-name: nodejs_autoscaling_tests @@ -56,7 +56,7 @@ jobs: - name: "Get Github Action IP" if: always() id: ip - uses: haythem/public-ip@v1.3 + uses: haythem/public-ip@bdddd92c198b0955f0b494a8ebeac529754262ff # v1.3 - name: "Remove Github Action IP" if: always() run: | @@ -68,7 +68,7 @@ jobs: 2>&1 > /dev/null; - name: Archive results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: autoscaling-report path: ./tests/integration/container/reports diff --git a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts index 4669b0fb..d256d8bc 100644 --- a/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts +++ b/common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin.ts @@ -76,7 +76,8 @@ export class AuroraConnectionTrackerPlugin extends AbstractConnectionPlugin impl } } const host = this.tracker.populateOpenedConnectionQueue(connectionHostInfo, targetClient); - this.pluginService.setTrackedConnectionHost(host); } + this.pluginService.setTrackedConnectionHost(host); + } return targetClient; } diff --git a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts index 55c3b6c2..6efa0839 100644 --- a/common/lib/plugins/connection_tracker/opened_connection_tracker.ts +++ b/common/lib/plugins/connection_tracker/opened_connection_tracker.ts @@ -98,11 +98,7 @@ export class OpenedConnectionTracker { } private trackConnection(instanceEndpoint: string, client: ClientWrapper): TrackedConnectionListHost { - const connectionList = MapUtils.computeIfAbsent( - OpenedConnectionTracker.openedConnections, - instanceEndpoint, - (_) => new TrackedConnectionList() - ); + const connectionList = MapUtils.computeIfAbsent(OpenedConnectionTracker.openedConnections, instanceEndpoint, (_) => new TrackedConnectionList()); return connectionList!.add(client); } From 17928a1c8e207a1879e898205da20753e9547415 Mon Sep 17 00:00:00 2001 From: Karen <64801825+karenc-bq@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:42:05 +0000 Subject: [PATCH 20/20] chore: refactor efm plugins (#662) --- common/lib/connection_plugin_chain_builder.ts | 4 +- .../plugins/efm/base/connection_context.ts | 147 ++++++++ common/lib/plugins/efm/base/host_monitor.ts | 230 +++++++++++++ .../plugins/efm/base/host_monitor_service.ts | 117 +++++++ common/lib/plugins/efm/monitor.ts | 290 ---------------- .../plugins/efm/monitor_connection_context.ts | 155 --------- common/lib/plugins/efm/monitor_service.ts | 174 ---------- .../host_monitoring_connection_plugin.ts | 102 +++--- .../host_monitoring_plugin_factory.ts | 17 +- .../v2/host_monitoring2_connection_plugin.ts | 70 ++++ .../v2}/host_monitoring2_plugin_factory.ts | 17 +- .../host_monitoring2_connection_plugin.ts | 163 --------- common/lib/plugins/efm2/monitor.ts | 319 ------------------ .../efm2/monitor_connection_context.ts | 59 ---- common/lib/plugins/efm2/monitor_service.ts | 157 --------- .../profile/driver_configuration_profiles.ts | 2 +- common/lib/utils/monitoring/monitor.ts | 2 +- index.ts | 4 +- .../container/tests/performance.test.ts | 2 +- tests/unit/host_monitoring_plugin.test.ts | 91 ++--- tests/unit/monitor_connection_context.test.ts | 160 +++++---- tests/unit/monitor_impl.test.ts | 73 ++-- tests/unit/monitor_service_impl.test.ts | 247 ++++++-------- 23 files changed, 894 insertions(+), 1708 deletions(-) create mode 100644 common/lib/plugins/efm/base/connection_context.ts create mode 100644 common/lib/plugins/efm/base/host_monitor.ts create mode 100644 common/lib/plugins/efm/base/host_monitor_service.ts delete mode 100644 common/lib/plugins/efm/monitor.ts delete mode 100644 common/lib/plugins/efm/monitor_connection_context.ts delete mode 100644 common/lib/plugins/efm/monitor_service.ts rename common/lib/plugins/efm/{ => v1}/host_monitoring_connection_plugin.ts (64%) rename common/lib/plugins/efm/{ => v1}/host_monitoring_plugin_factory.ts (72%) create mode 100644 common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts rename common/lib/plugins/{efm2 => efm/v2}/host_monitoring2_plugin_factory.ts (72%) delete mode 100644 common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts delete mode 100644 common/lib/plugins/efm2/monitor.ts delete mode 100644 common/lib/plugins/efm2/monitor_connection_context.ts delete mode 100644 common/lib/plugins/efm2/monitor_service.ts diff --git a/common/lib/connection_plugin_chain_builder.ts b/common/lib/connection_plugin_chain_builder.ts index 38ec0a4f..1f4fd277 100644 --- a/common/lib/connection_plugin_chain_builder.ts +++ b/common/lib/connection_plugin_chain_builder.ts @@ -31,7 +31,7 @@ import { StaleDnsPluginFactory } from "./plugins/stale_dns/stale_dns_plugin_fact import { FederatedAuthPluginFactory } from "./plugins/federated_auth/federated_auth_plugin_factory"; import { ReadWriteSplittingPluginFactory } from "./plugins/read_write_splitting/read_write_splitting_plugin_factory"; import { OktaAuthPluginFactory } from "./plugins/federated_auth/okta_auth_plugin_factory"; -import { HostMonitoringPluginFactory } from "./plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "./plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "./plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "./plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; import { ConnectionProviderManager } from "./connection_provider_manager"; @@ -41,7 +41,7 @@ import { LimitlessConnectionPluginFactory } from "./plugins/limitless/limitless_ import { FastestResponseStrategyPluginFactory } from "./plugins/strategy/fastest_response/fastest_respose_strategy_plugin_factory"; import { CustomEndpointPluginFactory } from "./plugins/custom_endpoint/custom_endpoint_plugin_factory"; import { ConfigurationProfile } from "./profile/configuration_profile"; -import { HostMonitoring2PluginFactory } from "./plugins/efm2/host_monitoring2_plugin_factory"; +import { HostMonitoring2PluginFactory } from "./plugins/efm/v2/host_monitoring2_plugin_factory"; import { BlueGreenPluginFactory } from "./plugins/bluegreen/blue_green_plugin_factory"; import { GlobalDbFailoverPluginFactory } from "./plugins/gdb_failover/global_db_failover_plugin_factory"; import { FullServicesContainer } from "./utils/full_services_container"; diff --git a/common/lib/plugins/efm/base/connection_context.ts b/common/lib/plugins/efm/base/connection_context.ts new file mode 100644 index 00000000..bfcaffd5 --- /dev/null +++ b/common/lib/plugins/efm/base/connection_context.ts @@ -0,0 +1,147 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ClientWrapper } from "../../../client_wrapper"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; + +/** + * Monitoring context for each connection. This contains each connection's criteria for whether a + * host should be considered unhealthy. The context is shared between the main task and the + * monitor task. + */ +export interface ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + isActiveContext(): boolean; + isHostUnhealthy(): boolean; + setInactive(): void; + abortConnection(): Promise; + updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise; +} + +export class ConnectionContextImpl implements ConnectionContext { + readonly failureDetectionIntervalMillis: number; + readonly failureDetectionCount: number; + readonly expectedActiveMonitoringStartTimeNano: number; + + private readonly failureDetectionTimeMillis: number; + private readonly connectionToAbortRef: WeakRef; + private readonly abortedConnectionsCounter: TelemetryCounter; + private readonly startMonitorTimeNano: number; + + private _activeContext: boolean = true; + private _hostUnhealthy: boolean = false; + private invalidHostStartTimeNano: number = 0; + private failureCount: number = 0; + + constructor( + connectionToAbort: ClientWrapper, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number, + abortedConnectionsCounter: TelemetryCounter + ) { + this.connectionToAbortRef = new WeakRef(connectionToAbort); + this.failureDetectionTimeMillis = failureDetectionTimeMillis; + this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; + this.failureDetectionCount = failureDetectionCount; + this.abortedConnectionsCounter = abortedConnectionsCounter; + this.startMonitorTimeNano = getCurrentTimeNano(); + this.expectedActiveMonitoringStartTimeNano = this.startMonitorTimeNano + this.failureDetectionTimeMillis * 1_000_000; + } + + isActiveContext(): boolean { + return this._activeContext; + } + + isHostUnhealthy(): boolean { + return this._hostUnhealthy; + } + + setInactive(): void { + this._activeContext = false; + } + + async abortConnection(): Promise { + const connectionToAbort = this.connectionToAbortRef.deref(); + if (connectionToAbort == null || !this._activeContext) { + return; + } + + try { + await connectionToAbort.abort(); + this.abortedConnectionsCounter.inc(); + } catch (error: any) { + // ignore + logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); + } + } + + /** + * Update whether the connection is still valid if the total elapsed time has passed the + * grace period. + */ + async updateConnectionStatus(hostName: string, statusCheckStartTimeNano: number, statusCheckEndTimeNano: number, isValid: boolean): Promise { + if (!this._activeContext) { + return; + } + + const totalElapsedTimeNano = statusCheckEndTimeNano - this.startMonitorTimeNano; + + if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1_000_000) { + await this.setConnectionValid(hostName, isValid, statusCheckStartTimeNano, statusCheckEndTimeNano); + } + } + + private async setConnectionValid( + hostName: string, + connectionValid: boolean, + statusCheckStartNano: number, + statusCheckEndNano: number + ): Promise { + if (!connectionValid) { + this.failureCount++; + + if (this.invalidHostStartTimeNano === 0) { + this.invalidHostStartTimeNano = statusCheckStartNano; + } + + const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; + const maxInvalidHostDurationNano = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount) * 1_000_000; + + if (invalidHostDurationNano >= maxInvalidHostDurationNano) { + logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); + this._hostUnhealthy = true; + await this.abortConnection(); + return; + } + + logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); + return; + } + + this.failureCount = 0; + this.invalidHostStartTimeNano = 0; + this._hostUnhealthy = false; + + logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); + } +} diff --git a/common/lib/plugins/efm/base/host_monitor.ts b/common/lib/plugins/efm/base/host_monitor.ts new file mode 100644 index 00000000..9cebcd58 --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor.ts @@ -0,0 +1,230 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext } from "./connection_context"; +import { HostInfo } from "../../../host_info"; +import { PluginService } from "../../../plugin_service"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { logger } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { getCurrentTimeNano } from "../../../utils/utils"; +import { getTimeInNanos } from "../../../utils/utils"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { TelemetryFactory } from "../../../utils/telemetry/telemetry_factory"; +import { TelemetryTraceLevel } from "../../../utils/telemetry/telemetry_trace_level"; +import { AbstractMonitor, MonitorState } from "../../../utils/monitoring/monitor"; + +export interface HostMonitor { + startMonitoring(context: ConnectionContext): void; + + stopMonitoring(context: ConnectionContext): void; + + clearContexts(): void; + + isStopped(): boolean; + + run(): Promise; + + releaseResources(): Promise; +} + +type ConnectionStatus = [isValid: boolean, elapsedTimeNano: number]; + +export class HostMonitorImpl extends AbstractMonitor implements HostMonitor { + private static readonly SLEEP_WHEN_INACTIVE_MILLIS = 100; + private static readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS = 3000; + private static readonly MONITOR_TERMINATION_TIMEOUT_SEC = 30; + + private readonly pluginService: PluginService; + private readonly telemetryFactory: TelemetryFactory; + private readonly hostInvalidCounter: TelemetryCounter; + private readonly properties: Map; + private readonly hostInfo: HostInfo; + private readonly monitorDisposalTimeMillis: number; + private contexts: ConnectionContext[] = []; + + private contextLastUsedTimestampNano: number; + private monitoringClient: ClientWrapper | null = null; + private delayTimeoutId: ReturnType | undefined; + private sleepTimeoutId: ReturnType | undefined; + + constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { + super(HostMonitorImpl.MONITOR_TERMINATION_TIMEOUT_SEC); + this.pluginService = pluginService; + this.telemetryFactory = this.pluginService.getTelemetryFactory(); + this.hostInfo = hostInfo; + this.properties = properties; + this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; + this.hostInvalidCounter = this.telemetryFactory.createCounter(`efm.nodeUnhealthy.count.${hostInfo.hostId || hostInfo.host}`); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + } + + startMonitoring(context: ConnectionContext): void { + if (this._stop) { + logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); + } + + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + this.contexts.push(context); + } + + stopMonitoring(context: ConnectionContext): void { + if (!context) { + logger.warn(Messages.get("MonitorImpl.contextNullWarning")); + return; + } + + context.setInactive(); + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + this.lastActivityTimestampNanos = getTimeInNanos(); + } + + clearContexts(): void { + this.contexts.length = 0; + } + + isStopped(): boolean { + return this._stop || this.state === MonitorState.STOPPED; + } + + canDispose(): boolean { + return this.contexts.length === 0; + } + + async monitor(): Promise { + logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); + + try { + while (!this._stop) { + try { + this.lastActivityTimestampNanos = getTimeInNanos(); + + const activeContexts = this.contexts.filter((ctx) => ctx.isActiveContext()); + + if (activeContexts.length > 0) { + this.contextLastUsedTimestampNano = getCurrentTimeNano(); + + const statusCheckStartTimeNano = getCurrentTimeNano(); + const [isValid, elapsedTimeNano] = await this.checkConnectionStatus(); + + let delayMillis = -1; + + for (const context of activeContexts) { + if (!context.isActiveContext()) { + continue; + } + + await context.updateConnectionStatus(this.hostInfo.url, statusCheckStartTimeNano, statusCheckStartTimeNano + elapsedTimeNano, isValid); + + if ( + context.isActiveContext() && + !context.isHostUnhealthy() && + statusCheckStartTimeNano >= context.expectedActiveMonitoringStartTimeNano + ) { + if (delayMillis === -1 || delayMillis > context.failureDetectionIntervalMillis) { + delayMillis = context.failureDetectionIntervalMillis; + } + } + } + + this.contexts = this.contexts.filter((ctx) => ctx.isActiveContext() && !ctx.isHostUnhealthy()); + + if (delayMillis === -1) { + delayMillis = HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS; + } else { + delayMillis -= Math.round(elapsedTimeNano / 1_000_000); + if (delayMillis <= 0) { + delayMillis = HostMonitorImpl.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; + } + } + + await new Promise((resolve) => { + this.delayTimeoutId = setTimeout(resolve, delayMillis); + }); + } else { + if (getCurrentTimeNano() - this.contextLastUsedTimestampNano >= this.monitorDisposalTimeMillis * 1_000_000) { + break; + } + await new Promise((resolve) => { + this.sleepTimeoutId = setTimeout(resolve, HostMonitorImpl.SLEEP_WHEN_INACTIVE_MILLIS); + }); + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); + } + } + } catch (error: any) { + logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); + } + + logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); + } + + async close(): Promise { + this.contexts.length = 0; + await this.closeMonitoringClient(); + } + + async releaseResources(): Promise { + clearTimeout(this.delayTimeoutId); + clearTimeout(this.sleepTimeoutId); + await this.stop(); + } + + protected async checkConnectionStatus(): Promise { + const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); + connectContext.setAttribute("url", this.hostInfo.host); + return await connectContext.start(async () => { + const startNanos = getCurrentTimeNano(); + try { + if (this.monitoringClient != null && (await this.pluginService.isClientValid(this.monitoringClient))) { + return [true, getCurrentTimeNano() - startNanos]; + } + + await this.closeMonitoringClient(); + + const monitoringConnProperties = new Map(this.properties); + for (const key of monitoringConnProperties.keys()) { + if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { + continue; + } + monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); + monitoringConnProperties.delete(key); + } + + this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); + return [true, getCurrentTimeNano() - startNanos]; + } catch (error: any) { + this.hostInvalidCounter.inc(); + await this.closeMonitoringClient(); + return [false, getCurrentTimeNano() - startNanos]; + } + }); + } + + private async closeMonitoringClient(): Promise { + if (this.monitoringClient) { + try { + await this.pluginService.abortTargetClient(this.monitoringClient); + } catch { + // ignore + } + this.monitoringClient = null; + } + } +} diff --git a/common/lib/plugins/efm/base/host_monitor_service.ts b/common/lib/plugins/efm/base/host_monitor_service.ts new file mode 100644 index 00000000..1c976a17 --- /dev/null +++ b/common/lib/plugins/efm/base/host_monitor_service.ts @@ -0,0 +1,117 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { ConnectionContext, ConnectionContextImpl } from "./connection_context"; +import { HostMonitor, HostMonitorImpl } from "./host_monitor"; +import { HostInfo } from "../../../host_info"; +import { ClientWrapper } from "../../../client_wrapper"; +import { WrapperProperties } from "../../../wrapper_property"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { MonitorService } from "../../../utils/monitoring/monitor_service"; +import { MonitorInitializer } from "../../../utils/monitoring/monitor"; +import { TelemetryCounter } from "../../../utils/telemetry/telemetry_counter"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export interface HostMonitorService { + startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise; + + stopMonitoring(context: ConnectionContext): void; + + releaseResources(): Promise; +} + +export class HostMonitorServiceImpl implements HostMonitorService { + private static readonly MONITOR_DISPOSAL_TIME_NS = BigInt(10 * 60 * 1_000_000_000); // 10 minutes + private static readonly INACTIVE_TIMEOUT_NS = BigInt(3 * 60 * 1_000_000_000); // 3 minutes + + private readonly servicesContainer: FullServicesContainer; + private readonly coreMonitorService: MonitorService; + private readonly abortedConnectionsCounter: TelemetryCounter; + + constructor(servicesContainer: FullServicesContainer) { + this.servicesContainer = servicesContainer; + this.coreMonitorService = servicesContainer.monitorService; + const telemetryFactory = servicesContainer.telemetryFactory; + this.abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); + + this.coreMonitorService.registerMonitorTypeIfAbsent( + HostMonitorImpl, + HostMonitorServiceImpl.MONITOR_DISPOSAL_TIME_NS, + HostMonitorServiceImpl.INACTIVE_TIMEOUT_NS, + new Set(), + undefined + ); + } + + async startMonitoring( + connectionToAbort: ClientWrapper, + hostInfo: HostInfo, + properties: Map, + failureDetectionTimeMillis: number, + failureDetectionIntervalMillis: number, + failureDetectionCount: number + ): Promise { + const monitorKey = hostInfo.hostId || hostInfo.host; + + const monitor = await this.getMonitor(monitorKey, hostInfo, properties); + + if (!monitor) { + throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); + } + + const context = new ConnectionContextImpl( + connectionToAbort, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount, + this.abortedConnectionsCounter + ); + + monitor.startMonitoring(context); + return context; + } + + stopMonitoring(context: ConnectionContext): void { + context.setInactive(); + } + + private async getMonitor(monitorKey: string, hostInfo: HostInfo, properties: Map): Promise { + const monitorDisposalTimeMillis = WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties); + + const initializer: MonitorInitializer = { + createMonitor: (_servicesContainer) => + new HostMonitorImpl(this.servicesContainer.pluginService, hostInfo, properties, monitorDisposalTimeMillis) + }; + + return await this.coreMonitorService.runIfAbsent(HostMonitorImpl, monitorKey, this.servicesContainer, properties, initializer); + } + + async releaseResources(): Promise { + await this.coreMonitorService.stopAndRemoveMonitors(HostMonitorImpl); + } + + static async clearMonitors(monitorService: MonitorService): Promise { + await monitorService.stopAndRemoveMonitors(HostMonitorImpl); + } +} diff --git a/common/lib/plugins/efm/monitor.ts b/common/lib/plugins/efm/monitor.ts deleted file mode 100644 index 3094427f..00000000 --- a/common/lib/plugins/efm/monitor.ts +++ /dev/null @@ -1,290 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - stopMonitoring(context: MonitorConnectionContext): void; - - clearContexts(): void; - - isStopped(): boolean; - - run(): Promise; - - releaseResources(): Promise; - - endMonitoringClient(): Promise; -} - -class ConnectionStatus { - isValid: boolean; - elapsedTimeNano: number; - - constructor(isValid: boolean, elapsedTimeNano: number) { - this.isValid = isValid; - this.elapsedTimeNano = elapsedTimeNano; - } -} - -export class MonitorImpl implements Monitor { - private readonly SLEEP_WHEN_INACTIVE_MILLIS: number = 100; - private readonly MIN_CONNECTION_CHECK_TIMEOUT_MILLIS: number = 3000; - - private readonly activeContexts: MonitorConnectionContext[] = []; - private readonly newContexts: MonitorConnectionContext[] = []; - - private readonly pluginService: PluginService; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private readonly monitorDisposalTimeMillis: number; - private readonly telemetryFactory: TelemetryFactory; - private readonly instanceInvalidCounter: TelemetryCounter; - private contextLastUsedTimestampNanos: number; - private started = false; - private stopped: boolean = false; - private cancelled: boolean = false; - private monitoringClient: ClientWrapper | null = null; - private delayMillisTimeoutId: any; - private sleepWhenInactiveTimeoutId: any; - - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.properties = properties; - this.hostInfo = hostInfo; - this.monitorDisposalTimeMillis = monitorDisposalTimeMillis; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - const instanceId = this.hostInfo.hostId ?? this.hostInfo.host; - this.instanceInvalidCounter = this.telemetryFactory.createCounter(`efm.hostUnhealthy.count.${instanceId}`); - } - - startRun() { - this.run(); - this.started = true; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.stopped) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const currentTimeNanos: number = getCurrentTimeNano(); - context.startMonitorTimeNano = currentTimeNanos; - this.contextLastUsedTimestampNanos = currentTimeNanos; - this.newContexts.push(context); - if (!this.started) { - this.startRun(); - } - } - - stopMonitoring(context: MonitorConnectionContext): void { - if (context == null) { - logger.warn(Messages.get("MonitorImpl.contextNullWarning")); - return; - } - - context.isActiveContext = false; - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.cancelled) { - try { - let newMonitorContext: MonitorConnectionContext | undefined; - let firstAddedNewMonitorContext: MonitorConnectionContext | null = null; - const currentTimeNano: number = getCurrentTimeNano(); - while ((newMonitorContext = this.newContexts?.shift()) != null) { - if (firstAddedNewMonitorContext === newMonitorContext) { - this.newContexts.push(newMonitorContext); - - break; - } - - if (newMonitorContext.isActiveContext) { - if (newMonitorContext.expectedActiveMonitoringStartTimeNano > currentTimeNano) { - this.newContexts.push(newMonitorContext); - - firstAddedNewMonitorContext = firstAddedNewMonitorContext ?? newMonitorContext; - } else { - this.activeContexts.push(newMonitorContext); - } - } - } - - if (this.activeContexts.length > 0) { - this.contextLastUsedTimestampNanos = getCurrentTimeNano(); - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - this.contextLastUsedTimestampNanos = statusCheckStartTimeNanos; - - const status: ConnectionStatus = await this.checkConnectionStatus(); - let delayMillis: number = -1; - - let monitorContext: MonitorConnectionContext | undefined; - let firstAddedMonitorContext: MonitorConnectionContext | null = null; - - while ((monitorContext = this.activeContexts?.shift()) != null) { - // If context is already invalid, just skip it. - if (!monitorContext.isActiveContext) { - continue; - } - - if (firstAddedMonitorContext == monitorContext) { - // This context is already processed by this loop. - // Add it to the array and exit this loop. - - this.activeContexts?.push(monitorContext); - break; - } - - // Otherwise, process this context. - await monitorContext.updateConnectionStatus( - this.hostInfo.url, - statusCheckStartTimeNanos, - statusCheckStartTimeNanos + status.elapsedTimeNano, - status.isValid - ); - - if (monitorContext.isActiveContext && !monitorContext.isHostUnhealthy) { - this.activeContexts?.push(monitorContext); - - if (firstAddedMonitorContext == null) { - firstAddedMonitorContext = monitorContext; - } - - if (delayMillis === -1 || delayMillis > monitorContext.failureDetectionIntervalMillis) { - delayMillis = monitorContext.failureDetectionIntervalMillis; - } - } - } - - if (delayMillis === -1) { - // No active contexts. - delayMillis = this.SLEEP_WHEN_INACTIVE_MILLIS; - } else { - delayMillis -= Math.round(status.elapsedTimeNano / 1_000_000); - // Check for minimum delay between host health check; - if (delayMillis <= 0) { - delayMillis = this.MIN_CONNECTION_CHECK_TIMEOUT_MILLIS; - } - } - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, delayMillis); - }); - } else { - if (getCurrentTimeNano() - this.contextLastUsedTimestampNanos >= this.monitorDisposalTimeMillis * 1_000_000) { - break; - } - await new Promise((resolve) => { - this.sleepWhenInactiveTimeoutId = setTimeout(resolve, this.SLEEP_WHEN_INACTIVE_MILLIS); - }); - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - this.stopped = true; - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - return await connectContext.start(async () => { - const startNanos = getCurrentTimeNano(); - try { - const clientIsValid = this.monitoringClient && (await this.pluginService.isClientValid(this.monitoringClient)); - - if (this.monitoringClient !== null && clientIsValid) { - return Promise.resolve(new ConnectionStatus(clientIsValid, getCurrentTimeNano() - startNanos)); - } - - await this.endMonitoringClient(); - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return Promise.resolve(new ConnectionStatus(true, getCurrentTimeNano() - startNanos)); - } catch (error: any) { - this.instanceInvalidCounter.inc(); - await this.endMonitoringClient(); - return Promise.resolve(new ConnectionStatus(false, getCurrentTimeNano() - startNanos)); - } - }); - } - - clearContexts(): void { - this.activeContexts.length = 0; - this.newContexts.length = 0; - } - - isStopped(): boolean { - return this.stopped || this.cancelled; - } - - async releaseResources() { - this.cancelled = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenInactiveTimeoutId); - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient() { - if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - } -} diff --git a/common/lib/plugins/efm/monitor_connection_context.ts b/common/lib/plugins/efm/monitor_connection_context.ts deleted file mode 100644 index a6889322..00000000 --- a/common/lib/plugins/efm/monitor_connection_context.ts +++ /dev/null @@ -1,155 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { Monitor } from "./monitor"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { sleep } from "../../utils/utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { PluginService } from "../../plugin_service"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; - -export class MonitorConnectionContext { - readonly failureDetectionIntervalMillis: number; - private readonly failureDetectionTimeMillis: number; - private readonly failureDetectionCount: number; - private readonly telemetryAbortedConnectionCounter: TelemetryCounter; - readonly clientToAbort: ClientWrapper; - readonly monitor: Monitor; - readonly pluginService: PluginService; - - isActiveContext: boolean = true; - isHostUnhealthy: boolean = false; - startMonitorTimeNano: number = 0; - expectedActiveMonitoringStartTimeNano: number = 0; - private invalidHostStartTimeNano: number = 0; - private abortedConnectionCounter: number = 0; - failureCount: number = 0; - id: string = uniqueId("_monitorContext"); - - constructor( - monitor: Monitor, - clientToAbort: ClientWrapper, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - pluginService: PluginService, - telemetryAbortedConnectionCounter: TelemetryCounter - ) { - this.monitor = monitor; - this.clientToAbort = clientToAbort; - this.failureDetectionTimeMillis = failureDetectionTimeMillis; - this.failureDetectionIntervalMillis = failureDetectionIntervalMillis; - this.failureDetectionCount = failureDetectionCount; - this.pluginService = pluginService; - this.telemetryAbortedConnectionCounter = telemetryAbortedConnectionCounter; - } - - resetInvalidHostStartTimeNano(): void { - this.invalidHostStartTimeNano = 0; - } - - isInvalidHostStartTimeDefined(): boolean { - return this.invalidHostStartTimeNano > 0; - } - - async abortConnection(): Promise { - if (this.clientToAbort == null || !this.isActiveContext) { - return Promise.resolve(); - } - - try { - await this.pluginService.abortTargetClient(this.clientToAbort); - this.telemetryAbortedConnectionCounter.inc(); - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - this.abortedConnectionCounter++; - } - - /** - * Update whether the connection is still valid if the total elapsed time has passed the grace period. - * @param hostName A host name for logging purposes. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @param isValid Whether the connection is valid. - */ - async updateConnectionStatus(hostName: string, statusCheckStartNano: number, statusCheckEndNano: number, isValid: boolean): Promise { - if (!this.isActiveContext) { - return; - } - - const totalElapsedTimeNano: number = statusCheckEndNano - this.startMonitorTimeNano; - if (totalElapsedTimeNano > this.failureDetectionTimeMillis * 1000000) { - await this.setConnectionValid(hostName, isValid, statusCheckStartNano, statusCheckEndNano); - } - } - - /** - * Set whether the connection to the server is still valid based on the monitoring settings set in the {@link AwsClient}. - * - *

These monitoring settings include: - * - *

    - *
  • {@code failureDetectionInterval} - *
  • {@code failureDetectionTime} - *
  • {@code failureDetectionCount} - *
- * - * @param hostName A host name for logging purposes. - * @param connectionValid Boolean indicating whether the server is still responsive. - * @param statusCheckStartNano The time when connection status check started in nanos. - * @param statusCheckEndNano The time when connection status check ended in nanos. - * @protected - */ - async setConnectionValid(hostName: string, connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number) { - if (!connectionValid) { - this.failureCount++; - - if (!this.isInvalidHostStartTimeDefined()) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano: number = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationMillis: number = this.failureDetectionIntervalMillis * Math.max(0, this.failureDetectionCount); - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationMillis * 1_000_000) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", hostName)); - this.isHostUnhealthy = true; - await this.abortConnection(); - return; - } - - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", hostName)); - return; - } - - this.failureCount = 0; - this.resetInvalidHostStartTimeNano(); - this.isHostUnhealthy = false; - - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", hostName)); - } - - async trackHealthStatus() { - while (!this.isHostUnhealthy && this.isActiveContext) { - await sleep(100); - } - - throw new AwsWrapperError("trackHealthStatus stopped"); - } -} diff --git a/common/lib/plugins/efm/monitor_service.ts b/common/lib/plugins/efm/monitor_service.ts deleted file mode 100644 index c81639f5..00000000 --- a/common/lib/plugins/efm/monitor_service.ts +++ /dev/null @@ -1,174 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError, IllegalArgumentError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; -import { ClientWrapper } from "../../client_wrapper"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - stopMonitoring(context: MonitorConnectionContext): Promise; - - stopMonitoringForAllConnections(hostKeys: Set): void; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - undefined, - async (monitor: Monitor) => { - await monitor.releaseResources(); - }, - "efm/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private cachedMonitorHostKeys: Set | undefined; - private cachedMonitorRef: WeakRef | undefined; - monitorSupplier = (pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) => - new MonitorImpl(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostKeys: Set, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - if (hostKeys.size === 0) { - throw new IllegalArgumentError(Messages.get("MonitorService.emptyAliasSet", hostInfo.host)); - } - - let monitor: Monitor | null = this.cachedMonitorRef?.deref() ?? null; - - if (!monitor || (monitor && monitor.isStopped()) || this.cachedMonitorHostKeys?.size === 0 || this.cachedMonitorHostKeys !== hostKeys) { - monitor = await this.getMonitor(hostKeys, hostInfo, properties); - if (monitor) { - this.cachedMonitorRef = new WeakRef(monitor); - this.cachedMonitorHostKeys = hostKeys; - } - } - - const telemetryFactory = this.pluginService.getTelemetryFactory(); - const abortedConnectionsCounter = telemetryFactory.createCounter("efm.connections.aborted"); - - if (monitor) { - const context = new MonitorConnectionContext( - monitor, - clientToAbort, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.pluginService, - abortedConnectionsCounter - ); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext) { - context.monitor.stopMonitoring(context); - await context.monitor.endMonitoringClient(); - } - - stopMonitoringForAllConnections(hostKeys: Set) { - let monitor: Monitor; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - if (monitor) { - monitor.clearContexts(); - return; - } - } - } - - async getMonitor(hostKeys: Set, hostInfo: HostInfo, properties: Map): Promise { - let monitor: Monitor; - let anyHostKey: string; - for (const hostKey of hostKeys) { - monitor = MonitorServiceImpl.monitors.get(hostKey); - anyHostKey = hostKey; - if (monitor) { - break; - } - } - - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - if (anyHostKey && (!monitor || (monitor && monitor.isStopped()))) { - monitor = MonitorServiceImpl.monitors.computeIfAbsent( - anyHostKey, - () => this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)), - cacheExpirationNanos - ); - - if (monitor && monitor.isStopped()) { - await monitor.releaseResources(); - MonitorServiceImpl.monitors.remove(anyHostKey); - monitor = this.monitorSupplier(this.pluginService, hostInfo, properties, WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties)); - } - } - - if (monitor) { - this.populateMonitorMap(hostKeys, monitor, cacheExpirationNanos); - return monitor; - } - - return null; - } - - private populateMonitorMap(hostKeys: Set, monitor: Monitor, cacheExpirationNanos: bigint) { - for (const hostKey of hostKeys) { - MonitorServiceImpl.monitors.putIfAbsent(hostKey, monitor, cacheExpirationNanos); - } - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - this.cachedMonitorHostKeys = undefined; - this.cachedMonitorRef = undefined; - } - - // Used for performance testing. - static async clearMonitors() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts similarity index 64% rename from common/lib/plugins/efm/host_monitoring_connection_plugin.ts rename to common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts index 59ad986f..3c7548d2 100644 --- a/common/lib/plugins/efm/host_monitoring_connection_plugin.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_connection_plugin.ts @@ -14,37 +14,42 @@ limitations under the License. */ -import { HostInfo, AwsWrapperError, UnavailableHostError, HostAvailability } from "../../"; -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; +import { HostInfo } from "../../../host_info"; +import { AwsWrapperError, UnavailableHostError } from "../../../utils/errors"; +import { PluginService } from "../../../plugin_service"; +import { HostChangeOptions } from "../../../host_change_options"; +import { OldConnectionSuggestionAction } from "../../../old_connection_suggestion_action"; +import { RdsUtils } from "../../../utils/rds_utils"; +import { AbstractConnectionPlugin } from "../../../abstract_connection_plugin"; +import { RdsUrlType } from "../../../utils/rds_url_type"; +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { HostListProvider } from "../../../host_list_provider/host_list_provider"; +import { CanReleaseResources } from "../../../can_release_resources"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { ClientWrapper } from "../../../client_wrapper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; +import { sleep } from "../../../utils/utils"; export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { + private static readonly UNHEALTHY_STATE = Symbol("unhealthy"); + id: string = uniqueId("_efmPlugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; + protected readonly properties: Map; + protected readonly pluginService: PluginService; + protected readonly rdsUtils: RdsUtils; + protected readonly monitorService: HostMonitorService; private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { super(); - this.pluginService = pluginService; + this.pluginService = servicesContainer.pluginService; this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); + this.rdsUtils = new RdsUtils(); + this.monitorService = monitorService ?? new HostMonitorServiceImpl(servicesContainer); } getSubscribedMethods(): Set { @@ -78,20 +83,19 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return methodFunc(); } - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties) as number; - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties) as number; - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties) as number; + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); let result: T; - let monitorContext: MonitorConnectionContext | null = null; + let context: ConnectionContext | null = null; try { logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - monitorContext = await this.monitorService.startMonitoring( + context = await this.monitorService.startMonitoring( this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo.allAliases, monitoringHostInfo, this.properties, failureDetectionTimeMillis, @@ -99,19 +103,21 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp failureDetectionCount ); - result = await Promise.race([monitorContext.trackHealthStatus(), methodFunc()]) - .then((result: any) => { - return result; - }) - .catch((error: any) => { - throw error; + const methodPromise = methodFunc(); + const raceResult = await Promise.race([this.waitForUnhealthy(context), methodPromise]); + if (raceResult === HostMonitoringConnectionPlugin.UNHEALTHY_STATE) { + methodPromise.catch(() => { + // Attach a no-op rejection handler to prevent it from throwing an unhandled promise rejection error when waitForUnhealthy times out first. }); + throw new AwsWrapperError("Host monitoring detected unhealthy host"); + } + result = raceResult as T; } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext); + if (context != null) { + this.monitorService.stopMonitoring(context); logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - if (monitorContext.isHostUnhealthy) { + if (context.isHostUnhealthy()) { const monitoringHostInfo = await this.getMonitoringHostInfo(); const targetClient = this.pluginService.getCurrentClient().targetClient; let isClientValid = false; @@ -135,6 +141,13 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp return result; } + private async waitForUnhealthy(context: ConnectionContext): Promise { + while (!context.isHostUnhealthy() && context.isActiveContext()) { + await sleep(100); + } + return HostMonitoringConnectionPlugin.UNHEALTHY_STATE; + } + private throwUnableToIdentifyConnection(host: HostInfo | null): never { const provider: HostListProvider | null = this.pluginService.getHostListProvider(); throw new AwsWrapperError( @@ -167,6 +180,7 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); this.throwUnableToIdentifyConnection(host); } + // Update identified HostInfo for the current connection await this.pluginService.setCurrentClient(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); } @@ -180,15 +194,9 @@ export class HostMonitoringConnectionPlugin extends AbstractConnectionPlugin imp } async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.WENT_DOWN) || changes.has(HostChangeOptions.HOST_DELETED)) { - const aliases = (await this.getMonitoringHostInfo())?.allAliases; - if (aliases && aliases.size > 0) { - this.monitorService.stopMonitoringForAllConnections(aliases); - } + if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { + this.monitoringHostInfo = null; } - - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; return OldConnectionSuggestionAction.NO_OPINION; } diff --git a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts similarity index 72% rename from common/lib/plugins/efm/host_monitoring_plugin_factory.ts rename to common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts index d6b43095..49f3d628 100644 --- a/common/lib/plugins/efm/host_monitoring_plugin_factory.ts +++ b/common/lib/plugins/efm/v1/host_monitoring_plugin_factory.ts @@ -14,12 +14,11 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; -import { FullServicesContainer } from "../../utils/full_services_container"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoringPluginFactory extends ConnectionPluginFactory { private static hostMonitoringPlugin: any; @@ -29,11 +28,7 @@ export class HostMonitoringPluginFactory extends ConnectionPluginFactory { if (!HostMonitoringPluginFactory.hostMonitoringPlugin) { HostMonitoringPluginFactory.hostMonitoringPlugin = await import("./host_monitoring_connection_plugin"); } - return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin( - servicesContainer.pluginService, - properties, - new RdsUtils() - ); + return new HostMonitoringPluginFactory.hostMonitoringPlugin.HostMonitoringConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts new file mode 100644 index 00000000..3ca1194d --- /dev/null +++ b/common/lib/plugins/efm/v2/host_monitoring2_connection_plugin.ts @@ -0,0 +1,70 @@ +/* + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"). + You may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import { WrapperProperties } from "../../../wrapper_property"; +import { ConnectionContext } from "../base/connection_context"; +import { HostMonitorService, HostMonitorServiceImpl } from "../base/host_monitor_service"; +import { HostMonitoringConnectionPlugin } from "../v1/host_monitoring_connection_plugin"; +import { logger, uniqueId } from "../../../../logutils"; +import { Messages } from "../../../utils/messages"; +import { SubscribedMethodHelper } from "../../../utils/subscribed_method_helper"; +import { FullServicesContainer } from "../../../utils/full_services_container"; + +export class HostMonitoring2ConnectionPlugin extends HostMonitoringConnectionPlugin { + id: string = uniqueId("_efm2Plugin"); + + constructor(servicesContainer: FullServicesContainer, properties: Map, monitorService?: HostMonitorService) { + super(servicesContainer, properties, monitorService ?? new HostMonitorServiceImpl(servicesContainer)); + } + + async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { + const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); + + if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { + return methodFunc(); + } + + const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); + const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); + const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); + + let result: T; + let context: ConnectionContext | null = null; + + try { + logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); + const monitoringHostInfo = await this.getMonitoringHostInfo(); + + context = await this.monitorService.startMonitoring( + this.pluginService.getCurrentClient().targetClient, + monitoringHostInfo, + this.properties, + failureDetectionTimeMillis, + failureDetectionIntervalMillis, + failureDetectionCount + ); + + result = await methodFunc(); + } finally { + if (context != null) { + this.monitorService.stopMonitoring(context); + logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); + } + } + + return result; + } +} diff --git a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts similarity index 72% rename from common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts rename to common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts index 0ac0319f..e21e4c8b 100644 --- a/common/lib/plugins/efm2/host_monitoring2_plugin_factory.ts +++ b/common/lib/plugins/efm/v2/host_monitoring2_plugin_factory.ts @@ -14,12 +14,11 @@ limitations under the License. */ -import { ConnectionPluginFactory } from "../../plugin_factory"; -import { ConnectionPlugin } from "../../connection_plugin"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AwsWrapperError } from "../../utils/errors"; -import { Messages } from "../../utils/messages"; -import { FullServicesContainer } from "../../utils/full_services_container"; +import { ConnectionPluginFactory } from "../../../plugin_factory"; +import { ConnectionPlugin } from "../../../connection_plugin"; +import { AwsWrapperError } from "../../../utils/errors"; +import { Messages } from "../../../utils/messages"; +import { FullServicesContainer } from "../../../utils/full_services_container"; export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { private static hostMonitoring2Plugin: any; @@ -29,11 +28,7 @@ export class HostMonitoring2PluginFactory extends ConnectionPluginFactory { if (!HostMonitoring2PluginFactory.hostMonitoring2Plugin) { HostMonitoring2PluginFactory.hostMonitoring2Plugin = await import("./host_monitoring2_connection_plugin"); } - return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin( - servicesContainer.pluginService, - properties, - new RdsUtils() - ); + return new HostMonitoring2PluginFactory.hostMonitoring2Plugin.HostMonitoring2ConnectionPlugin(servicesContainer, properties); } catch (error: any) { throw new AwsWrapperError(Messages.get("ConnectionPluginChainBuilder.errorImportingPlugin", error.message, "HostMonitoringPlugin")); } diff --git a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts b/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts deleted file mode 100644 index 036b6dcd..00000000 --- a/common/lib/plugins/efm2/host_monitoring2_connection_plugin.ts +++ /dev/null @@ -1,163 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { PluginService } from "../../plugin_service"; -import { HostChangeOptions } from "../../host_change_options"; -import { HostInfo } from "../../host_info"; -import { OldConnectionSuggestionAction } from "../../old_connection_suggestion_action"; -import { RdsUtils } from "../../utils/rds_utils"; -import { AbstractConnectionPlugin } from "../../abstract_connection_plugin"; -import { RdsUrlType } from "../../utils/rds_url_type"; -import { WrapperProperties } from "../../wrapper_property"; -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { logger, uniqueId } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { MonitorService, MonitorServiceImpl } from "./monitor_service"; -import { AwsWrapperError } from "../../"; -import { HostListProvider } from "../../host_list_provider/host_list_provider"; -import { CanReleaseResources } from "../../can_release_resources"; -import { SubscribedMethodHelper } from "../../utils/subscribed_method_helper"; -import { ClientWrapper } from "../../client_wrapper"; - -export class HostMonitoring2ConnectionPlugin extends AbstractConnectionPlugin implements CanReleaseResources { - id: string = uniqueId("_efm2Plugin"); - private readonly properties: Map; - private pluginService: PluginService; - private rdsUtils: RdsUtils; - private monitoringHostInfo: HostInfo | null = null; - private monitorService: MonitorService; - - constructor(pluginService: PluginService, properties: Map, rdsUtils: RdsUtils = new RdsUtils(), monitorService?: MonitorServiceImpl) { - super(); - this.pluginService = pluginService; - this.properties = properties; - this.rdsUtils = rdsUtils; - this.monitorService = monitorService ?? new MonitorServiceImpl(pluginService); - } - - getSubscribedMethods(): Set { - return new Set(["*"]); - } - - async connect( - hostInfo: HostInfo, - props: Map, - isInitialConnection: boolean, - connectFunc: () => Promise - ): Promise { - const targetClient = await connectFunc(); - if (targetClient != null) { - const type: RdsUrlType = this.rdsUtils.identifyRdsType(hostInfo.host); - if (type.isRdsCluster) { - hostInfo.resetAliases(); - await this.pluginService.fillAliases(targetClient, hostInfo); - } - } - return targetClient; - } - - async execute(methodName: string, methodFunc: () => Promise, methodArgs: any): Promise { - const isEnabled: boolean = WrapperProperties.FAILURE_DETECTION_ENABLED.get(this.properties); - - if (!isEnabled || !SubscribedMethodHelper.NETWORK_BOUND_METHODS.includes(methodName)) { - return methodFunc(); - } - - const failureDetectionTimeMillis: number = WrapperProperties.FAILURE_DETECTION_TIME_MS.get(this.properties); - const failureDetectionIntervalMillis: number = WrapperProperties.FAILURE_DETECTION_INTERVAL_MS.get(this.properties); - const failureDetectionCount: number = WrapperProperties.FAILURE_DETECTION_COUNT.get(this.properties); - - let result: T; - let monitorContext: MonitorConnectionContext | null = null; - - try { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.activatedMonitoring", methodName)); - const monitoringHostInfo: HostInfo = await this.getMonitoringHostInfo(); - - monitorContext = await this.monitorService.startMonitoring( - this.pluginService.getCurrentClient().targetClient, - monitoringHostInfo, - this.properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount - ); - - result = await methodFunc(); - } finally { - if (monitorContext != null) { - await this.monitorService.stopMonitoring(monitorContext, this.pluginService.getCurrentClient().targetClient); - - logger.debug(Messages.get("HostMonitoringConnectionPlugin.monitoringDeactivated", methodName)); - } - } - - return result; - } - - private throwUnableToIdentifyConnection(host: HostInfo | null): never { - const provider: HostListProvider | null = this.pluginService.getHostListProvider(); - throw new AwsWrapperError( - Messages.get( - "HostMonitoringConnectionPlugin.unableToIdentifyConnection", - host !== null ? host.host : "unknown host", - provider !== null ? provider.getHostProviderType() : "unknown provider" - ) - ); - } - - async getMonitoringHostInfo(): Promise { - if (this.monitoringHostInfo) { - return this.monitoringHostInfo; - } - this.monitoringHostInfo = this.pluginService.getCurrentHostInfo(); - if (this.monitoringHostInfo === null) { - this.throwUnableToIdentifyConnection(null); - } - const rdsUrlType: RdsUrlType = this.rdsUtils.identifyRdsType(this.monitoringHostInfo.url); - - try { - if (rdsUrlType.isRdsCluster) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.identifyClusterConnection")); - this.monitoringHostInfo = await this.pluginService.identifyConnection(this.pluginService.getCurrentClient().targetClient!); - if (this.monitoringHostInfo == null) { - const host: HostInfo | null = this.pluginService.getCurrentHostInfo(); - this.throwUnableToIdentifyConnection(host); - } - await this.pluginService.fillAliases(this.pluginService.getCurrentClient().targetClient!, this.monitoringHostInfo); - } - } catch (error: any) { - if (!(error instanceof AwsWrapperError)) { - logger.debug(Messages.get("HostMonitoringConnectionPlugin.errorIdentifyingConnection", error.message)); - } - throw error; - } - - return this.monitoringHostInfo; - } - - async notifyConnectionChanged(changes: Set): Promise { - if (changes.has(HostChangeOptions.HOSTNAME) || changes.has(HostChangeOptions.HOST_CHANGED)) { - // Reset monitoring host info since the associated connection has changed. - this.monitoringHostInfo = null; - } - return OldConnectionSuggestionAction.NO_OPINION; - } - - async releaseResources(): Promise { - await this.monitorService.releaseResources(); - } -} diff --git a/common/lib/plugins/efm2/monitor.ts b/common/lib/plugins/efm2/monitor.ts deleted file mode 100644 index f1d72a61..00000000 --- a/common/lib/plugins/efm2/monitor.ts +++ /dev/null @@ -1,319 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { PluginService } from "../../plugin_service"; -import { logger } from "../../../logutils"; -import { Messages } from "../../utils/messages"; -import { ClientWrapper } from "../../client_wrapper"; -import { getCurrentTimeNano, sleep } from "../../utils/utils"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryTraceLevel } from "../../utils/telemetry/telemetry_trace_level"; -import { HostAvailability } from "../../host_availability/host_availability"; -import { MapUtils } from "../../utils/map_utils"; -import { WrapperProperties } from "../../wrapper_property"; - -export interface Monitor { - startMonitoring(context: MonitorConnectionContext): void; - - run(): Promise; - - canDispose(): boolean; - - endMonitoringClient(): Promise; - - releaseResources(): Promise; -} - -export class MonitorImpl implements Monitor { - private static readonly TASK_SLEEP_MILLIS: number = 100; - private activeContexts: WeakRef[] = []; - static newContexts: Map>> = new Map>>(); - private readonly pluginService: PluginService; - private readonly telemetryFactory: TelemetryFactory; - private readonly properties: Map; - private readonly hostInfo: HostInfo; - private stopped: boolean = false; - - private monitoringClient: ClientWrapper | null = null; - - private readonly failureDetectionTimeNano: number; - private readonly failureDetectionIntervalNanos: number; - private readonly failureDetectionCount: number; - - private invalidHostStartTimeNano: number; - private failureCount: number; - private hostUnhealthy: boolean = false; - private readonly abortedConnectionsCounter: TelemetryCounter; - private delayMillisTimeoutId: any; - private sleepWhenHostHealthyTimeoutId: any; - - constructor( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) { - this.pluginService = pluginService; - this.telemetryFactory = this.pluginService.getTelemetryFactory(); - this.hostInfo = hostInfo; - this.properties = properties; - this.failureDetectionTimeNano = failureDetectionTimeMillis * 1000000; - this.failureDetectionIntervalNanos = failureDetectionIntervalMillis * 1000000; - this.failureDetectionCount = failureDetectionCount; - this.abortedConnectionsCounter = abortedConnectionsCounter; - - const hostId: string = this.hostInfo.hostId ?? this.hostInfo.host; - - this.telemetryFactory.createGauge(`efm2.newContexts.size.${hostId}`, () => MonitorImpl.newContexts.size === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.activeContexts.size.${hostId}`, () => this.activeContexts.length === Number.MAX_SAFE_INTEGER); - - this.telemetryFactory.createGauge(`efm2.hostHealthy.${hostId}`, () => (this.hostUnhealthy ? 0 : 1)); - - Promise.race([this.newContextRun(), this.run()]).finally(() => { - this.stopped = true; - }); - } - - canDispose(): boolean { - return this.activeContexts.length === 0 && MonitorImpl.newContexts.size === 0; - } - - startMonitoring(context: MonitorConnectionContext): void { - if (this.isStopped()) { - logger.warn(Messages.get("MonitorImpl.monitorIsStopped", this.hostInfo.host)); - } - - const startMonitorTimeNano = getCurrentTimeNano() + this.failureDetectionTimeNano; - const connectionQueue = MapUtils.computeIfAbsent( - MonitorImpl.newContexts, - startMonitorTimeNano, - () => new Array>() - ); - connectionQueue.push(new WeakRef(context)); - } - - async newContextRun(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoringTaskNewContext", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - const currentTimeNanos = getCurrentTimeNano(); - // Get entries with key (that is a time in nanos) less than current time. - const processedKeys: number[] = new Array(); - for (const [key, val] of MonitorImpl.newContexts.entries()) { - if (key < currentTimeNanos) { - const queue: Array> = val; - - processedKeys.push(key); - // Each value of found entry is a queue of monitoring contexts awaiting active monitoring. - // Add all contexts to an active monitoring contexts queue. - // Ignore disposed contexts. - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = queue?.shift()) != null) { - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - if (monitorContext && monitorContext.isActive()) { - this.activeContexts.push(monitorContextRef); - } - } - } - } - processedKeys.forEach((key) => { - MonitorImpl.newContexts.delete(key); - }); - await sleep(1000); - } - return; - } catch (err) { - // do nothing, exit task - } - logger.debug(Messages.get("MonitorImpl.stopMonitoringTaskNewContext", this.hostInfo.host)); - } - - async run(): Promise { - logger.debug(Messages.get("MonitorImpl.startMonitoring", this.hostInfo.host)); - - try { - while (!this.isStopped()) { - try { - if (this.activeContexts.length === 0 && !this.hostUnhealthy) { - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout(resolve, MonitorImpl.TASK_SLEEP_MILLIS); - }); - continue; - } - - const statusCheckStartTimeNanos: number = getCurrentTimeNano(); - const isValid = await this.checkConnectionStatus(); - const statusCheckEndTimeNanos: number = getCurrentTimeNano(); - - await this.updateHostHealthStatus(isValid, statusCheckStartTimeNanos, statusCheckEndTimeNanos); - - if (this.hostUnhealthy) { - this.pluginService.setAvailability(this.hostInfo.aliases, HostAvailability.NOT_AVAILABLE); - } - const tmpActiveContexts: WeakRef[] = []; - - let monitorContextRef: WeakRef | undefined; - - while ((monitorContextRef = this.activeContexts?.shift()) != null) { - if (this.isStopped()) { - break; - } - - const monitorContext: MonitorConnectionContext = monitorContextRef?.deref() ?? null; - - if (!monitorContext) { - continue; - } - - if (this.hostUnhealthy) { - // Kill connection - monitorContext.setHostUnhealthy(true); - const clientToAbort = monitorContext.getClient(); - - monitorContext.setInactive(); - if (clientToAbort != null) { - await this.endMonitoringClient(clientToAbort); - - this.abortedConnectionsCounter.inc(); - } - } else if (monitorContext && monitorContext.isActive()) { - tmpActiveContexts.push(monitorContextRef); - } - } - - // activeContexts is empty now and tmpActiveContexts contains all yet active contexts - // Add active contexts back to the queue. - this.activeContexts.push(...tmpActiveContexts); - - const delayMillis = (this.failureDetectionIntervalNanos - (statusCheckEndTimeNanos - statusCheckStartTimeNanos)) / 1000000; - - await new Promise((resolve) => { - this.delayMillisTimeoutId = setTimeout( - resolve, - delayMillis < MonitorImpl.TASK_SLEEP_MILLIS ? MonitorImpl.TASK_SLEEP_MILLIS : delayMillis - ); - }); - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringContinue", error.message)); - } - } - } catch (error: any) { - logger.debug(Messages.get("MonitorImpl.errorDuringMonitoringStop", error.message)); - } finally { - await this.endMonitoringClient(); - } - - logger.debug(Messages.get("MonitorImpl.stopMonitoring", this.hostInfo.host)); - } - - /** - * Check the status of the monitored server by sending a ping. - * - * @return whether the server is still alive and the elapsed time spent checking. - */ - async checkConnectionStatus(): Promise { - const connectContext = this.telemetryFactory.openTelemetryContext("Connection status check", TelemetryTraceLevel.FORCE_TOP_LEVEL); - connectContext.setAttribute("url", this.hostInfo.host); - try { - if (!(await this.pluginService.isClientValid(this.monitoringClient))) { - // Open a new connection. - const monitoringConnProperties: Map = new Map(this.properties); - for (const key of monitoringConnProperties.keys()) { - if (!key.startsWith(WrapperProperties.MONITORING_PROPERTY_PREFIX)) { - continue; - } - monitoringConnProperties.set(key.substring(WrapperProperties.MONITORING_PROPERTY_PREFIX.length), this.properties.get(key)); - monitoringConnProperties.delete(key); - } - - logger.debug(`Opening a monitoring connection to ${this.hostInfo.url}`); - this.monitoringClient = await this.pluginService.forceConnect(this.hostInfo, monitoringConnProperties); - logger.debug(`Successfully opened monitoring connection to ${this.monitoringClient.id} - ${this.hostInfo.url}`); - return true; - } - return true; - } catch (error: any) { - return false; - } - } - - isStopped(): boolean { - return this.stopped; - } - - updateHostHealthStatus(connectionValid: boolean, statusCheckStartNano: number, statusCheckEndNano: number): Promise { - if (!connectionValid) { - this.failureCount++; - - if (this.invalidHostStartTimeNano === 0) { - this.invalidHostStartTimeNano = statusCheckStartNano; - } - - const invalidHostDurationNano = statusCheckEndNano - this.invalidHostStartTimeNano; - const maxInvalidHostDurationNano = this.failureDetectionIntervalNanos * Math.max(0, this.failureDetectionCount - 1); - - if (this.failureCount >= this.failureDetectionCount || invalidHostDurationNano >= maxInvalidHostDurationNano) { - logger.debug(Messages.get("MonitorConnectionContext.hostDead", this.hostInfo.host)); - this.hostUnhealthy = true; - return Promise.resolve(); - } - logger.debug(Messages.get("MonitorConnectionContext.hostNotResponding", this.hostInfo.host)); - return Promise.resolve(); - } - - if (this.failureCount > 0) { - // Host is back alive - logger.debug(Messages.get("MonitorConnectionContext.hostAlive", this.hostInfo.host)); - } - this.failureCount = 0; - this.invalidHostStartTimeNano = 0; - this.hostUnhealthy = false; - } - - async releaseResources() { - this.stopped = true; - clearTimeout(this.delayMillisTimeoutId); - clearTimeout(this.sleepWhenHostHealthyTimeoutId); - this.activeContexts = null; - await this.endMonitoringClient(); - // Allow time for monitor loop to close. - await sleep(500); - } - - async endMonitoringClient(clientToAbort?: ClientWrapper) { - try { - if (clientToAbort) { - await this.pluginService.abortTargetClient(clientToAbort); - } else if (this.monitoringClient) { - await this.pluginService.abortTargetClient(this.monitoringClient); - this.monitoringClient = null; - } - this.stopped = true; - } catch (error: any) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } -} diff --git a/common/lib/plugins/efm2/monitor_connection_context.ts b/common/lib/plugins/efm2/monitor_connection_context.ts deleted file mode 100644 index 156b4196..00000000 --- a/common/lib/plugins/efm2/monitor_connection_context.ts +++ /dev/null @@ -1,59 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { uniqueId } from "../../../logutils"; -import { ClientWrapper } from "../../client_wrapper"; - -/** - * Monitoring context for each connection. This contains each connection's criteria for whether a - * server should be considered unhealthy. The context is shared between the main task and the monitor task. - */ -export class MonitorConnectionContext { - private clientToAbortRef: WeakRef | undefined; - isHostUnhealthy: boolean = false; - id: string = uniqueId("_monitorContext"); - - /** - * Constructor. - * - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - constructor(clientToAbort: ClientWrapper) { - if (clientToAbort) { - this.clientToAbortRef = new WeakRef(clientToAbort); - } - } - - setHostUnhealthy(hostUnhealthy: boolean) { - this.isHostUnhealthy = hostUnhealthy; - } - - shouldAbort(): boolean { - return this.isHostUnhealthy && this.clientToAbortRef != null; - } - - setInactive(): void { - this.clientToAbortRef = null; - } - - getClient(): ClientWrapper | null { - return this.clientToAbortRef?.deref() ?? null; - } - - isActive() { - return !!this.clientToAbortRef?.deref(); - } -} diff --git a/common/lib/plugins/efm2/monitor_service.ts b/common/lib/plugins/efm2/monitor_service.ts deleted file mode 100644 index 40a78817..00000000 --- a/common/lib/plugins/efm2/monitor_service.ts +++ /dev/null @@ -1,157 +0,0 @@ -/* - Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - - Licensed under the Apache License, Version 2.0 (the "License"). - You may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -import { MonitorConnectionContext } from "./monitor_connection_context"; -import { HostInfo } from "../../host_info"; -import { AwsWrapperError } from "../../utils/errors"; -import { Monitor, MonitorImpl } from "./monitor"; -import { WrapperProperties } from "../../wrapper_property"; -import { PluginService } from "../../plugin_service"; -import { Messages } from "../../utils/messages"; -import { TelemetryCounter } from "../../utils/telemetry/telemetry_counter"; -import { TelemetryFactory } from "../../utils/telemetry/telemetry_factory"; -import { ClientWrapper } from "../../client_wrapper"; -import { logger } from "../../../logutils"; -import { SlidingExpirationCacheWithCleanupTask } from "../../utils/sliding_expiration_cache_with_cleanup_task"; - -export interface MonitorService { - startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise; - - /** - * Stop monitoring for a connection represented by the given {@link MonitorConnectionContext}. - * Removes the context from the {@link MonitorImpl}. - * - * @param context The {@link MonitorConnectionContext} representing a connection. - * @param clientToAbort A reference to the connection associated with this context that will be aborted. - */ - stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise; - - releaseResources(): Promise; -} - -export class MonitorServiceImpl implements MonitorService { - private static readonly CACHE_CLEANUP_NANOS = BigInt(60_000_000_000); - - protected static readonly monitors: SlidingExpirationCacheWithCleanupTask = new SlidingExpirationCacheWithCleanupTask( - MonitorServiceImpl.CACHE_CLEANUP_NANOS, - (monitor: Monitor) => monitor.canDispose(), - async (monitor: Monitor) => { - { - try { - await monitor.releaseResources(); - } catch (error) { - // ignore - } - } - }, - "efm2/MonitorServiceImpl.monitors" - ); - private readonly pluginService: PluginService; - private telemetryFactory: TelemetryFactory; - private readonly abortedConnectionsCounter: TelemetryCounter; - monitorSupplier = ( - pluginService: PluginService, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number, - abortedConnectionsCounter: TelemetryCounter - ) => - new MonitorImpl( - pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - abortedConnectionsCounter - ); - - constructor(pluginService: PluginService) { - this.pluginService = pluginService; - this.telemetryFactory = pluginService.getTelemetryFactory(); - this.abortedConnectionsCounter = this.telemetryFactory.createCounter("efm2.connections.aborted"); - } - - async startMonitoring( - clientToAbort: ClientWrapper, - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitor = await this.getMonitor(hostInfo, properties, failureDetectionTimeMillis, failureDetectionIntervalMillis, failureDetectionCount); - - if (monitor) { - const context = new MonitorConnectionContext(clientToAbort); - monitor.startMonitoring(context); - return context; - } - - throw new AwsWrapperError(Messages.get("MonitorService.startMonitoringNullMonitor", hostInfo.host)); - } - - async stopMonitoring(context: MonitorConnectionContext, clientToAbort: ClientWrapper): Promise { - context.setInactive(); - if (context.shouldAbort()) { - try { - await clientToAbort.abort(); - this.abortedConnectionsCounter.inc(); - } catch (error) { - // ignore - logger.debug(Messages.get("MonitorConnectionContext.errorAbortingConnection", error.message)); - } - } - } - - async getMonitor( - hostInfo: HostInfo, - properties: Map, - failureDetectionTimeMillis: number, - failureDetectionIntervalMillis: number, - failureDetectionCount: number - ): Promise { - const monitorKey: string = `${failureDetectionTimeMillis.toString()} ${failureDetectionIntervalMillis.toString()} ${failureDetectionCount.toString()} ${hostInfo.host}`; - const cacheExpirationNanos = BigInt(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.get(properties) * 1_000_000); - return MonitorServiceImpl.monitors.computeIfAbsent( - monitorKey, - () => - this.monitorSupplier( - this.pluginService, - hostInfo, - properties, - failureDetectionTimeMillis, - failureDetectionIntervalMillis, - failureDetectionCount, - this.abortedConnectionsCounter - ), - cacheExpirationNanos - ); - } - - async releaseResources() { - await MonitorServiceImpl.monitors.clear(); - } -} diff --git a/common/lib/profile/driver_configuration_profiles.ts b/common/lib/profile/driver_configuration_profiles.ts index 958d5dea..a9e2ffb6 100644 --- a/common/lib/profile/driver_configuration_profiles.ts +++ b/common/lib/profile/driver_configuration_profiles.ts @@ -17,7 +17,7 @@ import { ConfigurationProfile } from "./configuration_profile"; import { ConfigurationProfilePresetCodes } from "./configuration_profile_codes"; import { WrapperProperties } from "../wrapper_property"; -import { HostMonitoringPluginFactory } from "../plugins/efm/host_monitoring_plugin_factory"; +import { HostMonitoringPluginFactory } from "../plugins/efm/v1/host_monitoring_plugin_factory"; import { AuroraInitialConnectionStrategyFactory } from "../plugins/aurora_initial_connection_strategy_plugin_factory"; import { AuroraConnectionTrackerPluginFactory } from "../plugins/connection_tracker/aurora_connection_tracker_plugin_factory"; import { ReadWriteSplittingPluginFactory } from "../plugins/read_write_splitting/read_write_splitting_plugin_factory"; diff --git a/common/lib/utils/monitoring/monitor.ts b/common/lib/utils/monitoring/monitor.ts index 81dadea4..5bf93ffc 100644 --- a/common/lib/utils/monitoring/monitor.ts +++ b/common/lib/utils/monitoring/monitor.ts @@ -78,7 +78,7 @@ export abstract class AbstractMonitor implements Monitor { this.monitorPromise = this.run(); } - protected async run(): Promise { + async run(): Promise { try { this.state = MonitorState.RUNNING; this.lastActivityTimestampNanos = BigInt(Date.now() * 1_000_000); diff --git a/index.ts b/index.ts index 6292539c..74a840be 100644 --- a/index.ts +++ b/index.ts @@ -30,8 +30,8 @@ export { DefaultPlugin } from "./common/lib/plugins/default_plugin"; export { ReadWriteSplittingPlugin } from "./common/lib/plugins/read_write_splitting/read_write_splitting_plugin"; export { FailoverPlugin } from "./common/lib/plugins/failover/failover_plugin"; export { Failover2Plugin } from "./common/lib/plugins/failover2/failover2_plugin"; -export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/host_monitoring_connection_plugin"; -export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm2/host_monitoring2_connection_plugin"; +export { HostMonitoringConnectionPlugin } from "./common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; +export { HostMonitoring2ConnectionPlugin } from "./common/lib/plugins/efm/v2/host_monitoring2_connection_plugin"; export { DeveloperConnectionPlugin } from "./common/lib/plugins/dev/developer_connection_plugin"; export { BlueGreenPlugin } from "./common/lib/plugins/bluegreen/blue_green_plugin"; export { AuroraConnectionTrackerPlugin } from "./common/lib/plugins/connection_tracker/aurora_connection_tracker_plugin"; diff --git a/tests/integration/container/tests/performance.test.ts b/tests/integration/container/tests/performance.test.ts index f15d1268..43d10ac1 100644 --- a/tests/integration/container/tests/performance.test.ts +++ b/tests/integration/container/tests/performance.test.ts @@ -25,7 +25,7 @@ import { WrapperProperties, PluginManager } from "../../../../index"; import { features } from "./config"; import { PerfStat } from "./utils/perf_stat"; import { PerfTestUtility } from "./utils/perf_util"; -import { MonitorServiceImpl } from "../../../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl as MonitorServiceImpl } from "../../../../common/lib/plugins/efm/base/host_monitor_service"; const itIf = features.includes(TestEnvironmentFeatures.FAILOVER_SUPPORTED) && diff --git a/tests/unit/host_monitoring_plugin.test.ts b/tests/unit/host_monitoring_plugin.test.ts index 02b70dd8..88c7c012 100644 --- a/tests/unit/host_monitoring_plugin.test.ts +++ b/tests/unit/host_monitoring_plugin.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,15 +15,12 @@ */ import { WrapperProperties } from "../../common/lib/wrapper_property"; -import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/host_monitoring_connection_plugin"; -import { DatabaseDialect } from "../../common/lib/database_dialect/database_dialect"; -import { PgDatabaseDialect } from "../../pg/lib/dialect/pg_database_dialect"; +import { HostMonitoringConnectionPlugin } from "../../common/lib/plugins/efm/v1/host_monitoring_connection_plugin"; import { anything, instance, mock, reset, verify, when } from "ts-mockito"; -import { RdsUtils } from "../../common/lib/utils/rds_utils"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; import { PluginServiceImpl } from "../../common/lib/plugin_service"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { AwsWrapperError, HostAvailability, HostInfo } from "../../common/lib"; +import { ConnectionContext, ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { AwsWrapperError, HostInfo } from "../../common/lib"; import { RdsUrlType } from "../../common/lib/utils/rds_url_type"; import { HostChangeOptions } from "../../common/lib/host_change_options"; import { OldConnectionSuggestionAction } from "../../common/lib/old_connection_suggestion_action"; @@ -31,6 +28,8 @@ import { Messages } from "../../common/lib/utils/messages"; import { AwsPGClient } from "../../pg/lib"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; const FAILURE_DETECTION_TIME = 10; const FAILURE_DETECTION_INTERVAL = 100; @@ -38,12 +37,10 @@ const FAILURE_DETECTION_COUNT = 5; const MONITOR_METHOD_NAME = "query"; const NO_MONITOR_METHOD_NAME = "end"; -const mockDialect: DatabaseDialect = mock(PgDatabaseDialect); -const mockMonitorService: MonitorServiceImpl = mock(MonitorServiceImpl); -const mockMonitorConnectionContext: MonitorConnectionContext = mock(MonitorConnectionContext); +const mockMonitorService: HostMonitorServiceImpl = mock(HostMonitorServiceImpl); +const mockConnectionContext: ConnectionContextImpl = mock(ConnectionContextImpl); const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); const mockClient: AwsPGClient = mock(AwsPGClient); -const mockRdsUtils = mock(RdsUtils); const mockHostInfo1 = mock(HostInfo); const mockHostInfo2 = mock(HostInfo); @@ -59,18 +56,19 @@ function incrementQueryCounter() { const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); function initDefaultMockReturns() { - when(mockDialect.getHostAliasQuery()).thenReturn("any"); - when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( - instance(mockMonitorConnectionContext) + when(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).thenResolve( + instance(mockConnectionContext) ); + when(mockConnectionContext.isActiveContext()).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(false); when(mockHostInfo1.host).thenReturn("host"); when(mockHostInfo1.port).thenReturn(1234); when(mockHostInfo2.host).thenReturn("host"); when(mockHostInfo2.port).thenReturn(1234); when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); + when(mockPluginService.getRoutedHostInfo()).thenReturn(instance(mockHostInfo1)); when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockClient.targetClient).thenReturn(mockClientWrapper); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_INSTANCE); properties.set(WrapperProperties.FAILURE_DETECTION_ENABLED.name, true); properties.set(WrapperProperties.FAILURE_DETECTION_TIME_MS.name, FAILURE_DETECTION_TIME); @@ -79,13 +77,18 @@ function initDefaultMockReturns() { } function initializePlugin() { - plugin = new HostMonitoringConnectionPlugin(instance(mockPluginService), properties, instance(mockRdsUtils), instance(mockMonitorService)); + const servicesContainer = { + pluginService: instance(mockPluginService), + telemetryFactory: new NullTelemetryFactory() + } as unknown as FullServicesContainer; + + plugin = new HostMonitoringConnectionPlugin(servicesContainer, properties, instance(mockMonitorService)); } describe("host monitoring plugin test", () => { beforeEach(() => { reset(mockMonitorService); - reset(mockMonitorConnectionContext); + reset(mockConnectionContext); reset(mockPluginService); reset(mockClient); @@ -99,7 +102,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -108,7 +111,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(NO_MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).never(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).never(); verify(mockMonitorService.stopMonitoring(anything())).never(); expect(queryCounter).toBe(1); }); @@ -117,7 +120,7 @@ describe("host monitoring plugin test", () => { initializePlugin(); await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything(), anything())).once(); + verify(mockMonitorService.startMonitoring(anything(), anything(), anything(), anything(), anything(), anything())).once(); verify(mockMonitorService.stopMonitoring(anything())).once(); expect(queryCounter).toBe(1); }); @@ -126,55 +129,25 @@ describe("host monitoring plugin test", () => { initializePlugin(); const expectedError = new AwsWrapperError("Error thrown during isClientValid"); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); when(mockPluginService.isClientValid(mockClientWrapper)).thenThrow(expectedError); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); }); - it("execute cleanup when abort connection throws error", async () => { + it("execute cleanup when connection is invalid throws unavailable host error", async () => { initializePlugin(); when(mockPluginService.isClientValid(mockClientWrapper)).thenResolve(false); - when(mockMonitorConnectionContext.isHostUnhealthy).thenReturn(true); + when(mockConnectionContext.isHostUnhealthy()).thenReturn(true); const expectedError = new AwsWrapperError(Messages.get("HostMonitoringConnectionPlugin.unavailableHost", "host")); await expect(plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {})).rejects.toThrow(expectedError); - verify(mockPluginService.setAvailability(anything(), HostAvailability.NOT_AVAILABLE)).once(); }); - it("test connect", async () => { + it("notify connection changed resets monitoring host info", async () => { initializePlugin(); - when(mockRdsUtils.identifyRdsType(anything())).thenReturn(RdsUrlType.RDS_WRITER_CLUSTER); - await plugin.connect(instance(mockHostInfo1), properties, true, () => { - return Promise.resolve(mockClientWrapper); - }); - verify(mockPluginService.fillAliases(anything(), anything())).once(); + const result = await plugin.notifyConnectionChanged(new Set([HostChangeOptions.HOSTNAME])); + expect(result).toBe(OldConnectionSuggestionAction.NO_OPINION); }); - - it.each([[HostChangeOptions.WENT_DOWN], [HostChangeOptions.HOST_DELETED]])( - "notify connection changed when node went down", - async (options: HostChangeOptions) => { - initializePlugin(); - - await plugin.execute(MONITOR_METHOD_NAME, incrementQueryCounter, {}); - - const aliases1 = new Set(["alias1", "alias2"]); - const aliases2 = new Set(["alias3", "alias4"]); - - when(mockHostInfo1.allAliases).thenReturn(aliases1); - when(mockHostInfo2.allAliases).thenReturn(aliases2); - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo1)); - - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NodeKeys should contain {"alias1", "alias2"}. - verify(mockMonitorService.stopMonitoringForAllConnections(aliases1)).once(); - - when(mockPluginService.getCurrentHostInfo()).thenReturn(instance(mockHostInfo2)); - expect(await plugin.notifyConnectionChanged(new Set([options]))).toBe(OldConnectionSuggestionAction.NO_OPINION); - // NotifyConnectionChanged should reset the monitoringHostSpec. - // NodeKeys should contain {"alias3", "alias4"} - verify(mockMonitorService.stopMonitoringForAllConnections(aliases2)).once(); - } - ); }); diff --git a/tests/unit/monitor_connection_context.test.ts b/tests/unit/monitor_connection_context.test.ts index 47e5eab0..3f11b3de 100644 --- a/tests/unit/monitor_connection_context.test.ts +++ b/tests/unit/monitor_connection_context.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,113 +14,131 @@ limitations under the License. */ -import { instance, mock, spy, verify } from "ts-mockito"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { PluginServiceImpl } from "../../common/lib/plugin_service"; +import { mock } from "ts-mockito"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; +import { HostInfo } from "../../common/lib"; +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { getCurrentTimeNano } from "../../common/lib/utils/utils"; -const mockPluginService: PluginServiceImpl = mock(PluginServiceImpl); -const mockMonitor = mock(MonitorImpl); -const mockTargetClient = { - end() { - throw new Error("close"); - } -}; -const mockClientWrapper = mock(MySQLClientWrapper); +const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; const VALIDATION_INTERVAL_MILLIS = 50; -let context: MonitorConnectionContext; +let context: ConnectionContextImpl; -describe("monitor connection context test", () => { +describe("connection context test", () => { beforeEach(() => { - context = new MonitorConnectionContext( - instance(mockMonitor), - null, + context = new ConnectionContextImpl( + mockClientWrapper, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT, - instance(mockPluginService), new NullTelemetryFactory().createCounter("counter") ); }); it("isHostUnhealthy with valid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", true, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(0); + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + // Timestamps must exceed the grace period relative to context's startMonitorTimeNano (hrtime-based) + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, true); + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy with invalid connection", async () => { - const currentTimeNano = Date.now(); - await context.setConnectionValid("test-node", false, currentTimeNano, currentTimeNano); - expect(context.isHostUnhealthy).toBe(false); - expect(context.failureCount).toBe(1); + it("isHostUnhealthy with invalid connection - single failure", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); + const checkStart = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; + const checkEnd = checkStart + 1_000_000; + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); + // Single failure doesn't exceed threshold + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("isHostUnhealthy exceeds failure detection count", async () => { - let currentTimeNano = Date.now(); - context.failureCount = 0; - context.resetInvalidHostStartTimeNano(); + it("isHostUnhealthy exceeds failure detection threshold", async () => { + const ctx = new ConnectionContextImpl( + mockClientWrapper, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - for (let i = 0; i < 5; i++) { - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; + // Use timestamps far enough in the future to exceed the grace period (failureDetectionTimeMillis). + // The context's startMonitorTimeNano is hrtime-based (~now), so we simulate checks starting after the grace period. + const futureStartNano = getCurrentTimeNano() + (FAILURE_DETECTION_TIME_MILLIS + 100) * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - if (i >= FAILURE_DETECTION_COUNT - 1) { - expect(context.isHostUnhealthy).toBe(true); - } else { - expect(context.isHostUnhealthy).toBe(false); - } + // maxInvalidHostDuration = failureDetectionIntervalMillis * failureDetectionCount = 100 * 3 = 300ms + // Each check spans FAILURE_DETECTION_INTERVAL_MILLIS (100ms), so after 3 checks the duration exceeds threshold. + for (let i = 0; i < FAILURE_DETECTION_COUNT + 2; i++) { + const checkStart = futureStartNano + i * FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + const checkEnd = checkStart + FAILURE_DETECTION_INTERVAL_MILLIS * 1_000_000; + + await ctx.updateConnectionStatus("test-node", checkStart, checkEnd, false); - currentTimeNano += VALIDATION_INTERVAL_MILLIS * 1_000_000; + if (ctx.isHostUnhealthy()) { + break; + } } - const statusCheckStartTime = currentTimeNano; - const statusCheckEndTime = currentTimeNano + VALIDATION_INTERVAL_MILLIS * 1_000_000; - await context.setConnectionValid("test-node", false, statusCheckStartTime, statusCheckEndTime); - expect(context.isHostUnhealthy).toBe(true); + expect(ctx.isHostUnhealthy()).toBe(true); }); - it.each([[true], [false]])("updateConnectionStatus", async (isValid: boolean) => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - FAILURE_DETECTION_TIME_MILLIS * 1_000_000; + it("updateConnectionStatus skips within grace period", async () => { + // Use a context with a long grace period + const ctx = new ConnectionContextImpl( + mockClientWrapper, + 30000, // 30s grace + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT, + new NullTelemetryFactory().createCounter("counter") + ); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, isValid); + // Call with a time that's within the grace period (elapsed < failureDetectionTime) + const now = getCurrentTimeNano(); + await ctx.updateConnectionStatus("test-node", now, now + 1_000_000, false); - verify(contextSpy.setConnectionValid("test-node", isValid, statusCheckStartTime, currentTimeNano)).once(); + // Should not have detected unhealthy since we're within grace period + expect(ctx.isHostUnhealthy()).toBe(false); }); - it("updateConnectionStatus - inactive context", async () => { - const currentTimeNano = Date.now(); - const statusCheckStartTime = Date.now() - 1000; - context.isActiveContext = false; + it("updateConnectionStatus skips for inactive context", async () => { + context.setInactive(); + expect(context.isActiveContext()).toBe(false); - const contextSpy = spy(context); - await context.updateConnectionStatus("test-node", statusCheckStartTime, currentTimeNano, true); + const now = getCurrentTimeNano(); + await context.updateConnectionStatus("test-node", now, now + 100_000_000, false); - verify(contextSpy.setConnectionValid("test-node", true, statusCheckStartTime, currentTimeNano)).never(); + // Should remain not-unhealthy since the context is inactive + expect(context.isHostUnhealthy()).toBe(false); }); - it("abort client ignores error", async () => { - context = new MonitorConnectionContext( - instance(mockMonitor), - mockClientWrapper, - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("counter") - ); + it("setInactive marks context inactive", () => { + expect(context.isActiveContext()).toBe(true); + context.setInactive(); + expect(context.isActiveContext()).toBe(false); + }); + it("abortConnection does nothing for inactive context", async () => { + context.setInactive(); + // Should not throw await context.abortConnection(); }); }); diff --git a/tests/unit/monitor_impl.test.ts b/tests/unit/monitor_impl.test.ts index 8ba1116c..2ba18316 100644 --- a/tests/unit/monitor_impl.test.ts +++ b/tests/unit/monitor_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,61 +14,56 @@ limitations under the License. */ -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { anything, instance, mock, reset, spy, verify, when } from "ts-mockito"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { ConnectionContextImpl } from "../../common/lib/plugins/efm/base/connection_context"; +import { anything, instance, mock, reset, when } from "ts-mockito"; import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo } from "../../common/lib"; -import { AwsClient } from "../../common/lib/aws_client"; -import { MonitorConnectionContext } from "../../common/lib/plugins/efm/monitor_connection_context"; import { sleep } from "../../common/lib/utils/utils"; import { ClientWrapper } from "../../common/lib/client_wrapper"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; -class MonitorImplTest extends MonitorImpl { +class MonitorImplTest extends HostMonitorImpl { constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); } - override startRun() { - // do nothing + override checkConnectionStatus() { + // Exposes the protected checkConnectionStatus method for testing purposes. + return super.checkConnectionStatus(); } } const mockPluginService = mock(PluginServiceImpl); const mockHostInfo = mock(HostInfo); -const mockClient = mock(AwsClient); const mockClientWrapper: ClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const SHORT_INTERVAL_MILLIS = 30; const properties = new Map(); -let monitor: MonitorImpl; -let monitorSpy: MonitorImpl; +let monitor: MonitorImplTest; describe("monitor impl test", () => { beforeEach(() => { reset(mockPluginService); - when(mockPluginService.getCurrentClient()).thenReturn(instance(mockClient)); when(mockPluginService.forceConnect(anything(), anything())).thenResolve(mockClientWrapper); when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); monitor = new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); - monitorSpy = spy(monitor); }); - afterEach(() => { - monitor.releaseResources(); + afterEach(async () => { + await monitor.releaseResources(); }); it("is client healthy with no existing client", async () => { - const status = await monitor.checkConnectionStatus(); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); - verify(mockPluginService.forceConnect(anything(), anything())).once(); - expect(status.isValid).toBe(true); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + expect(isValid).toBe(true); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("is client healthy with existing client", async () => { @@ -77,13 +72,11 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status1 = await monitor.checkConnectionStatus(); - expect(status1.isValid).toBe(true); + const [isValid1] = await monitor.checkConnectionStatus(); + expect(isValid1).toBe(true); - const status2 = await monitor.checkConnectionStatus(); - expect(status2.isValid).toBe(true); - - verify(mockPluginService.isClientValid(mockClientWrapper)).twice(); + const [isValid2] = await monitor.checkConnectionStatus(); + expect(isValid2).toBe(true); }); it("is client healthy with error", async () => { @@ -92,31 +85,23 @@ describe("monitor impl test", () => { // Start up a monitoring client. await monitor.checkConnectionStatus(); - const status = await monitor.checkConnectionStatus(); - expect(status.isValid).toBe(false); - expect(status.elapsedTimeNano).toBeGreaterThanOrEqual(0); + const [isValid, elapsedTimeNano] = await monitor.checkConnectionStatus(); + expect(isValid).toBe(false); + expect(elapsedTimeNano).toBeGreaterThanOrEqual(0); }); it("run without context", async () => { - // Should end by itself. + // Should end by itself (monitorDisposalTimeMillis = 0). await monitor.run(); - verify(monitorSpy.checkConnectionStatus()).never(); }); it("run with context", async () => { - const monitorContextInstance = new MonitorConnectionContext( - monitor, - mockClientWrapper, - 30000, - 5000, - 3, - instance(mockPluginService), - new NullTelemetryFactory().createCounter("name") - ); - monitor.startMonitoring(monitorContextInstance); - // Should end by itself. + when(mockPluginService.isClientValid(anything())).thenResolve(true); + + const context = new ConnectionContextImpl(mockClientWrapper, 30000, 5000, 3, new NullTelemetryFactory().createCounter("counter")); + monitor.startMonitoring(context); monitor.run(); await sleep(SHORT_INTERVAL_MILLIS); - monitor.stopMonitoring(monitorContextInstance); + monitor.stopMonitoring(context); }); }); diff --git a/tests/unit/monitor_service_impl.test.ts b/tests/unit/monitor_service_impl.test.ts index 676d8c2b..4093196c 100644 --- a/tests/unit/monitor_service_impl.test.ts +++ b/tests/unit/monitor_service_impl.test.ts @@ -1,12 +1,12 @@ /* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - + Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,203 +14,168 @@ limitations under the License. */ -import { anything, capture, instance, mock, reset, verify, when } from "ts-mockito"; -import { MonitorImpl } from "../../common/lib/plugins/efm/monitor"; -import { MonitorServiceImpl } from "../../common/lib/plugins/efm/monitor_service"; -import { PluginService, PluginServiceImpl } from "../../common/lib/plugin_service"; +import { instance, mock, reset, when } from "ts-mockito"; +import { HostMonitorServiceImpl } from "../../common/lib/plugins/efm/base/host_monitor_service"; +import { HostMonitorImpl } from "../../common/lib/plugins/efm/base/host_monitor"; +import { PluginServiceImpl } from "../../common/lib/plugin_service"; import { HostInfo, HostInfoBuilder } from "../../common/lib"; import { SimpleHostAvailabilityStrategy } from "../../common/lib/host_availability/simple_host_availability_strategy"; import { NullTelemetryFactory } from "../../common/lib/utils/telemetry/null_telemetry_factory"; import { MySQLClientWrapper } from "../../common/lib/mysql_client_wrapper"; - -class MonitorImplTest extends MonitorImpl { - constructor(pluginService: PluginService, hostInfo: HostInfo, properties: Map, monitorDisposalTimeMillis: number) { - super(pluginService, hostInfo, properties, monitorDisposalTimeMillis); - } - - override startRun() { - // do nothing - } -} - -const mockPluginService = mock(PluginServiceImpl); -const mockMonitorA = mock(MonitorImpl); -const mockMonitorB = mock(MonitorImpl); -const mockHostInfo = mock(HostInfo); -const mockClientWrapper = mock(MySQLClientWrapper); +import { MySQL2DriverDialect } from "../../mysql/lib/dialect/mysql2_driver_dialect"; +import { MonitorServiceImpl } from "../../common/lib/utils/monitoring/monitor_service"; +import { FullServicesContainer } from "../../common/lib/utils/full_services_container"; +import { BatchingEventPublisher } from "../../common/lib/utils/events/batching_event_publisher"; +import { WrapperProperties } from "../../common/lib/wrapper_property"; const FAILURE_DETECTION_TIME_MILLIS = 10; const FAILURE_DETECTION_INTERVAL_MILLIS = 100; const FAILURE_DETECTION_COUNT = 3; -const NODE_KEYS = new Set(["any"]); -const properties = new Map(); -let monitorService: MonitorServiceImpl; +const mockPluginService = mock(PluginServiceImpl); + +const properties: Map = new Map(); +properties.set(WrapperProperties.MONITOR_DISPOSAL_TIME_MS.name, 600000); + +let monitorService: HostMonitorServiceImpl; +let coreMonitorService: MonitorServiceImpl; +let servicesContainer: FullServicesContainer; +let eventPublisher: BatchingEventPublisher; describe("monitor service impl test", () => { beforeEach(() => { reset(mockPluginService); - reset(mockMonitorA); - when(mockPluginService.getTelemetryFactory()).thenReturn(new NullTelemetryFactory()); - monitorService = new MonitorServiceImpl(instance(mockPluginService)); - monitorService.monitorSupplier = () => new MonitorImplTest(instance(mockPluginService), instance(mockHostInfo), properties, 0); + const telemetryFactory = new NullTelemetryFactory(); + when(mockPluginService.getTelemetryFactory()).thenReturn(telemetryFactory); + when(mockPluginService.isClientValid(undefined)).thenResolve(false); + + eventPublisher = new BatchingEventPublisher(60_000); + coreMonitorService = new MonitorServiceImpl(eventPublisher); + + servicesContainer = { + pluginService: instance(mockPluginService), + monitorService: coreMonitorService, + telemetryFactory: telemetryFactory + } as unknown as FullServicesContainer; + + monitorService = new HostMonitorServiceImpl(servicesContainer); }); afterEach(async () => { await monitorService.releaseResources(); + await coreMonitorService.releaseResources(); + eventPublisher.releaseResources(); }); - it("start monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring creates context", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - await monitorService.startMonitoring( + const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - }); - it("start monitoring called multiple times", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); - - const runs = 5; - - for (let i = 0; i < runs; i++) { - await monitorService.startMonitoring( - mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), - FAILURE_DETECTION_TIME_MILLIS, - FAILURE_DETECTION_INTERVAL_MILLIS, - FAILURE_DETECTION_COUNT - ); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg).toBeDefined(); - expect(arg).not.toBeNull(); - } + expect(context).toBeDefined(); + expect(context).not.toBeNull(); + expect(context.isActiveContext()).toBe(true); + expect(context.isHostUnhealthy()).toBe(false); }); - it("start and stop monitoring", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("stop monitoring sets context inactive", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); const context = await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); - verify(mockMonitorA.stopMonitoring(anything())).once(); + monitorService.stopMonitoring(context); + expect(context.isActiveContext()).toBe(false); }); - it("stop monitoring called twice", async () => { - monitorService.monitorSupplier = () => instance(mockMonitorA); + it("start monitoring reuses monitor for same host", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const context = await monitorService.startMonitoring( + await monitorService.startMonitoring( mockClientWrapper, - NODE_KEYS, - instance(mockHostInfo), - new Map(), + hostInfo, + properties, FAILURE_DETECTION_TIME_MILLIS, FAILURE_DETECTION_INTERVAL_MILLIS, FAILURE_DETECTION_COUNT ); - await monitorService.stopMonitoring(context); - const arg = capture(mockMonitorA.startMonitoring).last(); - expect(arg[0]).toBe(context); + const monitor1 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); - await monitorService.stopMonitoring(context); - verify(mockMonitorA.stopMonitoring(anything())).twice(); - }); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - it("stop monitoring for all connections with invalid node keys", async () => { - monitorService.stopMonitoringForAllConnections(new Set()); - monitorService.stopMonitoringForAllConnections(new Set(["foo"])); + const monitor2 = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor2).toBe(monitor1); }); - it("stop monitoring for all connections", async () => { - const keysA = new Set(["monitorA"]); - const keysB = new Set(["monitorB"]); + it("start monitoring creates separate monitors for different hosts", async () => { + const hostInfoA = new HostInfoBuilder({ host: "host-a", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const hostInfoB = new HostInfoBuilder({ host: "host-b", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - monitorService.monitorSupplier = () => instance(mockMonitorA); - await monitorService.getMonitor( - keysA, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoA, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.monitorSupplier = () => instance(mockMonitorB); - await monitorService.getMonitor( - keysB, - new HostInfoBuilder({ host: "test", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(), - properties + await monitorService.startMonitoring( + mockClientWrapper, + hostInfoB, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT ); - monitorService.stopMonitoringForAllConnections(keysA); - verify(mockMonitorA.clearContexts()).once(); - - monitorService.stopMonitoringForAllConnections(keysB); - verify(mockMonitorB.clearContexts()).once(); + const monitorA = coreMonitorService.get(HostMonitorImpl, hostInfoA.host); + const monitorB = coreMonitorService.get(HostMonitorImpl, hostInfoB.host); + expect(monitorA).not.toBe(monitorB); }); - it("getMonitor called with multiple hosts in keys", async () => { - const keysA = new Set(["host1.domain", "host2.domain"]); - const keysB = new Set(["host2.domain"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - }); - - it("getMonitor called with different host keys", async () => { - const keys = new Set(["hostNEW.domain"]); - - const monitorOne = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); + it("release resources clears monitors", async () => { + const hostInfo = new HostInfoBuilder({ host: "test-host", hostAvailabilityStrategy: new SimpleHostAvailabilityStrategy() }).build(); + const mockClientWrapper = new MySQLClientWrapper(undefined, mock(HostInfo), new Map(), new MySQL2DriverDialect()); - const monitorOneDupe = await monitorService.getMonitor(keys, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); - - const monitorTwo = await monitorService.getMonitor(NODE_KEYS, instance(mockHostInfo), properties); - expect(monitorTwo).not.toBeNull(); - expect(monitorTwo).not.toBe(monitorOne); - }); - - it("getMonitor called with same keys in different sets", async () => { - const keysA = new Set(["hostA"]); - const keysB = new Set(["hostA", "hostB"]); - const keysC = new Set(["hostB"]); - - const monitorOne = await monitorService.getMonitor(keysA, instance(mockHostInfo), properties); - expect(monitorOne).not.toBeNull(); - - const monitorOneDupe = await monitorService.getMonitor(keysB, instance(mockHostInfo), properties); - expect(monitorOneDupe).toBe(monitorOne); + await monitorService.startMonitoring( + mockClientWrapper, + hostInfo, + properties, + FAILURE_DETECTION_TIME_MILLIS, + FAILURE_DETECTION_INTERVAL_MILLIS, + FAILURE_DETECTION_COUNT + ); - const monitorOneDupeTwo = await monitorService.getMonitor(keysC, instance(mockHostInfo), properties); - expect(monitorOneDupeTwo).toBe(monitorOne); - }); + await monitorService.releaseResources(); - it("startMonitoring with no host keys", async () => { - const keys: Set = new Set(); - expect(await monitorService.getMonitor(keys, instance(mockHostInfo), properties)).toBeNull(); + const monitor = coreMonitorService.get(HostMonitorImpl, hostInfo.host); + expect(monitor).toBeNull(); }); });